1 //===-- AMDGPUAsmParser.cpp - Parse SI asm to MCInst instructions ---------===//
2 //
3 //                     The LLVM Compiler Infrastructure
4 //
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
7 //
8 //===----------------------------------------------------------------------===//
9 
10 #include "AMDKernelCodeT.h"
11 #include "MCTargetDesc/AMDGPUMCTargetDesc.h"
12 #include "MCTargetDesc/AMDGPUTargetStreamer.h"
13 #include "SIDefines.h"
14 #include "Utils/AMDGPUBaseInfo.h"
15 #include "Utils/AMDKernelCodeTUtils.h"
16 #include "Utils/AMDGPUAsmUtils.h"
17 #include "llvm/ADT/APFloat.h"
18 #include "llvm/ADT/STLExtras.h"
19 #include "llvm/ADT/SmallBitVector.h"
20 #include "llvm/ADT/SmallString.h"
21 #include "llvm/ADT/StringSwitch.h"
22 #include "llvm/ADT/Twine.h"
23 #include "llvm/MC/MCContext.h"
24 #include "llvm/MC/MCExpr.h"
25 #include "llvm/MC/MCInst.h"
26 #include "llvm/MC/MCInstrInfo.h"
27 #include "llvm/MC/MCParser/MCAsmLexer.h"
28 #include "llvm/MC/MCParser/MCAsmParser.h"
29 #include "llvm/MC/MCParser/MCParsedAsmOperand.h"
30 #include "llvm/MC/MCParser/MCTargetAsmParser.h"
31 #include "llvm/MC/MCRegisterInfo.h"
32 #include "llvm/MC/MCStreamer.h"
33 #include "llvm/MC/MCSubtargetInfo.h"
34 #include "llvm/MC/MCSymbolELF.h"
35 #include "llvm/Support/Debug.h"
36 #include "llvm/Support/ELF.h"
37 #include "llvm/Support/SourceMgr.h"
38 #include "llvm/Support/TargetRegistry.h"
39 #include "llvm/Support/raw_ostream.h"
40 #include "llvm/Support/MathExtras.h"
41 
42 using namespace llvm;
43 
44 namespace {
45 
46 struct OptionalOperand;
47 
48 enum RegisterKind { IS_UNKNOWN, IS_VGPR, IS_SGPR, IS_TTMP, IS_SPECIAL };
49 
50 class AMDGPUOperand : public MCParsedAsmOperand {
51   enum KindTy {
52     Token,
53     Immediate,
54     Register,
55     Expression
56   } Kind;
57 
58   SMLoc StartLoc, EndLoc;
59 
60 public:
61   AMDGPUOperand(enum KindTy K) : MCParsedAsmOperand(), Kind(K) {}
62 
63   MCContext *Ctx;
64 
65   typedef std::unique_ptr<AMDGPUOperand> Ptr;
66 
67   enum ImmTy {
68     ImmTyNone,
69     ImmTyGDS,
70     ImmTyOffen,
71     ImmTyIdxen,
72     ImmTyAddr64,
73     ImmTyOffset,
74     ImmTyOffset0,
75     ImmTyOffset1,
76     ImmTyGLC,
77     ImmTySLC,
78     ImmTyTFE,
79     ImmTyClampSI,
80     ImmTyOModSI,
81     ImmTyDppCtrl,
82     ImmTyDppRowMask,
83     ImmTyDppBankMask,
84     ImmTyDppBoundCtrl,
85     ImmTySdwaDstSel,
86     ImmTySdwaSrc0Sel,
87     ImmTySdwaSrc1Sel,
88     ImmTySdwaDstUnused,
89     ImmTyDMask,
90     ImmTyUNorm,
91     ImmTyDA,
92     ImmTyR128,
93     ImmTyLWE,
94     ImmTyHwreg,
95     ImmTySendMsg,
96   };
97 
98   struct TokOp {
99     const char *Data;
100     unsigned Length;
101   };
102 
103   struct ImmOp {
104     bool IsFPImm;
105     ImmTy Type;
106     int64_t Val;
107     int Modifiers;
108   };
109 
110   struct RegOp {
111     unsigned RegNo;
112     int Modifiers;
113     const MCRegisterInfo *TRI;
114     const MCSubtargetInfo *STI;
115     bool IsForcedVOP3;
116   };
117 
118   union {
119     TokOp Tok;
120     ImmOp Imm;
121     RegOp Reg;
122     const MCExpr *Expr;
123   };
124 
125   void addImmOperands(MCInst &Inst, unsigned N, bool ApplyModifiers = true) const {
126     if (Imm.Type == ImmTyNone && ApplyModifiers && Imm.Modifiers != 0) {
127       // Apply modifiers to immediate value
128       int64_t Val = Imm.Val;
129       bool Negate = Imm.Modifiers & 0x1;
130       bool Abs = Imm.Modifiers & 0x2;
131       if (Imm.IsFPImm) {
132         APFloat F(BitsToFloat(Val));
133         if (Abs) {
134           F.clearSign();
135         }
136         if (Negate) {
137           F.changeSign();
138         }
139         Val = F.bitcastToAPInt().getZExtValue();
140       } else {
141         Val = Abs ? std::abs(Val) : Val;
142         Val = Negate ? -Val : Val;
143       }
144       Inst.addOperand(MCOperand::createImm(Val));
145     } else {
146       Inst.addOperand(MCOperand::createImm(getImm()));
147     }
148   }
149 
150   StringRef getToken() const {
151     return StringRef(Tok.Data, Tok.Length);
152   }
153 
154   void addRegOperands(MCInst &Inst, unsigned N) const {
155     Inst.addOperand(MCOperand::createReg(AMDGPU::getMCReg(getReg(), *Reg.STI)));
156   }
157 
158   void addRegOrImmOperands(MCInst &Inst, unsigned N) const {
159     if (isRegKind())
160       addRegOperands(Inst, N);
161     else
162       addImmOperands(Inst, N);
163   }
164 
165   void addRegOrImmWithInputModsOperands(MCInst &Inst, unsigned N) const {
166     if (isRegKind()) {
167       Inst.addOperand(MCOperand::createImm(Reg.Modifiers));
168       addRegOperands(Inst, N);
169     } else {
170       Inst.addOperand(MCOperand::createImm(Imm.Modifiers));
171       addImmOperands(Inst, N, false);
172     }
173   }
174 
175   void addSoppBrTargetOperands(MCInst &Inst, unsigned N) const {
176     if (isImm())
177       addImmOperands(Inst, N);
178     else {
179       assert(isExpr());
180       Inst.addOperand(MCOperand::createExpr(Expr));
181     }
182   }
183 
184   bool isToken() const override {
185     return Kind == Token;
186   }
187 
188   bool isImm() const override {
189     return Kind == Immediate;
190   }
191 
192   bool isInlinableImm() const {
193     if (!isImm() || Imm.Type != AMDGPUOperand::ImmTyNone /* Only plain
194       immediates are inlinable (e.g. "clamp" attribute is not) */ )
195       return false;
196     // TODO: We should avoid using host float here. It would be better to
197     // check the float bit values which is what a few other places do.
198     // We've had bot failures before due to weird NaN support on mips hosts.
199     const float F = BitsToFloat(Imm.Val);
200     // TODO: Add 1/(2*pi) for VI
201     return (Imm.Val <= 64 && Imm.Val >= -16) ||
202            (F == 0.0 || F == 0.5 || F == -0.5 || F == 1.0 || F == -1.0 ||
203            F == 2.0 || F == -2.0 || F == 4.0 || F == -4.0);
204   }
205 
206   int64_t getImm() const {
207     return Imm.Val;
208   }
209 
210   enum ImmTy getImmTy() const {
211     assert(isImm());
212     return Imm.Type;
213   }
214 
215   bool isRegKind() const {
216     return Kind == Register;
217   }
218 
219   bool isReg() const override {
220     return Kind == Register && Reg.Modifiers == 0;
221   }
222 
223   bool isRegOrImmWithInputMods() const {
224     return Kind == Register || isInlinableImm();
225   }
226 
227   bool isImmTy(ImmTy ImmT) const {
228     return isImm() && Imm.Type == ImmT;
229   }
230 
231   bool isClampSI() const {
232     return isImmTy(ImmTyClampSI);
233   }
234 
235   bool isOModSI() const {
236     return isImmTy(ImmTyOModSI);
237   }
238 
239   bool isImmModifier() const {
240     return Kind == Immediate && Imm.Type != ImmTyNone;
241   }
242 
243   bool isDMask() const {
244     return isImmTy(ImmTyDMask);
245   }
246 
247   bool isUNorm() const { return isImmTy(ImmTyUNorm); }
248   bool isDA() const { return isImmTy(ImmTyDA); }
249   bool isR128() const { return isImmTy(ImmTyUNorm); }
250   bool isLWE() const { return isImmTy(ImmTyLWE); }
251 
252   bool isMod() const {
253     return isClampSI() || isOModSI();
254   }
255 
256   bool isOffen() const { return isImmTy(ImmTyOffen); }
257   bool isIdxen() const { return isImmTy(ImmTyIdxen); }
258   bool isAddr64() const { return isImmTy(ImmTyAddr64); }
259   bool isOffset() const { return isImmTy(ImmTyOffset) && isUInt<16>(getImm()); }
260   bool isOffset0() const { return isImmTy(ImmTyOffset0) && isUInt<16>(getImm()); }
261   bool isOffset1() const { return isImmTy(ImmTyOffset1) && isUInt<8>(getImm()); }
262   bool isGDS() const { return isImmTy(ImmTyGDS); }
263   bool isGLC() const { return isImmTy(ImmTyGLC); }
264   bool isSLC() const { return isImmTy(ImmTySLC); }
265   bool isTFE() const { return isImmTy(ImmTyTFE); }
266 
267   bool isBankMask() const {
268     return isImmTy(ImmTyDppBankMask);
269   }
270 
271   bool isRowMask() const {
272     return isImmTy(ImmTyDppRowMask);
273   }
274 
275   bool isBoundCtrl() const {
276     return isImmTy(ImmTyDppBoundCtrl);
277   }
278 
279   bool isSDWADstSel() const {
280     return isImmTy(ImmTySdwaDstSel);
281   }
282 
283   bool isSDWASrc0Sel() const {
284     return isImmTy(ImmTySdwaSrc0Sel);
285   }
286 
287   bool isSDWASrc1Sel() const {
288     return isImmTy(ImmTySdwaSrc1Sel);
289   }
290 
291   bool isSDWADstUnused() const {
292     return isImmTy(ImmTySdwaDstUnused);
293   }
294 
295   void setModifiers(unsigned Mods) {
296     assert(isReg() || (isImm() && Imm.Modifiers == 0));
297     if (isReg())
298       Reg.Modifiers = Mods;
299     else
300       Imm.Modifiers = Mods;
301   }
302 
303   bool hasModifiers() const {
304     assert(isRegKind() || isImm());
305     return isRegKind() ? Reg.Modifiers != 0 : Imm.Modifiers != 0;
306   }
307 
308   unsigned getReg() const override {
309     return Reg.RegNo;
310   }
311 
312   bool isRegOrImm() const {
313     return isReg() || isImm();
314   }
315 
316   bool isRegClass(unsigned RCID) const {
317     return isReg() && Reg.TRI->getRegClass(RCID).contains(getReg());
318   }
319 
320   bool isSCSrc32() const {
321     return isInlinableImm() || isRegClass(AMDGPU::SReg_32RegClassID);
322   }
323 
324   bool isSCSrc64() const {
325     return isInlinableImm() || isRegClass(AMDGPU::SReg_64RegClassID);
326   }
327 
328   bool isSSrc32() const {
329     return isImm() || isSCSrc32();
330   }
331 
332   bool isSSrc64() const {
333     // TODO: Find out how SALU supports extension of 32-bit literals to 64 bits.
334     // See isVSrc64().
335     return isImm() || isSCSrc64();
336   }
337 
338   bool isVCSrc32() const {
339     return isInlinableImm() || isRegClass(AMDGPU::VS_32RegClassID);
340   }
341 
342   bool isVCSrc64() const {
343     return isInlinableImm() || isRegClass(AMDGPU::VS_64RegClassID);
344   }
345 
346   bool isVSrc32() const {
347     return isImm() || isVCSrc32();
348   }
349 
350   bool isVSrc64() const {
351     // TODO: Check if the 64-bit value (coming from assembly source) can be
352     // narrowed to 32 bits (in the instruction stream). That require knowledge
353     // of instruction type (unsigned/signed, floating or "untyped"/B64),
354     // see [AMD GCN3 ISA 6.3.1].
355     // TODO: How 64-bit values are formed from 32-bit literals in _B64 insns?
356     return isImm() || isVCSrc64();
357   }
358 
359   bool isMem() const override {
360     return false;
361   }
362 
363   bool isExpr() const {
364     return Kind == Expression;
365   }
366 
367   bool isSoppBrTarget() const {
368     return isExpr() || isImm();
369   }
370 
371   SMLoc getStartLoc() const override {
372     return StartLoc;
373   }
374 
375   SMLoc getEndLoc() const override {
376     return EndLoc;
377   }
378 
379   void printImmTy(raw_ostream& OS, ImmTy Type) const {
380     switch (Type) {
381     case ImmTyNone: OS << "None"; break;
382     case ImmTyGDS: OS << "GDS"; break;
383     case ImmTyOffen: OS << "Offen"; break;
384     case ImmTyIdxen: OS << "Idxen"; break;
385     case ImmTyAddr64: OS << "Addr64"; break;
386     case ImmTyOffset: OS << "Offset"; break;
387     case ImmTyOffset0: OS << "Offset0"; break;
388     case ImmTyOffset1: OS << "Offset1"; break;
389     case ImmTyGLC: OS << "GLC"; break;
390     case ImmTySLC: OS << "SLC"; break;
391     case ImmTyTFE: OS << "TFE"; break;
392     case ImmTyClampSI: OS << "ClampSI"; break;
393     case ImmTyOModSI: OS << "OModSI"; break;
394     case ImmTyDppCtrl: OS << "DppCtrl"; break;
395     case ImmTyDppRowMask: OS << "DppRowMask"; break;
396     case ImmTyDppBankMask: OS << "DppBankMask"; break;
397     case ImmTyDppBoundCtrl: OS << "DppBoundCtrl"; break;
398     case ImmTySdwaDstSel: OS << "SdwaDstSel"; break;
399     case ImmTySdwaSrc0Sel: OS << "SdwaSrc0Sel"; break;
400     case ImmTySdwaSrc1Sel: OS << "SdwaSrc1Sel"; break;
401     case ImmTySdwaDstUnused: OS << "SdwaDstUnused"; break;
402     case ImmTyDMask: OS << "DMask"; break;
403     case ImmTyUNorm: OS << "UNorm"; break;
404     case ImmTyDA: OS << "DA"; break;
405     case ImmTyR128: OS << "R128"; break;
406     case ImmTyLWE: OS << "LWE"; break;
407     case ImmTyHwreg: OS << "Hwreg"; break;
408     case ImmTySendMsg: OS << "SendMsg"; break;
409     }
410   }
411 
412   void print(raw_ostream &OS) const override {
413     switch (Kind) {
414     case Register:
415       OS << "<register " << getReg() << " mods: " << Reg.Modifiers << '>';
416       break;
417     case Immediate:
418       OS << '<' << getImm();
419       if (getImmTy() != ImmTyNone) {
420         OS << " type: "; printImmTy(OS, getImmTy());
421       }
422       OS << " mods: " << Imm.Modifiers << '>';
423       break;
424     case Token:
425       OS << '\'' << getToken() << '\'';
426       break;
427     case Expression:
428       OS << "<expr " << *Expr << '>';
429       break;
430     }
431   }
432 
433   static AMDGPUOperand::Ptr CreateImm(int64_t Val, SMLoc Loc,
434                                       enum ImmTy Type = ImmTyNone,
435                                       bool IsFPImm = false) {
436     auto Op = llvm::make_unique<AMDGPUOperand>(Immediate);
437     Op->Imm.Val = Val;
438     Op->Imm.IsFPImm = IsFPImm;
439     Op->Imm.Type = Type;
440     Op->Imm.Modifiers = 0;
441     Op->StartLoc = Loc;
442     Op->EndLoc = Loc;
443     return Op;
444   }
445 
446   static AMDGPUOperand::Ptr CreateToken(StringRef Str, SMLoc Loc,
447                                         bool HasExplicitEncodingSize = true) {
448     auto Res = llvm::make_unique<AMDGPUOperand>(Token);
449     Res->Tok.Data = Str.data();
450     Res->Tok.Length = Str.size();
451     Res->StartLoc = Loc;
452     Res->EndLoc = Loc;
453     return Res;
454   }
455 
456   static AMDGPUOperand::Ptr CreateReg(unsigned RegNo, SMLoc S,
457                                       SMLoc E,
458                                       const MCRegisterInfo *TRI,
459                                       const MCSubtargetInfo *STI,
460                                       bool ForceVOP3) {
461     auto Op = llvm::make_unique<AMDGPUOperand>(Register);
462     Op->Reg.RegNo = RegNo;
463     Op->Reg.TRI = TRI;
464     Op->Reg.STI = STI;
465     Op->Reg.Modifiers = 0;
466     Op->Reg.IsForcedVOP3 = ForceVOP3;
467     Op->StartLoc = S;
468     Op->EndLoc = E;
469     return Op;
470   }
471 
472   static AMDGPUOperand::Ptr CreateExpr(const class MCExpr *Expr, SMLoc S) {
473     auto Op = llvm::make_unique<AMDGPUOperand>(Expression);
474     Op->Expr = Expr;
475     Op->StartLoc = S;
476     Op->EndLoc = S;
477     return Op;
478   }
479 
480   bool isSWaitCnt() const;
481   bool isHwreg() const;
482   bool isSendMsg() const;
483   bool isMubufOffset() const;
484   bool isSMRDOffset() const;
485   bool isSMRDLiteralOffset() const;
486   bool isDPPCtrl() const;
487 };
488 
489 class AMDGPUAsmParser : public MCTargetAsmParser {
490   const MCInstrInfo &MII;
491   MCAsmParser &Parser;
492 
493   unsigned ForcedEncodingSize;
494   bool ForcedDPP;
495   bool ForcedSDWA;
496 
497   bool isSI() const {
498     return AMDGPU::isSI(getSTI());
499   }
500 
501   bool isCI() const {
502     return AMDGPU::isCI(getSTI());
503   }
504 
505   bool isVI() const {
506     return AMDGPU::isVI(getSTI());
507   }
508 
509   bool hasSGPR102_SGPR103() const {
510     return !isVI();
511   }
512 
513   /// @name Auto-generated Match Functions
514   /// {
515 
516 #define GET_ASSEMBLER_HEADER
517 #include "AMDGPUGenAsmMatcher.inc"
518 
519   /// }
520 
521 private:
522   bool ParseDirectiveMajorMinor(uint32_t &Major, uint32_t &Minor);
523   bool ParseDirectiveHSACodeObjectVersion();
524   bool ParseDirectiveHSACodeObjectISA();
525   bool ParseAMDKernelCodeTValue(StringRef ID, amd_kernel_code_t &Header);
526   bool ParseDirectiveAMDKernelCodeT();
527   bool ParseSectionDirectiveHSAText();
528   bool subtargetHasRegister(const MCRegisterInfo &MRI, unsigned RegNo) const;
529   bool ParseDirectiveAMDGPUHsaKernel();
530   bool ParseDirectiveAMDGPUHsaModuleGlobal();
531   bool ParseDirectiveAMDGPUHsaProgramGlobal();
532   bool ParseSectionDirectiveHSADataGlobalAgent();
533   bool ParseSectionDirectiveHSADataGlobalProgram();
534   bool ParseSectionDirectiveHSARodataReadonlyAgent();
535   bool AddNextRegisterToList(unsigned& Reg, unsigned& RegWidth, RegisterKind RegKind, unsigned Reg1, unsigned RegNum);
536   bool ParseAMDGPURegister(RegisterKind& RegKind, unsigned& Reg, unsigned& RegNum, unsigned& RegWidth);
537   void cvtMubufImpl(MCInst &Inst, const OperandVector &Operands, bool IsAtomic, bool IsAtomicReturn);
538 
539 public:
540   enum AMDGPUMatchResultTy {
541     Match_PreferE32 = FIRST_TARGET_MATCH_RESULT_TY
542   };
543 
544   AMDGPUAsmParser(const MCSubtargetInfo &STI, MCAsmParser &_Parser,
545                const MCInstrInfo &MII,
546                const MCTargetOptions &Options)
547       : MCTargetAsmParser(Options, STI), MII(MII), Parser(_Parser),
548         ForcedEncodingSize(0),
549         ForcedDPP(false),
550         ForcedSDWA(false) {
551     MCAsmParserExtension::Initialize(Parser);
552 
553     if (getSTI().getFeatureBits().none()) {
554       // Set default features.
555       copySTI().ToggleFeature("SOUTHERN_ISLANDS");
556     }
557 
558     setAvailableFeatures(ComputeAvailableFeatures(getSTI().getFeatureBits()));
559   }
560 
561   AMDGPUTargetStreamer &getTargetStreamer() {
562     MCTargetStreamer &TS = *getParser().getStreamer().getTargetStreamer();
563     return static_cast<AMDGPUTargetStreamer &>(TS);
564   }
565 
566   void setForcedEncodingSize(unsigned Size) { ForcedEncodingSize = Size; }
567   void setForcedDPP(bool ForceDPP_) { ForcedDPP = ForceDPP_; }
568   void setForcedSDWA(bool ForceSDWA_) { ForcedSDWA = ForceSDWA_; }
569 
570   unsigned getForcedEncodingSize() const { return ForcedEncodingSize; }
571   bool isForcedVOP3() const { return ForcedEncodingSize == 64; }
572   bool isForcedDPP() const { return ForcedDPP; }
573   bool isForcedSDWA() const { return ForcedSDWA; }
574 
575   std::unique_ptr<AMDGPUOperand> parseRegister();
576   bool ParseRegister(unsigned &RegNo, SMLoc &StartLoc, SMLoc &EndLoc) override;
577   unsigned checkTargetMatchPredicate(MCInst &Inst) override;
578   unsigned validateTargetOperandClass(MCParsedAsmOperand &Op,
579                                       unsigned Kind) override;
580   bool MatchAndEmitInstruction(SMLoc IDLoc, unsigned &Opcode,
581                                OperandVector &Operands, MCStreamer &Out,
582                                uint64_t &ErrorInfo,
583                                bool MatchingInlineAsm) override;
584   bool ParseDirective(AsmToken DirectiveID) override;
585   OperandMatchResultTy parseOperand(OperandVector &Operands, StringRef Mnemonic);
586   StringRef parseMnemonicSuffix(StringRef Name);
587   bool ParseInstruction(ParseInstructionInfo &Info, StringRef Name,
588                         SMLoc NameLoc, OperandVector &Operands) override;
589 
590   OperandMatchResultTy parseIntWithPrefix(const char *Prefix, int64_t &Int);
591   OperandMatchResultTy parseIntWithPrefix(const char *Prefix,
592                                           OperandVector &Operands,
593                                           enum AMDGPUOperand::ImmTy ImmTy = AMDGPUOperand::ImmTyNone,
594                                           bool (*ConvertResult)(int64_t&) = 0);
595   OperandMatchResultTy parseNamedBit(const char *Name, OperandVector &Operands,
596                                      enum AMDGPUOperand::ImmTy ImmTy = AMDGPUOperand::ImmTyNone);
597   OperandMatchResultTy parseStringWithPrefix(StringRef Prefix, StringRef &Value);
598 
599   OperandMatchResultTy parseImm(OperandVector &Operands);
600   OperandMatchResultTy parseRegOrImm(OperandVector &Operands);
601   OperandMatchResultTy parseRegOrImmWithInputMods(OperandVector &Operands);
602 
603   void cvtDSOffset01(MCInst &Inst, const OperandVector &Operands);
604   void cvtDS(MCInst &Inst, const OperandVector &Operands);
605 
606   bool parseCnt(int64_t &IntVal);
607   OperandMatchResultTy parseSWaitCntOps(OperandVector &Operands);
608   OperandMatchResultTy parseHwreg(OperandVector &Operands);
609 
610 private:
611   struct OperandInfoTy {
612     int64_t Id;
613     bool IsSymbolic;
614     OperandInfoTy(int64_t Id_) : Id(Id_), IsSymbolic(false) { }
615   };
616 
617   bool parseSendMsgConstruct(OperandInfoTy &Msg, OperandInfoTy &Operation, int64_t &StreamId);
618   bool parseHwregConstruct(OperandInfoTy &HwReg, int64_t &Offset, int64_t &Width);
619 public:
620   OperandMatchResultTy parseOptionalOperand(OperandVector &Operands);
621 
622   OperandMatchResultTy parseSendMsgOp(OperandVector &Operands);
623   OperandMatchResultTy parseSOppBrTarget(OperandVector &Operands);
624   AMDGPUOperand::Ptr defaultHwreg() const;
625 
626   void cvtMubuf(MCInst &Inst, const OperandVector &Operands) { cvtMubufImpl(Inst, Operands, false, false); }
627   void cvtMubufAtomic(MCInst &Inst, const OperandVector &Operands) { cvtMubufImpl(Inst, Operands, true, false); }
628   void cvtMubufAtomicReturn(MCInst &Inst, const OperandVector &Operands) { cvtMubufImpl(Inst, Operands, true, true); }
629   AMDGPUOperand::Ptr defaultMubufOffset() const;
630   AMDGPUOperand::Ptr defaultGLC() const;
631   AMDGPUOperand::Ptr defaultSLC() const;
632   AMDGPUOperand::Ptr defaultTFE() const;
633 
634   AMDGPUOperand::Ptr defaultDMask() const;
635   AMDGPUOperand::Ptr defaultUNorm() const;
636   AMDGPUOperand::Ptr defaultDA() const;
637   AMDGPUOperand::Ptr defaultR128() const;
638   AMDGPUOperand::Ptr defaultLWE() const;
639   AMDGPUOperand::Ptr defaultSMRDOffset() const;
640   AMDGPUOperand::Ptr defaultSMRDLiteralOffset() const;
641 
642   AMDGPUOperand::Ptr defaultClampSI() const;
643   AMDGPUOperand::Ptr defaultOModSI() const;
644 
645   OperandMatchResultTy parseOModOperand(OperandVector &Operands);
646 
647   void cvtId(MCInst &Inst, const OperandVector &Operands);
648   void cvtVOP3_2_mod(MCInst &Inst, const OperandVector &Operands);
649   void cvtVOP3(MCInst &Inst, const OperandVector &Operands);
650 
651   void cvtMIMG(MCInst &Inst, const OperandVector &Operands);
652   void cvtMIMGAtomic(MCInst &Inst, const OperandVector &Operands);
653 
654   OperandMatchResultTy parseDPPCtrl(OperandVector &Operands);
655   AMDGPUOperand::Ptr defaultRowMask() const;
656   AMDGPUOperand::Ptr defaultBankMask() const;
657   AMDGPUOperand::Ptr defaultBoundCtrl() const;
658   void cvtDPP(MCInst &Inst, const OperandVector &Operands);
659 
660   OperandMatchResultTy parseSDWASel(OperandVector &Operands, StringRef Prefix,
661                                     AMDGPUOperand::ImmTy Type);
662   OperandMatchResultTy parseSDWADstUnused(OperandVector &Operands);
663   AMDGPUOperand::Ptr defaultSDWASel(AMDGPUOperand::ImmTy Type) const;
664   AMDGPUOperand::Ptr defaultSDWADstSel() const;
665   AMDGPUOperand::Ptr defaultSDWASrc0Sel() const;
666   AMDGPUOperand::Ptr defaultSDWASrc1Sel() const;
667   AMDGPUOperand::Ptr defaultSDWADstUnused() const;
668   void cvtSdwaVop1_mod(MCInst &Inst, const OperandVector &Operands);
669   void cvtSdwaVop1_nomod(MCInst &Inst, const OperandVector &Operands);
670   void cvtSdwaVop2_mod(MCInst &Inst, const OperandVector &Operands);
671   void cvtSdwaVop2_nomod(MCInst &Inst, const OperandVector &Operands);
672   void cvtSDWA(MCInst &Inst, const OperandVector &Operands, bool HasMods,
673                bool IsVOP1);
674 };
675 
676 struct OptionalOperand {
677   const char *Name;
678   AMDGPUOperand::ImmTy Type;
679   bool IsBit;
680   bool (*ConvertResult)(int64_t&);
681 };
682 
683 }
684 
685 static int getRegClass(RegisterKind Is, unsigned RegWidth) {
686   if (Is == IS_VGPR) {
687     switch (RegWidth) {
688       default: return -1;
689       case 1: return AMDGPU::VGPR_32RegClassID;
690       case 2: return AMDGPU::VReg_64RegClassID;
691       case 3: return AMDGPU::VReg_96RegClassID;
692       case 4: return AMDGPU::VReg_128RegClassID;
693       case 8: return AMDGPU::VReg_256RegClassID;
694       case 16: return AMDGPU::VReg_512RegClassID;
695     }
696   } else if (Is == IS_TTMP) {
697     switch (RegWidth) {
698       default: return -1;
699       case 1: return AMDGPU::TTMP_32RegClassID;
700       case 2: return AMDGPU::TTMP_64RegClassID;
701       case 4: return AMDGPU::TTMP_128RegClassID;
702     }
703   } else if (Is == IS_SGPR) {
704     switch (RegWidth) {
705       default: return -1;
706       case 1: return AMDGPU::SGPR_32RegClassID;
707       case 2: return AMDGPU::SGPR_64RegClassID;
708       case 4: return AMDGPU::SGPR_128RegClassID;
709       case 8: return AMDGPU::SReg_256RegClassID;
710       case 16: return AMDGPU::SReg_512RegClassID;
711     }
712   }
713   return -1;
714 }
715 
716 static unsigned getSpecialRegForName(StringRef RegName) {
717   return StringSwitch<unsigned>(RegName)
718     .Case("exec", AMDGPU::EXEC)
719     .Case("vcc", AMDGPU::VCC)
720     .Case("flat_scratch", AMDGPU::FLAT_SCR)
721     .Case("m0", AMDGPU::M0)
722     .Case("scc", AMDGPU::SCC)
723     .Case("tba", AMDGPU::TBA)
724     .Case("tma", AMDGPU::TMA)
725     .Case("flat_scratch_lo", AMDGPU::FLAT_SCR_LO)
726     .Case("flat_scratch_hi", AMDGPU::FLAT_SCR_HI)
727     .Case("vcc_lo", AMDGPU::VCC_LO)
728     .Case("vcc_hi", AMDGPU::VCC_HI)
729     .Case("exec_lo", AMDGPU::EXEC_LO)
730     .Case("exec_hi", AMDGPU::EXEC_HI)
731     .Case("tma_lo", AMDGPU::TMA_LO)
732     .Case("tma_hi", AMDGPU::TMA_HI)
733     .Case("tba_lo", AMDGPU::TBA_LO)
734     .Case("tba_hi", AMDGPU::TBA_HI)
735     .Default(0);
736 }
737 
738 bool AMDGPUAsmParser::ParseRegister(unsigned &RegNo, SMLoc &StartLoc, SMLoc &EndLoc) {
739   auto R = parseRegister();
740   if (!R) return true;
741   assert(R->isReg());
742   RegNo = R->getReg();
743   StartLoc = R->getStartLoc();
744   EndLoc = R->getEndLoc();
745   return false;
746 }
747 
748 bool AMDGPUAsmParser::AddNextRegisterToList(unsigned& Reg, unsigned& RegWidth, RegisterKind RegKind, unsigned Reg1, unsigned RegNum)
749 {
750   switch (RegKind) {
751   case IS_SPECIAL:
752     if (Reg == AMDGPU::EXEC_LO && Reg1 == AMDGPU::EXEC_HI) { Reg = AMDGPU::EXEC; RegWidth = 2; return true; }
753     if (Reg == AMDGPU::FLAT_SCR_LO && Reg1 == AMDGPU::FLAT_SCR_HI) { Reg = AMDGPU::FLAT_SCR; RegWidth = 2; return true; }
754     if (Reg == AMDGPU::VCC_LO && Reg1 == AMDGPU::VCC_HI) { Reg = AMDGPU::VCC; RegWidth = 2; return true; }
755     if (Reg == AMDGPU::TBA_LO && Reg1 == AMDGPU::TBA_HI) { Reg = AMDGPU::TBA; RegWidth = 2; return true; }
756     if (Reg == AMDGPU::TMA_LO && Reg1 == AMDGPU::TMA_HI) { Reg = AMDGPU::TMA; RegWidth = 2; return true; }
757     return false;
758   case IS_VGPR:
759   case IS_SGPR:
760   case IS_TTMP:
761     if (Reg1 != Reg + RegWidth) { return false; }
762     RegWidth++;
763     return true;
764   default:
765     assert(false); return false;
766   }
767 }
768 
769 bool AMDGPUAsmParser::ParseAMDGPURegister(RegisterKind& RegKind, unsigned& Reg, unsigned& RegNum, unsigned& RegWidth)
770 {
771   const MCRegisterInfo *TRI = getContext().getRegisterInfo();
772   if (getLexer().is(AsmToken::Identifier)) {
773     StringRef RegName = Parser.getTok().getString();
774     if ((Reg = getSpecialRegForName(RegName))) {
775       Parser.Lex();
776       RegKind = IS_SPECIAL;
777     } else {
778       unsigned RegNumIndex = 0;
779       if (RegName[0] == 'v') {
780         RegNumIndex = 1;
781         RegKind = IS_VGPR;
782       } else if (RegName[0] == 's') {
783         RegNumIndex = 1;
784         RegKind = IS_SGPR;
785       } else if (RegName.startswith("ttmp")) {
786         RegNumIndex = strlen("ttmp");
787         RegKind = IS_TTMP;
788       } else {
789         return false;
790       }
791       if (RegName.size() > RegNumIndex) {
792         // Single 32-bit register: vXX.
793         if (RegName.substr(RegNumIndex).getAsInteger(10, RegNum))
794           return false;
795         Parser.Lex();
796         RegWidth = 1;
797       } else {
798         // Range of registers: v[XX:YY]. ":YY" is optional.
799         Parser.Lex();
800         int64_t RegLo, RegHi;
801         if (getLexer().isNot(AsmToken::LBrac))
802           return false;
803         Parser.Lex();
804 
805         if (getParser().parseAbsoluteExpression(RegLo))
806           return false;
807 
808         const bool isRBrace = getLexer().is(AsmToken::RBrac);
809         if (!isRBrace && getLexer().isNot(AsmToken::Colon))
810           return false;
811         Parser.Lex();
812 
813         if (isRBrace) {
814           RegHi = RegLo;
815         } else {
816           if (getParser().parseAbsoluteExpression(RegHi))
817             return false;
818 
819           if (getLexer().isNot(AsmToken::RBrac))
820             return false;
821           Parser.Lex();
822         }
823         RegNum = (unsigned) RegLo;
824         RegWidth = (RegHi - RegLo) + 1;
825       }
826     }
827   } else if (getLexer().is(AsmToken::LBrac)) {
828     // List of consecutive registers: [s0,s1,s2,s3]
829     Parser.Lex();
830     if (!ParseAMDGPURegister(RegKind, Reg, RegNum, RegWidth))
831       return false;
832     if (RegWidth != 1)
833       return false;
834     RegisterKind RegKind1;
835     unsigned Reg1, RegNum1, RegWidth1;
836     do {
837       if (getLexer().is(AsmToken::Comma)) {
838         Parser.Lex();
839       } else if (getLexer().is(AsmToken::RBrac)) {
840         Parser.Lex();
841         break;
842       } else if (ParseAMDGPURegister(RegKind1, Reg1, RegNum1, RegWidth1)) {
843         if (RegWidth1 != 1) {
844           return false;
845         }
846         if (RegKind1 != RegKind) {
847           return false;
848         }
849         if (!AddNextRegisterToList(Reg, RegWidth, RegKind1, Reg1, RegNum1)) {
850           return false;
851         }
852       } else {
853         return false;
854       }
855     } while (true);
856   } else {
857     return false;
858   }
859   switch (RegKind) {
860   case IS_SPECIAL:
861     RegNum = 0;
862     RegWidth = 1;
863     break;
864   case IS_VGPR:
865   case IS_SGPR:
866   case IS_TTMP:
867   {
868     unsigned Size = 1;
869     if (RegKind == IS_SGPR || RegKind == IS_TTMP) {
870       // SGPR and TTMP registers must be are aligned. Max required alignment is 4 dwords.
871       Size = std::min(RegWidth, 4u);
872     }
873     if (RegNum % Size != 0)
874       return false;
875     RegNum = RegNum / Size;
876     int RCID = getRegClass(RegKind, RegWidth);
877     if (RCID == -1)
878       return false;
879     const MCRegisterClass RC = TRI->getRegClass(RCID);
880     if (RegNum >= RC.getNumRegs())
881       return false;
882     Reg = RC.getRegister(RegNum);
883     break;
884   }
885 
886   default:
887     assert(false); return false;
888   }
889 
890   if (!subtargetHasRegister(*TRI, Reg))
891     return false;
892   return true;
893 }
894 
895 std::unique_ptr<AMDGPUOperand> AMDGPUAsmParser::parseRegister() {
896   const auto &Tok = Parser.getTok();
897   SMLoc StartLoc = Tok.getLoc();
898   SMLoc EndLoc = Tok.getEndLoc();
899   const MCRegisterInfo *TRI = getContext().getRegisterInfo();
900 
901   RegisterKind RegKind;
902   unsigned Reg, RegNum, RegWidth;
903 
904   if (!ParseAMDGPURegister(RegKind, Reg, RegNum, RegWidth)) {
905     return nullptr;
906   }
907   return AMDGPUOperand::CreateReg(Reg, StartLoc, EndLoc,
908                                   TRI, &getSTI(), false);
909 }
910 
911 AMDGPUAsmParser::OperandMatchResultTy
912 AMDGPUAsmParser::parseImm(OperandVector &Operands) {
913   bool Minus = false;
914   if (getLexer().getKind() == AsmToken::Minus) {
915     Minus = true;
916     Parser.Lex();
917   }
918 
919   SMLoc S = Parser.getTok().getLoc();
920   switch(getLexer().getKind()) {
921   case AsmToken::Integer: {
922     int64_t IntVal;
923     if (getParser().parseAbsoluteExpression(IntVal))
924       return MatchOperand_ParseFail;
925     if (!isInt<32>(IntVal) && !isUInt<32>(IntVal)) {
926       Error(S, "invalid immediate: only 32-bit values are legal");
927       return MatchOperand_ParseFail;
928     }
929 
930     if (Minus)
931       IntVal *= -1;
932     Operands.push_back(AMDGPUOperand::CreateImm(IntVal, S));
933     return MatchOperand_Success;
934   }
935   case AsmToken::Real: {
936     // FIXME: We should emit an error if a double precisions floating-point
937     // value is used.  I'm not sure the best way to detect this.
938     int64_t IntVal;
939     if (getParser().parseAbsoluteExpression(IntVal))
940       return MatchOperand_ParseFail;
941 
942     APFloat F((float)BitsToDouble(IntVal));
943     if (Minus)
944       F.changeSign();
945     Operands.push_back(
946         AMDGPUOperand::CreateImm(F.bitcastToAPInt().getZExtValue(), S,
947                                  AMDGPUOperand::ImmTyNone, true));
948     return MatchOperand_Success;
949   }
950   default:
951     return Minus ? MatchOperand_ParseFail : MatchOperand_NoMatch;
952   }
953 }
954 
955 AMDGPUAsmParser::OperandMatchResultTy
956 AMDGPUAsmParser::parseRegOrImm(OperandVector &Operands) {
957   auto res = parseImm(Operands);
958   if (res != MatchOperand_NoMatch) {
959     return res;
960   }
961 
962   if (auto R = parseRegister()) {
963     assert(R->isReg());
964     R->Reg.IsForcedVOP3 = isForcedVOP3();
965     Operands.push_back(std::move(R));
966     return MatchOperand_Success;
967   }
968   return MatchOperand_ParseFail;
969 }
970 
971 AMDGPUAsmParser::OperandMatchResultTy
972 AMDGPUAsmParser::parseRegOrImmWithInputMods(OperandVector &Operands) {
973   // XXX: During parsing we can't determine if minus sign means
974   // negate-modifier or negative immediate value.
975   // By default we suppose it is modifier.
976   bool Negate = false, Abs = false, Abs2 = false;
977 
978   if (getLexer().getKind()== AsmToken::Minus) {
979     Parser.Lex();
980     Negate = true;
981   }
982 
983   if (getLexer().getKind() == AsmToken::Identifier && Parser.getTok().getString() == "abs") {
984     Parser.Lex();
985     Abs2 = true;
986     if (getLexer().isNot(AsmToken::LParen)) {
987       Error(Parser.getTok().getLoc(), "expected left paren after abs");
988       return MatchOperand_ParseFail;
989     }
990     Parser.Lex();
991   }
992 
993   if (getLexer().getKind() == AsmToken::Pipe) {
994     if (Abs2) {
995       Error(Parser.getTok().getLoc(), "expected register or immediate");
996       return MatchOperand_ParseFail;
997     }
998     Parser.Lex();
999     Abs = true;
1000   }
1001 
1002   auto Res = parseRegOrImm(Operands);
1003   if (Res != MatchOperand_Success) {
1004     return Res;
1005   }
1006 
1007   unsigned Modifiers = 0;
1008   if (Negate) {
1009     Modifiers |= 0x1;
1010   }
1011   if (Abs) {
1012     if (getLexer().getKind() != AsmToken::Pipe) {
1013       Error(Parser.getTok().getLoc(), "expected vertical bar");
1014       return MatchOperand_ParseFail;
1015     }
1016     Parser.Lex();
1017     Modifiers |= 0x2;
1018   }
1019   if (Abs2) {
1020     if (getLexer().isNot(AsmToken::RParen)) {
1021       Error(Parser.getTok().getLoc(), "expected closing parentheses");
1022       return MatchOperand_ParseFail;
1023     }
1024     Parser.Lex();
1025     Modifiers |= 0x2;
1026   }
1027 
1028   if (Modifiers) {
1029     AMDGPUOperand &Op = static_cast<AMDGPUOperand &>(*Operands.back());
1030     Op.setModifiers(Modifiers);
1031   }
1032   return MatchOperand_Success;
1033 }
1034 
1035 
1036 unsigned AMDGPUAsmParser::checkTargetMatchPredicate(MCInst &Inst) {
1037 
1038   uint64_t TSFlags = MII.get(Inst.getOpcode()).TSFlags;
1039 
1040   if ((getForcedEncodingSize() == 32 && (TSFlags & SIInstrFlags::VOP3)) ||
1041       (getForcedEncodingSize() == 64 && !(TSFlags & SIInstrFlags::VOP3)) ||
1042       (isForcedDPP() && !(TSFlags & SIInstrFlags::DPP)) ||
1043       (isForcedSDWA() && !(TSFlags & SIInstrFlags::SDWA)) )
1044     return Match_InvalidOperand;
1045 
1046   if ((TSFlags & SIInstrFlags::VOP3) &&
1047       (TSFlags & SIInstrFlags::VOPAsmPrefer32Bit) &&
1048       getForcedEncodingSize() != 64)
1049     return Match_PreferE32;
1050 
1051   return Match_Success;
1052 }
1053 
1054 bool AMDGPUAsmParser::MatchAndEmitInstruction(SMLoc IDLoc, unsigned &Opcode,
1055                                               OperandVector &Operands,
1056                                               MCStreamer &Out,
1057                                               uint64_t &ErrorInfo,
1058                                               bool MatchingInlineAsm) {
1059   MCInst Inst;
1060 
1061   switch (MatchInstructionImpl(Operands, Inst, ErrorInfo, MatchingInlineAsm)) {
1062     default: break;
1063     case Match_Success:
1064       Inst.setLoc(IDLoc);
1065       Out.EmitInstruction(Inst, getSTI());
1066       return false;
1067     case Match_MissingFeature:
1068       return Error(IDLoc, "instruction not supported on this GPU");
1069 
1070     case Match_MnemonicFail:
1071       return Error(IDLoc, "unrecognized instruction mnemonic");
1072 
1073     case Match_InvalidOperand: {
1074       SMLoc ErrorLoc = IDLoc;
1075       if (ErrorInfo != ~0ULL) {
1076         if (ErrorInfo >= Operands.size()) {
1077           return Error(IDLoc, "too few operands for instruction");
1078         }
1079         ErrorLoc = ((AMDGPUOperand &)*Operands[ErrorInfo]).getStartLoc();
1080         if (ErrorLoc == SMLoc())
1081           ErrorLoc = IDLoc;
1082       }
1083       return Error(ErrorLoc, "invalid operand for instruction");
1084     }
1085     case Match_PreferE32:
1086       return Error(IDLoc, "internal error: instruction without _e64 suffix "
1087                           "should be encoded as e32");
1088   }
1089   llvm_unreachable("Implement any new match types added!");
1090 }
1091 
1092 bool AMDGPUAsmParser::ParseDirectiveMajorMinor(uint32_t &Major,
1093                                                uint32_t &Minor) {
1094   if (getLexer().isNot(AsmToken::Integer))
1095     return TokError("invalid major version");
1096 
1097   Major = getLexer().getTok().getIntVal();
1098   Lex();
1099 
1100   if (getLexer().isNot(AsmToken::Comma))
1101     return TokError("minor version number required, comma expected");
1102   Lex();
1103 
1104   if (getLexer().isNot(AsmToken::Integer))
1105     return TokError("invalid minor version");
1106 
1107   Minor = getLexer().getTok().getIntVal();
1108   Lex();
1109 
1110   return false;
1111 }
1112 
1113 bool AMDGPUAsmParser::ParseDirectiveHSACodeObjectVersion() {
1114 
1115   uint32_t Major;
1116   uint32_t Minor;
1117 
1118   if (ParseDirectiveMajorMinor(Major, Minor))
1119     return true;
1120 
1121   getTargetStreamer().EmitDirectiveHSACodeObjectVersion(Major, Minor);
1122   return false;
1123 }
1124 
1125 bool AMDGPUAsmParser::ParseDirectiveHSACodeObjectISA() {
1126 
1127   uint32_t Major;
1128   uint32_t Minor;
1129   uint32_t Stepping;
1130   StringRef VendorName;
1131   StringRef ArchName;
1132 
1133   // If this directive has no arguments, then use the ISA version for the
1134   // targeted GPU.
1135   if (getLexer().is(AsmToken::EndOfStatement)) {
1136     AMDGPU::IsaVersion Isa = AMDGPU::getIsaVersion(getSTI().getFeatureBits());
1137     getTargetStreamer().EmitDirectiveHSACodeObjectISA(Isa.Major, Isa.Minor,
1138                                                       Isa.Stepping,
1139                                                       "AMD", "AMDGPU");
1140     return false;
1141   }
1142 
1143 
1144   if (ParseDirectiveMajorMinor(Major, Minor))
1145     return true;
1146 
1147   if (getLexer().isNot(AsmToken::Comma))
1148     return TokError("stepping version number required, comma expected");
1149   Lex();
1150 
1151   if (getLexer().isNot(AsmToken::Integer))
1152     return TokError("invalid stepping version");
1153 
1154   Stepping = getLexer().getTok().getIntVal();
1155   Lex();
1156 
1157   if (getLexer().isNot(AsmToken::Comma))
1158     return TokError("vendor name required, comma expected");
1159   Lex();
1160 
1161   if (getLexer().isNot(AsmToken::String))
1162     return TokError("invalid vendor name");
1163 
1164   VendorName = getLexer().getTok().getStringContents();
1165   Lex();
1166 
1167   if (getLexer().isNot(AsmToken::Comma))
1168     return TokError("arch name required, comma expected");
1169   Lex();
1170 
1171   if (getLexer().isNot(AsmToken::String))
1172     return TokError("invalid arch name");
1173 
1174   ArchName = getLexer().getTok().getStringContents();
1175   Lex();
1176 
1177   getTargetStreamer().EmitDirectiveHSACodeObjectISA(Major, Minor, Stepping,
1178                                                     VendorName, ArchName);
1179   return false;
1180 }
1181 
1182 bool AMDGPUAsmParser::ParseAMDKernelCodeTValue(StringRef ID,
1183                                                amd_kernel_code_t &Header) {
1184   SmallString<40> ErrStr;
1185   raw_svector_ostream Err(ErrStr);
1186   if (!parseAmdKernelCodeField(ID, getLexer(), Header, Err)) {
1187     return TokError(Err.str());
1188   }
1189   Lex();
1190   return false;
1191 }
1192 
1193 bool AMDGPUAsmParser::ParseDirectiveAMDKernelCodeT() {
1194 
1195   amd_kernel_code_t Header;
1196   AMDGPU::initDefaultAMDKernelCodeT(Header, getSTI().getFeatureBits());
1197 
1198   while (true) {
1199 
1200     if (getLexer().isNot(AsmToken::EndOfStatement))
1201       return TokError("amd_kernel_code_t values must begin on a new line");
1202 
1203     // Lex EndOfStatement.  This is in a while loop, because lexing a comment
1204     // will set the current token to EndOfStatement.
1205     while(getLexer().is(AsmToken::EndOfStatement))
1206       Lex();
1207 
1208     if (getLexer().isNot(AsmToken::Identifier))
1209       return TokError("expected value identifier or .end_amd_kernel_code_t");
1210 
1211     StringRef ID = getLexer().getTok().getIdentifier();
1212     Lex();
1213 
1214     if (ID == ".end_amd_kernel_code_t")
1215       break;
1216 
1217     if (ParseAMDKernelCodeTValue(ID, Header))
1218       return true;
1219   }
1220 
1221   getTargetStreamer().EmitAMDKernelCodeT(Header);
1222 
1223   return false;
1224 }
1225 
1226 bool AMDGPUAsmParser::ParseSectionDirectiveHSAText() {
1227   getParser().getStreamer().SwitchSection(
1228       AMDGPU::getHSATextSection(getContext()));
1229   return false;
1230 }
1231 
1232 bool AMDGPUAsmParser::ParseDirectiveAMDGPUHsaKernel() {
1233   if (getLexer().isNot(AsmToken::Identifier))
1234     return TokError("expected symbol name");
1235 
1236   StringRef KernelName = Parser.getTok().getString();
1237 
1238   getTargetStreamer().EmitAMDGPUSymbolType(KernelName,
1239                                            ELF::STT_AMDGPU_HSA_KERNEL);
1240   Lex();
1241   return false;
1242 }
1243 
1244 bool AMDGPUAsmParser::ParseDirectiveAMDGPUHsaModuleGlobal() {
1245   if (getLexer().isNot(AsmToken::Identifier))
1246     return TokError("expected symbol name");
1247 
1248   StringRef GlobalName = Parser.getTok().getIdentifier();
1249 
1250   getTargetStreamer().EmitAMDGPUHsaModuleScopeGlobal(GlobalName);
1251   Lex();
1252   return false;
1253 }
1254 
1255 bool AMDGPUAsmParser::ParseDirectiveAMDGPUHsaProgramGlobal() {
1256   if (getLexer().isNot(AsmToken::Identifier))
1257     return TokError("expected symbol name");
1258 
1259   StringRef GlobalName = Parser.getTok().getIdentifier();
1260 
1261   getTargetStreamer().EmitAMDGPUHsaProgramScopeGlobal(GlobalName);
1262   Lex();
1263   return false;
1264 }
1265 
1266 bool AMDGPUAsmParser::ParseSectionDirectiveHSADataGlobalAgent() {
1267   getParser().getStreamer().SwitchSection(
1268       AMDGPU::getHSADataGlobalAgentSection(getContext()));
1269   return false;
1270 }
1271 
1272 bool AMDGPUAsmParser::ParseSectionDirectiveHSADataGlobalProgram() {
1273   getParser().getStreamer().SwitchSection(
1274       AMDGPU::getHSADataGlobalProgramSection(getContext()));
1275   return false;
1276 }
1277 
1278 bool AMDGPUAsmParser::ParseSectionDirectiveHSARodataReadonlyAgent() {
1279   getParser().getStreamer().SwitchSection(
1280       AMDGPU::getHSARodataReadonlyAgentSection(getContext()));
1281   return false;
1282 }
1283 
1284 bool AMDGPUAsmParser::ParseDirective(AsmToken DirectiveID) {
1285   StringRef IDVal = DirectiveID.getString();
1286 
1287   if (IDVal == ".hsa_code_object_version")
1288     return ParseDirectiveHSACodeObjectVersion();
1289 
1290   if (IDVal == ".hsa_code_object_isa")
1291     return ParseDirectiveHSACodeObjectISA();
1292 
1293   if (IDVal == ".amd_kernel_code_t")
1294     return ParseDirectiveAMDKernelCodeT();
1295 
1296   if (IDVal == ".hsatext")
1297     return ParseSectionDirectiveHSAText();
1298 
1299   if (IDVal == ".amdgpu_hsa_kernel")
1300     return ParseDirectiveAMDGPUHsaKernel();
1301 
1302   if (IDVal == ".amdgpu_hsa_module_global")
1303     return ParseDirectiveAMDGPUHsaModuleGlobal();
1304 
1305   if (IDVal == ".amdgpu_hsa_program_global")
1306     return ParseDirectiveAMDGPUHsaProgramGlobal();
1307 
1308   if (IDVal == ".hsadata_global_agent")
1309     return ParseSectionDirectiveHSADataGlobalAgent();
1310 
1311   if (IDVal == ".hsadata_global_program")
1312     return ParseSectionDirectiveHSADataGlobalProgram();
1313 
1314   if (IDVal == ".hsarodata_readonly_agent")
1315     return ParseSectionDirectiveHSARodataReadonlyAgent();
1316 
1317   return true;
1318 }
1319 
1320 bool AMDGPUAsmParser::subtargetHasRegister(const MCRegisterInfo &MRI,
1321                                            unsigned RegNo) const {
1322   if (isCI())
1323     return true;
1324 
1325   if (isSI()) {
1326     // No flat_scr
1327     switch (RegNo) {
1328     case AMDGPU::FLAT_SCR:
1329     case AMDGPU::FLAT_SCR_LO:
1330     case AMDGPU::FLAT_SCR_HI:
1331       return false;
1332     default:
1333       return true;
1334     }
1335   }
1336 
1337   // VI only has 102 SGPRs, so make sure we aren't trying to use the 2 more that
1338   // SI/CI have.
1339   for (MCRegAliasIterator R(AMDGPU::SGPR102_SGPR103, &MRI, true);
1340        R.isValid(); ++R) {
1341     if (*R == RegNo)
1342       return false;
1343   }
1344 
1345   return true;
1346 }
1347 
1348 AMDGPUAsmParser::OperandMatchResultTy
1349 AMDGPUAsmParser::parseOperand(OperandVector &Operands, StringRef Mnemonic) {
1350 
1351   // Try to parse with a custom parser
1352   OperandMatchResultTy ResTy = MatchOperandParserImpl(Operands, Mnemonic);
1353 
1354   // If we successfully parsed the operand or if there as an error parsing,
1355   // we are done.
1356   //
1357   // If we are parsing after we reach EndOfStatement then this means we
1358   // are appending default values to the Operands list.  This is only done
1359   // by custom parser, so we shouldn't continue on to the generic parsing.
1360   if (ResTy == MatchOperand_Success || ResTy == MatchOperand_ParseFail ||
1361       getLexer().is(AsmToken::EndOfStatement))
1362     return ResTy;
1363 
1364   ResTy = parseRegOrImm(Operands);
1365 
1366   if (ResTy == MatchOperand_Success)
1367     return ResTy;
1368 
1369   if (getLexer().getKind() == AsmToken::Identifier) {
1370     const auto &Tok = Parser.getTok();
1371     Operands.push_back(AMDGPUOperand::CreateToken(Tok.getString(), Tok.getLoc()));
1372     Parser.Lex();
1373     return MatchOperand_Success;
1374   }
1375   return MatchOperand_NoMatch;
1376 }
1377 
1378 StringRef AMDGPUAsmParser::parseMnemonicSuffix(StringRef Name) {
1379   // Clear any forced encodings from the previous instruction.
1380   setForcedEncodingSize(0);
1381   setForcedDPP(false);
1382   setForcedSDWA(false);
1383 
1384   if (Name.endswith("_e64")) {
1385     setForcedEncodingSize(64);
1386     return Name.substr(0, Name.size() - 4);
1387   } else if (Name.endswith("_e32")) {
1388     setForcedEncodingSize(32);
1389     return Name.substr(0, Name.size() - 4);
1390   } else if (Name.endswith("_dpp")) {
1391     setForcedDPP(true);
1392     return Name.substr(0, Name.size() - 4);
1393   } else if (Name.endswith("_sdwa")) {
1394     setForcedSDWA(true);
1395     return Name.substr(0, Name.size() - 5);
1396   }
1397   return Name;
1398 }
1399 
1400 bool AMDGPUAsmParser::ParseInstruction(ParseInstructionInfo &Info,
1401                                        StringRef Name,
1402                                        SMLoc NameLoc, OperandVector &Operands) {
1403   // Add the instruction mnemonic
1404   Name = parseMnemonicSuffix(Name);
1405   Operands.push_back(AMDGPUOperand::CreateToken(Name, NameLoc));
1406 
1407   while (!getLexer().is(AsmToken::EndOfStatement)) {
1408     AMDGPUAsmParser::OperandMatchResultTy Res = parseOperand(Operands, Name);
1409 
1410     // Eat the comma or space if there is one.
1411     if (getLexer().is(AsmToken::Comma))
1412       Parser.Lex();
1413 
1414     switch (Res) {
1415       case MatchOperand_Success: break;
1416       case MatchOperand_ParseFail:
1417         Error(getLexer().getLoc(), "failed parsing operand.");
1418         while (!getLexer().is(AsmToken::EndOfStatement)) {
1419           Parser.Lex();
1420         }
1421         return true;
1422       case MatchOperand_NoMatch:
1423         Error(getLexer().getLoc(), "not a valid operand.");
1424         while (!getLexer().is(AsmToken::EndOfStatement)) {
1425           Parser.Lex();
1426         }
1427         return true;
1428     }
1429   }
1430 
1431   return false;
1432 }
1433 
1434 //===----------------------------------------------------------------------===//
1435 // Utility functions
1436 //===----------------------------------------------------------------------===//
1437 
1438 AMDGPUAsmParser::OperandMatchResultTy
1439 AMDGPUAsmParser::parseIntWithPrefix(const char *Prefix, int64_t &Int) {
1440   switch(getLexer().getKind()) {
1441     default: return MatchOperand_NoMatch;
1442     case AsmToken::Identifier: {
1443       StringRef Name = Parser.getTok().getString();
1444       if (!Name.equals(Prefix)) {
1445         return MatchOperand_NoMatch;
1446       }
1447 
1448       Parser.Lex();
1449       if (getLexer().isNot(AsmToken::Colon))
1450         return MatchOperand_ParseFail;
1451 
1452       Parser.Lex();
1453       if (getLexer().isNot(AsmToken::Integer))
1454         return MatchOperand_ParseFail;
1455 
1456       if (getParser().parseAbsoluteExpression(Int))
1457         return MatchOperand_ParseFail;
1458       break;
1459     }
1460   }
1461   return MatchOperand_Success;
1462 }
1463 
1464 AMDGPUAsmParser::OperandMatchResultTy
1465 AMDGPUAsmParser::parseIntWithPrefix(const char *Prefix, OperandVector &Operands,
1466                                     enum AMDGPUOperand::ImmTy ImmTy,
1467                                     bool (*ConvertResult)(int64_t&)) {
1468 
1469   SMLoc S = Parser.getTok().getLoc();
1470   int64_t Value = 0;
1471 
1472   AMDGPUAsmParser::OperandMatchResultTy Res = parseIntWithPrefix(Prefix, Value);
1473   if (Res != MatchOperand_Success)
1474     return Res;
1475 
1476   if (ConvertResult && !ConvertResult(Value)) {
1477     return MatchOperand_ParseFail;
1478   }
1479 
1480   Operands.push_back(AMDGPUOperand::CreateImm(Value, S, ImmTy));
1481   return MatchOperand_Success;
1482 }
1483 
1484 AMDGPUAsmParser::OperandMatchResultTy
1485 AMDGPUAsmParser::parseNamedBit(const char *Name, OperandVector &Operands,
1486                                enum AMDGPUOperand::ImmTy ImmTy) {
1487   int64_t Bit = 0;
1488   SMLoc S = Parser.getTok().getLoc();
1489 
1490   // We are at the end of the statement, and this is a default argument, so
1491   // use a default value.
1492   if (getLexer().isNot(AsmToken::EndOfStatement)) {
1493     switch(getLexer().getKind()) {
1494       case AsmToken::Identifier: {
1495         StringRef Tok = Parser.getTok().getString();
1496         if (Tok == Name) {
1497           Bit = 1;
1498           Parser.Lex();
1499         } else if (Tok.startswith("no") && Tok.endswith(Name)) {
1500           Bit = 0;
1501           Parser.Lex();
1502         } else {
1503           return MatchOperand_NoMatch;
1504         }
1505         break;
1506       }
1507       default:
1508         return MatchOperand_NoMatch;
1509     }
1510   }
1511 
1512   Operands.push_back(AMDGPUOperand::CreateImm(Bit, S, ImmTy));
1513   return MatchOperand_Success;
1514 }
1515 
1516 typedef std::map<enum AMDGPUOperand::ImmTy, unsigned> OptionalImmIndexMap;
1517 
1518 void addOptionalImmOperand(MCInst& Inst, const OperandVector& Operands,
1519                            OptionalImmIndexMap& OptionalIdx,
1520                            enum AMDGPUOperand::ImmTy ImmT, int64_t Default = 0) {
1521   auto i = OptionalIdx.find(ImmT);
1522   if (i != OptionalIdx.end()) {
1523     unsigned Idx = i->second;
1524     ((AMDGPUOperand &)*Operands[Idx]).addImmOperands(Inst, 1);
1525   } else {
1526     Inst.addOperand(MCOperand::createImm(Default));
1527   }
1528 }
1529 
1530 AMDGPUAsmParser::OperandMatchResultTy
1531 AMDGPUAsmParser::parseStringWithPrefix(StringRef Prefix, StringRef &Value) {
1532   if (getLexer().isNot(AsmToken::Identifier)) {
1533     return MatchOperand_NoMatch;
1534   }
1535   StringRef Tok = Parser.getTok().getString();
1536   if (Tok != Prefix) {
1537     return MatchOperand_NoMatch;
1538   }
1539 
1540   Parser.Lex();
1541   if (getLexer().isNot(AsmToken::Colon)) {
1542     return MatchOperand_ParseFail;
1543   }
1544 
1545   Parser.Lex();
1546   if (getLexer().isNot(AsmToken::Identifier)) {
1547     return MatchOperand_ParseFail;
1548   }
1549 
1550   Value = Parser.getTok().getString();
1551   return MatchOperand_Success;
1552 }
1553 
1554 //===----------------------------------------------------------------------===//
1555 // ds
1556 //===----------------------------------------------------------------------===//
1557 
1558 void AMDGPUAsmParser::cvtDSOffset01(MCInst &Inst,
1559                                     const OperandVector &Operands) {
1560 
1561   OptionalImmIndexMap OptionalIdx;
1562 
1563   for (unsigned i = 1, e = Operands.size(); i != e; ++i) {
1564     AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[i]);
1565 
1566     // Add the register arguments
1567     if (Op.isReg()) {
1568       Op.addRegOperands(Inst, 1);
1569       continue;
1570     }
1571 
1572     // Handle optional arguments
1573     OptionalIdx[Op.getImmTy()] = i;
1574   }
1575 
1576   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyOffset0);
1577   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyOffset1);
1578   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyGDS);
1579 
1580   Inst.addOperand(MCOperand::createReg(AMDGPU::M0)); // m0
1581 }
1582 
1583 void AMDGPUAsmParser::cvtDS(MCInst &Inst, const OperandVector &Operands) {
1584 
1585   std::map<enum AMDGPUOperand::ImmTy, unsigned> OptionalIdx;
1586   bool GDSOnly = false;
1587 
1588   for (unsigned i = 1, e = Operands.size(); i != e; ++i) {
1589     AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[i]);
1590 
1591     // Add the register arguments
1592     if (Op.isReg()) {
1593       Op.addRegOperands(Inst, 1);
1594       continue;
1595     }
1596 
1597     if (Op.isToken() && Op.getToken() == "gds") {
1598       GDSOnly = true;
1599       continue;
1600     }
1601 
1602     // Handle optional arguments
1603     OptionalIdx[Op.getImmTy()] = i;
1604   }
1605 
1606   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyOffset);
1607   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyGDS);
1608 
1609   if (!GDSOnly) {
1610     addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyGDS);
1611   }
1612   Inst.addOperand(MCOperand::createReg(AMDGPU::M0)); // m0
1613 }
1614 
1615 
1616 //===----------------------------------------------------------------------===//
1617 // s_waitcnt
1618 //===----------------------------------------------------------------------===//
1619 
1620 bool AMDGPUAsmParser::parseCnt(int64_t &IntVal) {
1621   StringRef CntName = Parser.getTok().getString();
1622   int64_t CntVal;
1623 
1624   Parser.Lex();
1625   if (getLexer().isNot(AsmToken::LParen))
1626     return true;
1627 
1628   Parser.Lex();
1629   if (getLexer().isNot(AsmToken::Integer))
1630     return true;
1631 
1632   if (getParser().parseAbsoluteExpression(CntVal))
1633     return true;
1634 
1635   if (getLexer().isNot(AsmToken::RParen))
1636     return true;
1637 
1638   Parser.Lex();
1639   if (getLexer().is(AsmToken::Amp) || getLexer().is(AsmToken::Comma))
1640     Parser.Lex();
1641 
1642   int CntShift;
1643   int CntMask;
1644 
1645   if (CntName == "vmcnt") {
1646     CntMask = 0xf;
1647     CntShift = 0;
1648   } else if (CntName == "expcnt") {
1649     CntMask = 0x7;
1650     CntShift = 4;
1651   } else if (CntName == "lgkmcnt") {
1652     CntMask = 0xf;
1653     CntShift = 8;
1654   } else {
1655     return true;
1656   }
1657 
1658   IntVal &= ~(CntMask << CntShift);
1659   IntVal |= (CntVal << CntShift);
1660   return false;
1661 }
1662 
1663 AMDGPUAsmParser::OperandMatchResultTy
1664 AMDGPUAsmParser::parseSWaitCntOps(OperandVector &Operands) {
1665   // Disable all counters by default.
1666   // vmcnt   [3:0]
1667   // expcnt  [6:4]
1668   // lgkmcnt [11:8]
1669   int64_t CntVal = 0xf7f;
1670   SMLoc S = Parser.getTok().getLoc();
1671 
1672   switch(getLexer().getKind()) {
1673     default: return MatchOperand_ParseFail;
1674     case AsmToken::Integer:
1675       // The operand can be an integer value.
1676       if (getParser().parseAbsoluteExpression(CntVal))
1677         return MatchOperand_ParseFail;
1678       break;
1679 
1680     case AsmToken::Identifier:
1681       do {
1682         if (parseCnt(CntVal))
1683           return MatchOperand_ParseFail;
1684       } while(getLexer().isNot(AsmToken::EndOfStatement));
1685       break;
1686   }
1687   Operands.push_back(AMDGPUOperand::CreateImm(CntVal, S));
1688   return MatchOperand_Success;
1689 }
1690 
1691 bool AMDGPUAsmParser::parseHwregConstruct(OperandInfoTy &HwReg, int64_t &Offset, int64_t &Width) {
1692   using namespace llvm::AMDGPU::Hwreg;
1693 
1694   if (Parser.getTok().getString() != "hwreg")
1695     return true;
1696   Parser.Lex();
1697 
1698   if (getLexer().isNot(AsmToken::LParen))
1699     return true;
1700   Parser.Lex();
1701 
1702   if (getLexer().is(AsmToken::Identifier)) {
1703     HwReg.IsSymbolic = true;
1704     HwReg.Id = ID_UNKNOWN_;
1705     const StringRef tok = Parser.getTok().getString();
1706     for (int i = ID_SYMBOLIC_FIRST_; i < ID_SYMBOLIC_LAST_; ++i) {
1707       if (tok == IdSymbolic[i]) {
1708         HwReg.Id = i;
1709         break;
1710       }
1711     }
1712     Parser.Lex();
1713   } else {
1714     HwReg.IsSymbolic = false;
1715     if (getLexer().isNot(AsmToken::Integer))
1716       return true;
1717     if (getParser().parseAbsoluteExpression(HwReg.Id))
1718       return true;
1719   }
1720 
1721   if (getLexer().is(AsmToken::RParen)) {
1722     Parser.Lex();
1723     return false;
1724   }
1725 
1726   // optional params
1727   if (getLexer().isNot(AsmToken::Comma))
1728     return true;
1729   Parser.Lex();
1730 
1731   if (getLexer().isNot(AsmToken::Integer))
1732     return true;
1733   if (getParser().parseAbsoluteExpression(Offset))
1734     return true;
1735 
1736   if (getLexer().isNot(AsmToken::Comma))
1737     return true;
1738   Parser.Lex();
1739 
1740   if (getLexer().isNot(AsmToken::Integer))
1741     return true;
1742   if (getParser().parseAbsoluteExpression(Width))
1743     return true;
1744 
1745   if (getLexer().isNot(AsmToken::RParen))
1746     return true;
1747   Parser.Lex();
1748 
1749   return false;
1750 }
1751 
1752 AMDGPUAsmParser::OperandMatchResultTy
1753 AMDGPUAsmParser::parseHwreg(OperandVector &Operands) {
1754   using namespace llvm::AMDGPU::Hwreg;
1755 
1756   int64_t Imm16Val = 0;
1757   SMLoc S = Parser.getTok().getLoc();
1758 
1759   switch(getLexer().getKind()) {
1760     default: return MatchOperand_NoMatch;
1761     case AsmToken::Integer:
1762       // The operand can be an integer value.
1763       if (getParser().parseAbsoluteExpression(Imm16Val))
1764         return MatchOperand_NoMatch;
1765       if (Imm16Val < 0 || !isUInt<16>(Imm16Val)) {
1766         Error(S, "invalid immediate: only 16-bit values are legal");
1767         // Do not return error code, but create an imm operand anyway and proceed
1768         // to the next operand, if any. That avoids unneccessary error messages.
1769       }
1770       break;
1771 
1772     case AsmToken::Identifier: {
1773         OperandInfoTy HwReg(ID_UNKNOWN_);
1774         int64_t Offset = OFFSET_DEFAULT_;
1775         int64_t Width = WIDTH_M1_DEFAULT_ + 1;
1776         if (parseHwregConstruct(HwReg, Offset, Width))
1777           return MatchOperand_ParseFail;
1778         if (HwReg.Id < 0 || !isUInt<ID_WIDTH_>(HwReg.Id)) {
1779           if (HwReg.IsSymbolic)
1780             Error(S, "invalid symbolic name of hardware register");
1781           else
1782             Error(S, "invalid code of hardware register: only 6-bit values are legal");
1783         }
1784         if (Offset < 0 || !isUInt<OFFSET_WIDTH_>(Offset))
1785           Error(S, "invalid bit offset: only 5-bit values are legal");
1786         if ((Width-1) < 0 || !isUInt<WIDTH_M1_WIDTH_>(Width-1))
1787           Error(S, "invalid bitfield width: only values from 1 to 32 are legal");
1788         Imm16Val = (HwReg.Id << ID_SHIFT_) | (Offset << OFFSET_SHIFT_) | ((Width-1) << WIDTH_M1_SHIFT_);
1789       }
1790       break;
1791   }
1792   Operands.push_back(AMDGPUOperand::CreateImm(Imm16Val, S, AMDGPUOperand::ImmTyHwreg));
1793   return MatchOperand_Success;
1794 }
1795 
1796 bool AMDGPUOperand::isSWaitCnt() const {
1797   return isImm();
1798 }
1799 
1800 bool AMDGPUOperand::isHwreg() const {
1801   return isImmTy(ImmTyHwreg);
1802 }
1803 
1804 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultHwreg() const {
1805   return AMDGPUOperand::CreateImm(0, SMLoc(), AMDGPUOperand::ImmTyHwreg);
1806 }
1807 
1808 bool AMDGPUAsmParser::parseSendMsgConstruct(OperandInfoTy &Msg, OperandInfoTy &Operation, int64_t &StreamId) {
1809   using namespace llvm::AMDGPU::SendMsg;
1810 
1811   if (Parser.getTok().getString() != "sendmsg")
1812     return true;
1813   Parser.Lex();
1814 
1815   if (getLexer().isNot(AsmToken::LParen))
1816     return true;
1817   Parser.Lex();
1818 
1819   if (getLexer().is(AsmToken::Identifier)) {
1820     Msg.IsSymbolic = true;
1821     Msg.Id = ID_UNKNOWN_;
1822     const std::string tok = Parser.getTok().getString();
1823     for (int i = ID_GAPS_FIRST_; i < ID_GAPS_LAST_; ++i) {
1824       switch(i) {
1825         default: continue; // Omit gaps.
1826         case ID_INTERRUPT: case ID_GS: case ID_GS_DONE:  case ID_SYSMSG: break;
1827       }
1828       if (tok == IdSymbolic[i]) {
1829         Msg.Id = i;
1830         break;
1831       }
1832     }
1833     Parser.Lex();
1834   } else {
1835     Msg.IsSymbolic = false;
1836     if (getLexer().isNot(AsmToken::Integer))
1837       return true;
1838     if (getParser().parseAbsoluteExpression(Msg.Id))
1839       return true;
1840     if (getLexer().is(AsmToken::Integer))
1841       if (getParser().parseAbsoluteExpression(Msg.Id))
1842         Msg.Id = ID_UNKNOWN_;
1843   }
1844   if (Msg.Id == ID_UNKNOWN_) // Don't know how to parse the rest.
1845     return false;
1846 
1847   if (!(Msg.Id == ID_GS || Msg.Id == ID_GS_DONE || Msg.Id == ID_SYSMSG)) {
1848     if (getLexer().isNot(AsmToken::RParen))
1849       return true;
1850     Parser.Lex();
1851     return false;
1852   }
1853 
1854   if (getLexer().isNot(AsmToken::Comma))
1855     return true;
1856   Parser.Lex();
1857 
1858   assert(Msg.Id == ID_GS || Msg.Id == ID_GS_DONE || Msg.Id == ID_SYSMSG);
1859   Operation.Id = ID_UNKNOWN_;
1860   if (getLexer().is(AsmToken::Identifier)) {
1861     Operation.IsSymbolic = true;
1862     const char* const *S = (Msg.Id == ID_SYSMSG) ? OpSysSymbolic : OpGsSymbolic;
1863     const int F = (Msg.Id == ID_SYSMSG) ? OP_SYS_FIRST_ : OP_GS_FIRST_;
1864     const int L = (Msg.Id == ID_SYSMSG) ? OP_SYS_LAST_ : OP_GS_LAST_;
1865     const StringRef Tok = Parser.getTok().getString();
1866     for (int i = F; i < L; ++i) {
1867       if (Tok == S[i]) {
1868         Operation.Id = i;
1869         break;
1870       }
1871     }
1872     Parser.Lex();
1873   } else {
1874     Operation.IsSymbolic = false;
1875     if (getLexer().isNot(AsmToken::Integer))
1876       return true;
1877     if (getParser().parseAbsoluteExpression(Operation.Id))
1878       return true;
1879   }
1880 
1881   if ((Msg.Id == ID_GS || Msg.Id == ID_GS_DONE) && Operation.Id != OP_GS_NOP) {
1882     // Stream id is optional.
1883     if (getLexer().is(AsmToken::RParen)) {
1884       Parser.Lex();
1885       return false;
1886     }
1887 
1888     if (getLexer().isNot(AsmToken::Comma))
1889       return true;
1890     Parser.Lex();
1891 
1892     if (getLexer().isNot(AsmToken::Integer))
1893       return true;
1894     if (getParser().parseAbsoluteExpression(StreamId))
1895       return true;
1896   }
1897 
1898   if (getLexer().isNot(AsmToken::RParen))
1899     return true;
1900   Parser.Lex();
1901   return false;
1902 }
1903 
1904 AMDGPUAsmParser::OperandMatchResultTy
1905 AMDGPUAsmParser::parseSendMsgOp(OperandVector &Operands) {
1906   using namespace llvm::AMDGPU::SendMsg;
1907 
1908   int64_t Imm16Val = 0;
1909   SMLoc S = Parser.getTok().getLoc();
1910 
1911   switch(getLexer().getKind()) {
1912   default:
1913     return MatchOperand_NoMatch;
1914   case AsmToken::Integer:
1915     // The operand can be an integer value.
1916     if (getParser().parseAbsoluteExpression(Imm16Val))
1917       return MatchOperand_NoMatch;
1918     if (Imm16Val < 0 || !isUInt<16>(Imm16Val)) {
1919       Error(S, "invalid immediate: only 16-bit values are legal");
1920       // Do not return error code, but create an imm operand anyway and proceed
1921       // to the next operand, if any. That avoids unneccessary error messages.
1922     }
1923     break;
1924   case AsmToken::Identifier: {
1925       OperandInfoTy Msg(ID_UNKNOWN_);
1926       OperandInfoTy Operation(OP_UNKNOWN_);
1927       int64_t StreamId = STREAM_ID_DEFAULT_;
1928       if (parseSendMsgConstruct(Msg, Operation, StreamId))
1929         return MatchOperand_ParseFail;
1930       do {
1931         // Validate and encode message ID.
1932         if (! ((ID_INTERRUPT <= Msg.Id && Msg.Id <= ID_GS_DONE)
1933                 || Msg.Id == ID_SYSMSG)) {
1934           if (Msg.IsSymbolic)
1935             Error(S, "invalid/unsupported symbolic name of message");
1936           else
1937             Error(S, "invalid/unsupported code of message");
1938           break;
1939         }
1940         Imm16Val = (Msg.Id << ID_SHIFT_);
1941         // Validate and encode operation ID.
1942         if (Msg.Id == ID_GS || Msg.Id == ID_GS_DONE) {
1943           if (! (OP_GS_FIRST_ <= Operation.Id && Operation.Id < OP_GS_LAST_)) {
1944             if (Operation.IsSymbolic)
1945               Error(S, "invalid symbolic name of GS_OP");
1946             else
1947               Error(S, "invalid code of GS_OP: only 2-bit values are legal");
1948             break;
1949           }
1950           if (Operation.Id == OP_GS_NOP
1951               && Msg.Id != ID_GS_DONE) {
1952             Error(S, "invalid GS_OP: NOP is for GS_DONE only");
1953             break;
1954           }
1955           Imm16Val |= (Operation.Id << OP_SHIFT_);
1956         }
1957         if (Msg.Id == ID_SYSMSG) {
1958           if (! (OP_SYS_FIRST_ <= Operation.Id && Operation.Id < OP_SYS_LAST_)) {
1959             if (Operation.IsSymbolic)
1960               Error(S, "invalid/unsupported symbolic name of SYSMSG_OP");
1961             else
1962               Error(S, "invalid/unsupported code of SYSMSG_OP");
1963             break;
1964           }
1965           Imm16Val |= (Operation.Id << OP_SHIFT_);
1966         }
1967         // Validate and encode stream ID.
1968         if ((Msg.Id == ID_GS || Msg.Id == ID_GS_DONE) && Operation.Id != OP_GS_NOP) {
1969           if (! (STREAM_ID_FIRST_ <= StreamId && StreamId < STREAM_ID_LAST_)) {
1970             Error(S, "invalid stream id: only 2-bit values are legal");
1971             break;
1972           }
1973           Imm16Val |= (StreamId << STREAM_ID_SHIFT_);
1974         }
1975       } while (0);
1976     }
1977     break;
1978   }
1979   Operands.push_back(AMDGPUOperand::CreateImm(Imm16Val, S, AMDGPUOperand::ImmTySendMsg));
1980   return MatchOperand_Success;
1981 }
1982 
1983 bool AMDGPUOperand::isSendMsg() const {
1984   return isImmTy(ImmTySendMsg);
1985 }
1986 
1987 //===----------------------------------------------------------------------===//
1988 // sopp branch targets
1989 //===----------------------------------------------------------------------===//
1990 
1991 AMDGPUAsmParser::OperandMatchResultTy
1992 AMDGPUAsmParser::parseSOppBrTarget(OperandVector &Operands) {
1993   SMLoc S = Parser.getTok().getLoc();
1994 
1995   switch (getLexer().getKind()) {
1996     default: return MatchOperand_ParseFail;
1997     case AsmToken::Integer: {
1998       int64_t Imm;
1999       if (getParser().parseAbsoluteExpression(Imm))
2000         return MatchOperand_ParseFail;
2001       Operands.push_back(AMDGPUOperand::CreateImm(Imm, S));
2002       return MatchOperand_Success;
2003     }
2004 
2005     case AsmToken::Identifier:
2006       Operands.push_back(AMDGPUOperand::CreateExpr(
2007           MCSymbolRefExpr::create(getContext().getOrCreateSymbol(
2008                                   Parser.getTok().getString()), getContext()), S));
2009       Parser.Lex();
2010       return MatchOperand_Success;
2011   }
2012 }
2013 
2014 //===----------------------------------------------------------------------===//
2015 // mubuf
2016 //===----------------------------------------------------------------------===//
2017 
2018 bool AMDGPUOperand::isMubufOffset() const {
2019   return isImmTy(ImmTyOffset) && isUInt<12>(getImm());
2020 }
2021 
2022 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultMubufOffset() const {
2023   return AMDGPUOperand::CreateImm(0, SMLoc(), AMDGPUOperand::ImmTyOffset);
2024 }
2025 
2026 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultGLC() const {
2027   return AMDGPUOperand::CreateImm(0, SMLoc(), AMDGPUOperand::ImmTyGLC);
2028 }
2029 
2030 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultSLC() const {
2031   return AMDGPUOperand::CreateImm(0, SMLoc(), AMDGPUOperand::ImmTySLC);
2032 }
2033 
2034 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultTFE() const {
2035   return AMDGPUOperand::CreateImm(0, SMLoc(), AMDGPUOperand::ImmTyTFE);
2036 }
2037 
2038 void AMDGPUAsmParser::cvtMubufImpl(MCInst &Inst,
2039                                const OperandVector &Operands,
2040                                bool IsAtomic, bool IsAtomicReturn) {
2041   OptionalImmIndexMap OptionalIdx;
2042   assert(IsAtomicReturn ? IsAtomic : true);
2043 
2044   for (unsigned i = 1, e = Operands.size(); i != e; ++i) {
2045     AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[i]);
2046 
2047     // Add the register arguments
2048     if (Op.isReg()) {
2049       Op.addRegOperands(Inst, 1);
2050       continue;
2051     }
2052 
2053     // Handle the case where soffset is an immediate
2054     if (Op.isImm() && Op.getImmTy() == AMDGPUOperand::ImmTyNone) {
2055       Op.addImmOperands(Inst, 1);
2056       continue;
2057     }
2058 
2059     // Handle tokens like 'offen' which are sometimes hard-coded into the
2060     // asm string.  There are no MCInst operands for these.
2061     if (Op.isToken()) {
2062       continue;
2063     }
2064     assert(Op.isImm());
2065 
2066     // Handle optional arguments
2067     OptionalIdx[Op.getImmTy()] = i;
2068   }
2069 
2070   // Copy $vdata_in operand and insert as $vdata for MUBUF_Atomic RTN insns.
2071   if (IsAtomicReturn) {
2072     MCInst::iterator I = Inst.begin(); // $vdata_in is always at the beginning.
2073     Inst.insert(I, *I);
2074   }
2075 
2076   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyOffset);
2077   if (!IsAtomic) { // glc is hard-coded.
2078     addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyGLC);
2079   }
2080   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySLC);
2081   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyTFE);
2082 }
2083 
2084 //===----------------------------------------------------------------------===//
2085 // mimg
2086 //===----------------------------------------------------------------------===//
2087 
2088 void AMDGPUAsmParser::cvtMIMG(MCInst &Inst, const OperandVector &Operands) {
2089   unsigned I = 1;
2090   const MCInstrDesc &Desc = MII.get(Inst.getOpcode());
2091   for (unsigned J = 0; J < Desc.getNumDefs(); ++J) {
2092     ((AMDGPUOperand &)*Operands[I++]).addRegOperands(Inst, 1);
2093   }
2094 
2095   OptionalImmIndexMap OptionalIdx;
2096 
2097   for (unsigned E = Operands.size(); I != E; ++I) {
2098     AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[I]);
2099 
2100     // Add the register arguments
2101     if (Op.isRegOrImm()) {
2102       Op.addRegOrImmOperands(Inst, 1);
2103       continue;
2104     } else if (Op.isImmModifier()) {
2105       OptionalIdx[Op.getImmTy()] = I;
2106     } else {
2107       assert(false);
2108     }
2109   }
2110 
2111   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyDMask);
2112   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyUNorm);
2113   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyGLC);
2114   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyDA);
2115   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyR128);
2116   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyTFE);
2117   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyLWE);
2118   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySLC);
2119 }
2120 
2121 void AMDGPUAsmParser::cvtMIMGAtomic(MCInst &Inst, const OperandVector &Operands) {
2122   unsigned I = 1;
2123   const MCInstrDesc &Desc = MII.get(Inst.getOpcode());
2124   for (unsigned J = 0; J < Desc.getNumDefs(); ++J) {
2125     ((AMDGPUOperand &)*Operands[I++]).addRegOperands(Inst, 1);
2126   }
2127 
2128   // Add src, same as dst
2129   ((AMDGPUOperand &)*Operands[I]).addRegOperands(Inst, 1);
2130 
2131   OptionalImmIndexMap OptionalIdx;
2132 
2133   for (unsigned E = Operands.size(); I != E; ++I) {
2134     AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[I]);
2135 
2136     // Add the register arguments
2137     if (Op.isRegOrImm()) {
2138       Op.addRegOrImmOperands(Inst, 1);
2139       continue;
2140     } else if (Op.isImmModifier()) {
2141       OptionalIdx[Op.getImmTy()] = I;
2142     } else {
2143       assert(false);
2144     }
2145   }
2146 
2147   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyDMask);
2148   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyUNorm);
2149   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyGLC);
2150   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyDA);
2151   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyR128);
2152   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyTFE);
2153   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyLWE);
2154   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySLC);
2155 }
2156 
2157 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultDMask() const {
2158   return AMDGPUOperand::CreateImm(0, SMLoc(), AMDGPUOperand::ImmTyDMask);
2159 }
2160 
2161 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultUNorm() const {
2162   return AMDGPUOperand::CreateImm(0, SMLoc(), AMDGPUOperand::ImmTyUNorm);
2163 }
2164 
2165 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultDA() const {
2166   return AMDGPUOperand::CreateImm(0, SMLoc(), AMDGPUOperand::ImmTyDA);
2167 }
2168 
2169 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultR128() const {
2170   return AMDGPUOperand::CreateImm(0, SMLoc(), AMDGPUOperand::ImmTyR128);
2171 }
2172 
2173 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultLWE() const {
2174   return AMDGPUOperand::CreateImm(0, SMLoc(), AMDGPUOperand::ImmTyLWE);
2175 }
2176 
2177 //===----------------------------------------------------------------------===//
2178 // smrd
2179 //===----------------------------------------------------------------------===//
2180 
2181 bool AMDGPUOperand::isSMRDOffset() const {
2182 
2183   // FIXME: Support 20-bit offsets on VI.  We need to to pass subtarget
2184   // information here.
2185   return isImm() && isUInt<8>(getImm());
2186 }
2187 
2188 bool AMDGPUOperand::isSMRDLiteralOffset() const {
2189   // 32-bit literals are only supported on CI and we only want to use them
2190   // when the offset is > 8-bits.
2191   return isImm() && !isUInt<8>(getImm()) && isUInt<32>(getImm());
2192 }
2193 
2194 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultSMRDOffset() const {
2195   return AMDGPUOperand::CreateImm(0, SMLoc(), AMDGPUOperand::ImmTyOffset);
2196 }
2197 
2198 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultSMRDLiteralOffset() const {
2199   return AMDGPUOperand::CreateImm(0, SMLoc(), AMDGPUOperand::ImmTyOffset);
2200 }
2201 
2202 //===----------------------------------------------------------------------===//
2203 // vop3
2204 //===----------------------------------------------------------------------===//
2205 
2206 static bool ConvertOmodMul(int64_t &Mul) {
2207   if (Mul != 1 && Mul != 2 && Mul != 4)
2208     return false;
2209 
2210   Mul >>= 1;
2211   return true;
2212 }
2213 
2214 static bool ConvertOmodDiv(int64_t &Div) {
2215   if (Div == 1) {
2216     Div = 0;
2217     return true;
2218   }
2219 
2220   if (Div == 2) {
2221     Div = 3;
2222     return true;
2223   }
2224 
2225   return false;
2226 }
2227 
2228 static bool ConvertBoundCtrl(int64_t &BoundCtrl) {
2229   if (BoundCtrl == 0) {
2230     BoundCtrl = 1;
2231     return true;
2232   } else if (BoundCtrl == -1) {
2233     BoundCtrl = 0;
2234     return true;
2235   }
2236   return false;
2237 }
2238 
2239 // Note: the order in this table matches the order of operands in AsmString.
2240 static const OptionalOperand AMDGPUOptionalOperandTable[] = {
2241   {"offen",   AMDGPUOperand::ImmTyOffen, true, nullptr},
2242   {"idxen",   AMDGPUOperand::ImmTyIdxen, true, nullptr},
2243   {"addr64",  AMDGPUOperand::ImmTyAddr64, true, nullptr},
2244   {"offset0", AMDGPUOperand::ImmTyOffset0, false, nullptr},
2245   {"offset1", AMDGPUOperand::ImmTyOffset1, false, nullptr},
2246   {"gds",     AMDGPUOperand::ImmTyGDS, true, nullptr},
2247   {"offset",  AMDGPUOperand::ImmTyOffset, false, nullptr},
2248   {"glc",     AMDGPUOperand::ImmTyGLC, true, nullptr},
2249   {"slc",     AMDGPUOperand::ImmTySLC, true, nullptr},
2250   {"tfe",     AMDGPUOperand::ImmTyTFE, true, nullptr},
2251   {"clamp",   AMDGPUOperand::ImmTyClampSI, true, nullptr},
2252   {"omod",    AMDGPUOperand::ImmTyOModSI, false, ConvertOmodMul},
2253   {"unorm",   AMDGPUOperand::ImmTyUNorm, true, nullptr},
2254   {"da",      AMDGPUOperand::ImmTyDA,    true, nullptr},
2255   {"r128",    AMDGPUOperand::ImmTyR128,  true, nullptr},
2256   {"lwe",     AMDGPUOperand::ImmTyLWE,   true, nullptr},
2257   {"dmask",   AMDGPUOperand::ImmTyDMask, false, nullptr},
2258   {"row_mask",   AMDGPUOperand::ImmTyDppRowMask, false, nullptr},
2259   {"bank_mask",  AMDGPUOperand::ImmTyDppBankMask, false, nullptr},
2260   {"bound_ctrl", AMDGPUOperand::ImmTyDppBoundCtrl, false, ConvertBoundCtrl},
2261   {"dst_sel",    AMDGPUOperand::ImmTySdwaDstSel, false, nullptr},
2262   {"src0_sel",   AMDGPUOperand::ImmTySdwaSrc0Sel, false, nullptr},
2263   {"src1_sel",   AMDGPUOperand::ImmTySdwaSrc1Sel, false, nullptr},
2264   {"dst_unused", AMDGPUOperand::ImmTySdwaDstUnused, false, nullptr},
2265 };
2266 
2267 AMDGPUAsmParser::OperandMatchResultTy AMDGPUAsmParser::parseOptionalOperand(OperandVector &Operands) {
2268   OperandMatchResultTy res;
2269   for (const OptionalOperand &Op : AMDGPUOptionalOperandTable) {
2270     // try to parse any optional operand here
2271     if (Op.IsBit) {
2272       res = parseNamedBit(Op.Name, Operands, Op.Type);
2273     } else if (Op.Type == AMDGPUOperand::ImmTyOModSI) {
2274       res = parseOModOperand(Operands);
2275     } else if (Op.Type == AMDGPUOperand::ImmTySdwaDstSel ||
2276                Op.Type == AMDGPUOperand::ImmTySdwaSrc0Sel ||
2277                Op.Type == AMDGPUOperand::ImmTySdwaSrc1Sel) {
2278       res = parseSDWASel(Operands, Op.Name, Op.Type);
2279     } else if (Op.Type == AMDGPUOperand::ImmTySdwaDstUnused) {
2280       res = parseSDWADstUnused(Operands);
2281     } else {
2282       res = parseIntWithPrefix(Op.Name, Operands, Op.Type, Op.ConvertResult);
2283     }
2284     if (res != MatchOperand_NoMatch) {
2285       return res;
2286     }
2287   }
2288   return MatchOperand_NoMatch;
2289 }
2290 
2291 AMDGPUAsmParser::OperandMatchResultTy AMDGPUAsmParser::parseOModOperand(OperandVector &Operands)
2292 {
2293   StringRef Name = Parser.getTok().getString();
2294   if (Name == "mul") {
2295     return parseIntWithPrefix("mul", Operands, AMDGPUOperand::ImmTyOModSI, ConvertOmodMul);
2296   } else if (Name == "div") {
2297     return parseIntWithPrefix("div", Operands, AMDGPUOperand::ImmTyOModSI, ConvertOmodDiv);
2298   } else {
2299     return MatchOperand_NoMatch;
2300   }
2301 }
2302 
2303 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultClampSI() const {
2304   return AMDGPUOperand::CreateImm(0, SMLoc(), AMDGPUOperand::ImmTyClampSI);
2305 }
2306 
2307 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultOModSI() const {
2308   return AMDGPUOperand::CreateImm(1, SMLoc(), AMDGPUOperand::ImmTyOModSI);
2309 }
2310 
2311 void AMDGPUAsmParser::cvtId(MCInst &Inst, const OperandVector &Operands) {
2312   unsigned I = 1;
2313   const MCInstrDesc &Desc = MII.get(Inst.getOpcode());
2314   for (unsigned J = 0; J < Desc.getNumDefs(); ++J) {
2315     ((AMDGPUOperand &)*Operands[I++]).addRegOperands(Inst, 1);
2316   }
2317   for (unsigned E = Operands.size(); I != E; ++I)
2318     ((AMDGPUOperand &)*Operands[I]).addRegOrImmOperands(Inst, 1);
2319 }
2320 
2321 void AMDGPUAsmParser::cvtVOP3_2_mod(MCInst &Inst, const OperandVector &Operands) {
2322   uint64_t TSFlags = MII.get(Inst.getOpcode()).TSFlags;
2323   if (TSFlags & SIInstrFlags::VOP3) {
2324     cvtVOP3(Inst, Operands);
2325   } else {
2326     cvtId(Inst, Operands);
2327   }
2328 }
2329 
2330 void AMDGPUAsmParser::cvtVOP3(MCInst &Inst, const OperandVector &Operands) {
2331   OptionalImmIndexMap OptionalIdx;
2332   unsigned I = 1;
2333   const MCInstrDesc &Desc = MII.get(Inst.getOpcode());
2334   for (unsigned J = 0; J < Desc.getNumDefs(); ++J) {
2335     ((AMDGPUOperand &)*Operands[I++]).addRegOperands(Inst, 1);
2336   }
2337 
2338   for (unsigned E = Operands.size(); I != E; ++I) {
2339     AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[I]);
2340     if (Op.isRegOrImmWithInputMods()) {
2341       Op.addRegOrImmWithInputModsOperands(Inst, 2);
2342     } else if (Op.isImm()) {
2343       OptionalIdx[Op.getImmTy()] = I;
2344     } else {
2345       assert(false);
2346     }
2347   }
2348 
2349   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyClampSI);
2350   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyOModSI);
2351 }
2352 
2353 //===----------------------------------------------------------------------===//
2354 // dpp
2355 //===----------------------------------------------------------------------===//
2356 
2357 bool AMDGPUOperand::isDPPCtrl() const {
2358   bool result = isImm() && getImmTy() == ImmTyDppCtrl && isUInt<9>(getImm());
2359   if (result) {
2360     int64_t Imm = getImm();
2361     return ((Imm >= 0x000) && (Imm <= 0x0ff)) ||
2362            ((Imm >= 0x101) && (Imm <= 0x10f)) ||
2363            ((Imm >= 0x111) && (Imm <= 0x11f)) ||
2364            ((Imm >= 0x121) && (Imm <= 0x12f)) ||
2365            (Imm == 0x130) ||
2366            (Imm == 0x134) ||
2367            (Imm == 0x138) ||
2368            (Imm == 0x13c) ||
2369            (Imm == 0x140) ||
2370            (Imm == 0x141) ||
2371            (Imm == 0x142) ||
2372            (Imm == 0x143);
2373   }
2374   return false;
2375 }
2376 
2377 AMDGPUAsmParser::OperandMatchResultTy
2378 AMDGPUAsmParser::parseDPPCtrl(OperandVector &Operands) {
2379   SMLoc S = Parser.getTok().getLoc();
2380   StringRef Prefix;
2381   int64_t Int;
2382 
2383   if (getLexer().getKind() == AsmToken::Identifier) {
2384     Prefix = Parser.getTok().getString();
2385   } else {
2386     return MatchOperand_NoMatch;
2387   }
2388 
2389   if (Prefix == "row_mirror") {
2390     Int = 0x140;
2391   } else if (Prefix == "row_half_mirror") {
2392     Int = 0x141;
2393   } else {
2394     // Check to prevent parseDPPCtrlOps from eating invalid tokens
2395     if (Prefix != "quad_perm"
2396         && Prefix != "row_shl"
2397         && Prefix != "row_shr"
2398         && Prefix != "row_ror"
2399         && Prefix != "wave_shl"
2400         && Prefix != "wave_rol"
2401         && Prefix != "wave_shr"
2402         && Prefix != "wave_ror"
2403         && Prefix != "row_bcast") {
2404       return MatchOperand_NoMatch;
2405     }
2406 
2407     Parser.Lex();
2408     if (getLexer().isNot(AsmToken::Colon))
2409       return MatchOperand_ParseFail;
2410 
2411     if (Prefix == "quad_perm") {
2412       // quad_perm:[%d,%d,%d,%d]
2413       Parser.Lex();
2414       if (getLexer().isNot(AsmToken::LBrac))
2415         return MatchOperand_ParseFail;
2416 
2417       Parser.Lex();
2418       if (getLexer().isNot(AsmToken::Integer))
2419         return MatchOperand_ParseFail;
2420       Int = getLexer().getTok().getIntVal();
2421 
2422       Parser.Lex();
2423       if (getLexer().isNot(AsmToken::Comma))
2424         return MatchOperand_ParseFail;
2425       Parser.Lex();
2426       if (getLexer().isNot(AsmToken::Integer))
2427         return MatchOperand_ParseFail;
2428       Int += (getLexer().getTok().getIntVal() << 2);
2429 
2430       Parser.Lex();
2431       if (getLexer().isNot(AsmToken::Comma))
2432         return MatchOperand_ParseFail;
2433       Parser.Lex();
2434       if (getLexer().isNot(AsmToken::Integer))
2435         return MatchOperand_ParseFail;
2436       Int += (getLexer().getTok().getIntVal() << 4);
2437 
2438       Parser.Lex();
2439       if (getLexer().isNot(AsmToken::Comma))
2440         return MatchOperand_ParseFail;
2441       Parser.Lex();
2442       if (getLexer().isNot(AsmToken::Integer))
2443         return MatchOperand_ParseFail;
2444       Int += (getLexer().getTok().getIntVal() << 6);
2445 
2446       Parser.Lex();
2447       if (getLexer().isNot(AsmToken::RBrac))
2448         return MatchOperand_ParseFail;
2449 
2450     } else {
2451       // sel:%d
2452       Parser.Lex();
2453       if (getLexer().isNot(AsmToken::Integer))
2454         return MatchOperand_ParseFail;
2455       Int = getLexer().getTok().getIntVal();
2456 
2457       if (Prefix == "row_shl") {
2458         Int |= 0x100;
2459       } else if (Prefix == "row_shr") {
2460         Int |= 0x110;
2461       } else if (Prefix == "row_ror") {
2462         Int |= 0x120;
2463       } else if (Prefix == "wave_shl") {
2464         Int = 0x130;
2465       } else if (Prefix == "wave_rol") {
2466         Int = 0x134;
2467       } else if (Prefix == "wave_shr") {
2468         Int = 0x138;
2469       } else if (Prefix == "wave_ror") {
2470         Int = 0x13C;
2471       } else if (Prefix == "row_bcast") {
2472         if (Int == 15) {
2473           Int = 0x142;
2474         } else if (Int == 31) {
2475           Int = 0x143;
2476         }
2477       } else {
2478         return MatchOperand_ParseFail;
2479       }
2480     }
2481   }
2482   Parser.Lex(); // eat last token
2483 
2484   Operands.push_back(AMDGPUOperand::CreateImm(Int, S,
2485                                               AMDGPUOperand::ImmTyDppCtrl));
2486   return MatchOperand_Success;
2487 }
2488 
2489 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultRowMask() const {
2490   return AMDGPUOperand::CreateImm(0xf, SMLoc(), AMDGPUOperand::ImmTyDppRowMask);
2491 }
2492 
2493 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultBankMask() const {
2494   return AMDGPUOperand::CreateImm(0xf, SMLoc(), AMDGPUOperand::ImmTyDppBankMask);
2495 }
2496 
2497 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultBoundCtrl() const {
2498   return AMDGPUOperand::CreateImm(0, SMLoc(), AMDGPUOperand::ImmTyDppBoundCtrl);
2499 }
2500 
2501 void AMDGPUAsmParser::cvtDPP(MCInst &Inst, const OperandVector &Operands) {
2502   OptionalImmIndexMap OptionalIdx;
2503 
2504   unsigned I = 1;
2505   const MCInstrDesc &Desc = MII.get(Inst.getOpcode());
2506   for (unsigned J = 0; J < Desc.getNumDefs(); ++J) {
2507     ((AMDGPUOperand &)*Operands[I++]).addRegOperands(Inst, 1);
2508   }
2509 
2510   for (unsigned E = Operands.size(); I != E; ++I) {
2511     AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[I]);
2512     // Add the register arguments
2513     if (Op.isRegOrImmWithInputMods()) {
2514       // We convert only instructions with modifiers
2515       Op.addRegOrImmWithInputModsOperands(Inst, 2);
2516     } else if (Op.isDPPCtrl()) {
2517       Op.addImmOperands(Inst, 1);
2518     } else if (Op.isImm()) {
2519       // Handle optional arguments
2520       OptionalIdx[Op.getImmTy()] = I;
2521     } else {
2522       llvm_unreachable("Invalid operand type");
2523     }
2524   }
2525 
2526   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyDppRowMask, 0xf);
2527   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyDppBankMask, 0xf);
2528   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyDppBoundCtrl);
2529 }
2530 
2531 //===----------------------------------------------------------------------===//
2532 // sdwa
2533 //===----------------------------------------------------------------------===//
2534 
2535 AMDGPUAsmParser::OperandMatchResultTy
2536 AMDGPUAsmParser::parseSDWASel(OperandVector &Operands, StringRef Prefix,
2537                               AMDGPUOperand::ImmTy Type) {
2538   SMLoc S = Parser.getTok().getLoc();
2539   StringRef Value;
2540   AMDGPUAsmParser::OperandMatchResultTy res;
2541 
2542   res = parseStringWithPrefix(Prefix, Value);
2543   if (res != MatchOperand_Success) {
2544     return res;
2545   }
2546 
2547   int64_t Int;
2548   Int = StringSwitch<int64_t>(Value)
2549         .Case("BYTE_0", 0)
2550         .Case("BYTE_1", 1)
2551         .Case("BYTE_2", 2)
2552         .Case("BYTE_3", 3)
2553         .Case("WORD_0", 4)
2554         .Case("WORD_1", 5)
2555         .Case("DWORD", 6)
2556         .Default(0xffffffff);
2557   Parser.Lex(); // eat last token
2558 
2559   if (Int == 0xffffffff) {
2560     return MatchOperand_ParseFail;
2561   }
2562 
2563   Operands.push_back(AMDGPUOperand::CreateImm(Int, S, Type));
2564   return MatchOperand_Success;
2565 }
2566 
2567 AMDGPUAsmParser::OperandMatchResultTy
2568 AMDGPUAsmParser::parseSDWADstUnused(OperandVector &Operands) {
2569   SMLoc S = Parser.getTok().getLoc();
2570   StringRef Value;
2571   AMDGPUAsmParser::OperandMatchResultTy res;
2572 
2573   res = parseStringWithPrefix("dst_unused", Value);
2574   if (res != MatchOperand_Success) {
2575     return res;
2576   }
2577 
2578   int64_t Int;
2579   Int = StringSwitch<int64_t>(Value)
2580         .Case("UNUSED_PAD", 0)
2581         .Case("UNUSED_SEXT", 1)
2582         .Case("UNUSED_PRESERVE", 2)
2583         .Default(0xffffffff);
2584   Parser.Lex(); // eat last token
2585 
2586   if (Int == 0xffffffff) {
2587     return MatchOperand_ParseFail;
2588   }
2589 
2590   Operands.push_back(AMDGPUOperand::CreateImm(Int, S,
2591                                               AMDGPUOperand::ImmTySdwaDstUnused));
2592   return MatchOperand_Success;
2593 }
2594 
2595 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultSDWASel(AMDGPUOperand::ImmTy Type) const {
2596   return AMDGPUOperand::CreateImm(6, SMLoc(), Type);
2597 }
2598 
2599 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultSDWADstSel() const {
2600   return defaultSDWASel(AMDGPUOperand::ImmTySdwaDstSel);
2601 }
2602 
2603 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultSDWASrc0Sel() const {
2604   return defaultSDWASel(AMDGPUOperand::ImmTySdwaSrc0Sel);
2605 }
2606 
2607 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultSDWASrc1Sel() const {
2608   return defaultSDWASel(AMDGPUOperand::ImmTySdwaSrc1Sel);
2609 }
2610 
2611 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultSDWADstUnused() const {
2612   return AMDGPUOperand::CreateImm(0, SMLoc(), AMDGPUOperand::ImmTySdwaDstUnused);
2613 }
2614 
2615 void AMDGPUAsmParser::cvtSdwaVop1_mod(MCInst &Inst, const OperandVector &Operands) {
2616   cvtSDWA(Inst, Operands, true, true);
2617 }
2618 
2619 void AMDGPUAsmParser::cvtSdwaVop1_nomod(MCInst &Inst, const OperandVector &Operands) {
2620   cvtSDWA(Inst, Operands, false, true);
2621 }
2622 
2623 void AMDGPUAsmParser::cvtSdwaVop2_mod(MCInst &Inst, const OperandVector &Operands) {
2624   cvtSDWA(Inst, Operands, true, false);
2625 }
2626 
2627 void AMDGPUAsmParser::cvtSdwaVop2_nomod(MCInst &Inst, const OperandVector &Operands) {
2628   cvtSDWA(Inst, Operands, false, false);
2629 }
2630 
2631 void AMDGPUAsmParser::cvtSDWA(MCInst &Inst, const OperandVector &Operands,
2632                               bool HasMods, bool IsVOP1) {
2633   OptionalImmIndexMap OptionalIdx;
2634 
2635   unsigned I = 1;
2636   const MCInstrDesc &Desc = MII.get(Inst.getOpcode());
2637   for (unsigned J = 0; J < Desc.getNumDefs(); ++J) {
2638     ((AMDGPUOperand &)*Operands[I++]).addRegOperands(Inst, 1);
2639   }
2640 
2641   for (unsigned E = Operands.size(); I != E; ++I) {
2642     AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[I]);
2643     // Add the register arguments
2644     if (!HasMods && Op.isReg()) {
2645       Op.addRegOperands(Inst, 1);
2646     } else if (HasMods && Op.isRegOrImmWithInputMods()) {
2647       Op.addRegOrImmWithInputModsOperands(Inst, 2);
2648     } else if (Op.isImm()) {
2649       // Handle optional arguments
2650       OptionalIdx[Op.getImmTy()] = I;
2651     } else {
2652       llvm_unreachable("Invalid operand type");
2653     }
2654   }
2655 
2656   if (HasMods) {
2657     addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyClampSI, 0);
2658   }
2659   if (Inst.getOpcode() == AMDGPU::V_NOP_sdwa) {
2660     // V_NOP_sdwa has no optional sdwa arguments
2661     return;
2662   }
2663   if (IsVOP1) {
2664     addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySdwaDstSel, 6);
2665     addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySdwaDstUnused, 2);
2666     addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySdwaSrc0Sel, 6);
2667   } else { // VOP2
2668     addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySdwaDstSel, 6);
2669     addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySdwaDstUnused, 2);
2670     addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySdwaSrc0Sel, 6);
2671     addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySdwaSrc1Sel, 6);
2672   }
2673 }
2674 
2675 /// Force static initialization.
2676 extern "C" void LLVMInitializeAMDGPUAsmParser() {
2677   RegisterMCAsmParser<AMDGPUAsmParser> A(TheAMDGPUTarget);
2678   RegisterMCAsmParser<AMDGPUAsmParser> B(TheGCNTarget);
2679 }
2680 
2681 #define GET_REGISTER_MATCHER
2682 #define GET_MATCHER_IMPLEMENTATION
2683 #include "AMDGPUGenAsmMatcher.inc"
2684 
2685 
2686 // This fuction should be defined after auto-generated include so that we have
2687 // MatchClassKind enum defined
2688 unsigned AMDGPUAsmParser::validateTargetOperandClass(MCParsedAsmOperand &Op,
2689                                                      unsigned Kind) {
2690   // Tokens like "glc" would be parsed as immediate operands in ParseOperand().
2691   // But MatchInstructionImpl() expects to meet token and fails to validate
2692   // operand. This method checks if we are given immediate operand but expect to
2693   // get corresponding token.
2694   AMDGPUOperand &Operand = (AMDGPUOperand&)Op;
2695   switch (Kind) {
2696   case MCK_addr64:
2697     return Operand.isAddr64() ? Match_Success : Match_InvalidOperand;
2698   case MCK_gds:
2699     return Operand.isGDS() ? Match_Success : Match_InvalidOperand;
2700   case MCK_glc:
2701     return Operand.isGLC() ? Match_Success : Match_InvalidOperand;
2702   case MCK_idxen:
2703     return Operand.isIdxen() ? Match_Success : Match_InvalidOperand;
2704   case MCK_offen:
2705     return Operand.isOffen() ? Match_Success : Match_InvalidOperand;
2706   default: return Match_InvalidOperand;
2707   }
2708 }
2709