1 //===-- AMDGPUAsmParser.cpp - Parse SI asm to MCInst instructions ---------===//
2 //
3 //                     The LLVM Compiler Infrastructure
4 //
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
7 //
8 //===----------------------------------------------------------------------===//
9 
10 #include "AMDKernelCodeT.h"
11 #include "MCTargetDesc/AMDGPUMCTargetDesc.h"
12 #include "MCTargetDesc/AMDGPUTargetStreamer.h"
13 #include "SIDefines.h"
14 #include "Utils/AMDGPUBaseInfo.h"
15 #include "Utils/AMDKernelCodeTUtils.h"
16 #include "llvm/ADT/APFloat.h"
17 #include "llvm/ADT/STLExtras.h"
18 #include "llvm/ADT/SmallString.h"
19 #include "llvm/ADT/SmallVector.h"
20 #include "llvm/ADT/StringSwitch.h"
21 #include "llvm/ADT/Twine.h"
22 #include "llvm/MC/MCContext.h"
23 #include "llvm/MC/MCExpr.h"
24 #include "llvm/MC/MCInst.h"
25 #include "llvm/MC/MCInstrInfo.h"
26 #include "llvm/MC/MCParser/MCAsmLexer.h"
27 #include "llvm/MC/MCParser/MCAsmParser.h"
28 #include "llvm/MC/MCParser/MCParsedAsmOperand.h"
29 #include "llvm/MC/MCParser/MCTargetAsmParser.h"
30 #include "llvm/MC/MCRegisterInfo.h"
31 #include "llvm/MC/MCStreamer.h"
32 #include "llvm/MC/MCSubtargetInfo.h"
33 #include "llvm/MC/MCSymbolELF.h"
34 #include "llvm/Support/Debug.h"
35 #include "llvm/Support/ELF.h"
36 #include "llvm/Support/SourceMgr.h"
37 #include "llvm/Support/TargetRegistry.h"
38 #include "llvm/Support/raw_ostream.h"
39 
40 using namespace llvm;
41 
42 namespace {
43 
44 struct OptionalOperand;
45 
46 class AMDGPUOperand : public MCParsedAsmOperand {
47   enum KindTy {
48     Token,
49     Immediate,
50     Register,
51     Expression
52   } Kind;
53 
54   SMLoc StartLoc, EndLoc;
55 
56 public:
57   AMDGPUOperand(enum KindTy K) : MCParsedAsmOperand(), Kind(K) {}
58 
59   MCContext *Ctx;
60 
61   enum ImmTy {
62     ImmTyNone,
63     ImmTyDSOffset0,
64     ImmTyDSOffset1,
65     ImmTyGDS,
66     ImmTyOffset,
67     ImmTyGLC,
68     ImmTySLC,
69     ImmTyTFE,
70     ImmTyClamp,
71     ImmTyOMod,
72     ImmTyDppCtrl,
73     ImmTyDppRowMask,
74     ImmTyDppBankMask,
75     ImmTyDppBoundCtrl,
76     ImmTyDMask,
77     ImmTyUNorm,
78     ImmTyDA,
79     ImmTyR128,
80     ImmTyLWE,
81   };
82 
83   struct TokOp {
84     const char *Data;
85     unsigned Length;
86   };
87 
88   struct ImmOp {
89     bool IsFPImm;
90     ImmTy Type;
91     int64_t Val;
92     int Modifiers;
93   };
94 
95   struct RegOp {
96     unsigned RegNo;
97     int Modifiers;
98     const MCRegisterInfo *TRI;
99     const MCSubtargetInfo *STI;
100     bool IsForcedVOP3;
101   };
102 
103   union {
104     TokOp Tok;
105     ImmOp Imm;
106     RegOp Reg;
107     const MCExpr *Expr;
108   };
109 
110   void addImmOperands(MCInst &Inst, unsigned N) const {
111     Inst.addOperand(MCOperand::createImm(getImm()));
112   }
113 
114   StringRef getToken() const {
115     return StringRef(Tok.Data, Tok.Length);
116   }
117 
118   void addRegOperands(MCInst &Inst, unsigned N) const {
119     Inst.addOperand(MCOperand::createReg(AMDGPU::getMCReg(getReg(), *Reg.STI)));
120   }
121 
122   void addRegOrImmOperands(MCInst &Inst, unsigned N) const {
123     if (isRegKind())
124       addRegOperands(Inst, N);
125     else
126       addImmOperands(Inst, N);
127   }
128 
129   void addRegOrImmWithInputModsOperands(MCInst &Inst, unsigned N) const {
130     if (isRegKind()) {
131       Inst.addOperand(MCOperand::createImm(Reg.Modifiers));
132       addRegOperands(Inst, N);
133     } else {
134       Inst.addOperand(MCOperand::createImm(Imm.Modifiers));
135       addImmOperands(Inst, N);
136     }
137   }
138 
139   void addSoppBrTargetOperands(MCInst &Inst, unsigned N) const {
140     if (isImm())
141       addImmOperands(Inst, N);
142     else {
143       assert(isExpr());
144       Inst.addOperand(MCOperand::createExpr(Expr));
145     }
146   }
147 
148   bool defaultTokenHasSuffix() const {
149     StringRef Token(Tok.Data, Tok.Length);
150 
151     return Token.endswith("_e32") || Token.endswith("_e64") ||
152       Token.endswith("_dpp");
153   }
154 
155   bool isToken() const override {
156     return Kind == Token;
157   }
158 
159   bool isImm() const override {
160     return Kind == Immediate;
161   }
162 
163   bool isInlinableImm() const {
164     if (!isImm() || Imm.Type != AMDGPUOperand::ImmTyNone /* Only plain
165       immediates are inlinable (e.g. "clamp" attribute is not) */ )
166       return false;
167     // TODO: We should avoid using host float here. It would be better to
168     // check the float bit values which is what a few other places do.
169     // We've had bot failures before due to weird NaN support on mips hosts.
170     const float F = BitsToFloat(Imm.Val);
171     // TODO: Add 1/(2*pi) for VI
172     return (Imm.Val <= 64 && Imm.Val >= -16) ||
173            (F == 0.0 || F == 0.5 || F == -0.5 || F == 1.0 || F == -1.0 ||
174            F == 2.0 || F == -2.0 || F == 4.0 || F == -4.0);
175   }
176 
177   bool isDSOffset0() const {
178     assert(isImm());
179     return Imm.Type == ImmTyDSOffset0;
180   }
181 
182   bool isDSOffset1() const {
183     assert(isImm());
184     return Imm.Type == ImmTyDSOffset1;
185   }
186 
187   int64_t getImm() const {
188     return Imm.Val;
189   }
190 
191   enum ImmTy getImmTy() const {
192     assert(isImm());
193     return Imm.Type;
194   }
195 
196   bool isRegKind() const {
197     return Kind == Register;
198   }
199 
200   bool isReg() const override {
201     return Kind == Register && Reg.Modifiers == 0;
202   }
203 
204   bool isRegOrImmWithInputMods() const {
205     return Kind == Register || isInlinableImm();
206   }
207 
208   bool isImmTy(ImmTy ImmT) const {
209     return isImm() && Imm.Type == ImmT;
210   }
211 
212   bool isClamp() const {
213     return isImmTy(ImmTyClamp);
214   }
215 
216   bool isOMod() const {
217     return isImmTy(ImmTyOMod);
218   }
219 
220   bool isImmModifier() const {
221     return Kind == Immediate && Imm.Type != ImmTyNone;
222   }
223 
224   bool isDMask() const {
225     return isImmTy(ImmTyDMask);
226   }
227 
228   bool isUNorm() const { return isImmTy(ImmTyUNorm); }
229   bool isDA() const { return isImmTy(ImmTyDA); }
230   bool isR128() const { return isImmTy(ImmTyUNorm); }
231   bool isLWE() const { return isImmTy(ImmTyLWE); }
232 
233   bool isMod() const {
234     return isClamp() || isOMod();
235   }
236 
237   bool isGDS() const { return isImmTy(ImmTyGDS); }
238   bool isGLC() const { return isImmTy(ImmTyGLC); }
239   bool isSLC() const { return isImmTy(ImmTySLC); }
240   bool isTFE() const { return isImmTy(ImmTyTFE); }
241 
242   bool isBankMask() const {
243     return isImmTy(ImmTyDppBankMask);
244   }
245 
246   bool isRowMask() const {
247     return isImmTy(ImmTyDppRowMask);
248   }
249 
250   bool isBoundCtrl() const {
251     return isImmTy(ImmTyDppBoundCtrl);
252   }
253 
254   void setModifiers(unsigned Mods) {
255     assert(isReg() || (isImm() && Imm.Modifiers == 0));
256     if (isReg())
257       Reg.Modifiers = Mods;
258     else
259       Imm.Modifiers = Mods;
260   }
261 
262   bool hasModifiers() const {
263     assert(isRegKind() || isImm());
264     return isRegKind() ? Reg.Modifiers != 0 : Imm.Modifiers != 0;
265   }
266 
267   unsigned getReg() const override {
268     return Reg.RegNo;
269   }
270 
271   bool isRegOrImm() const {
272     return isReg() || isImm();
273   }
274 
275   bool isRegClass(unsigned RCID) const {
276     return isReg() && Reg.TRI->getRegClass(RCID).contains(getReg());
277   }
278 
279   bool isSCSrc32() const {
280     return isInlinableImm() || isRegClass(AMDGPU::SReg_32RegClassID);
281   }
282 
283   bool isSCSrc64() const {
284     return isInlinableImm() || isRegClass(AMDGPU::SReg_64RegClassID);
285   }
286 
287   bool isSSrc32() const {
288     return isImm() || isSCSrc32();
289   }
290 
291   bool isSSrc64() const {
292     // TODO: Find out how SALU supports extension of 32-bit literals to 64 bits.
293     // See isVSrc64().
294     return isImm() || isSCSrc64();
295   }
296 
297   bool isVCSrc32() const {
298     return isInlinableImm() || isRegClass(AMDGPU::VS_32RegClassID);
299   }
300 
301   bool isVCSrc64() const {
302     return isInlinableImm() || isRegClass(AMDGPU::VS_64RegClassID);
303   }
304 
305   bool isVSrc32() const {
306     return isImm() || isVCSrc32();
307   }
308 
309   bool isVSrc64() const {
310     // TODO: Check if the 64-bit value (coming from assembly source) can be
311     // narrowed to 32 bits (in the instruction stream). That require knowledge
312     // of instruction type (unsigned/signed, floating or "untyped"/B64),
313     // see [AMD GCN3 ISA 6.3.1].
314     // TODO: How 64-bit values are formed from 32-bit literals in _B64 insns?
315     return isImm() || isVCSrc64();
316   }
317 
318   bool isMem() const override {
319     return false;
320   }
321 
322   bool isExpr() const {
323     return Kind == Expression;
324   }
325 
326   bool isSoppBrTarget() const {
327     return isExpr() || isImm();
328   }
329 
330   SMLoc getStartLoc() const override {
331     return StartLoc;
332   }
333 
334   SMLoc getEndLoc() const override {
335     return EndLoc;
336   }
337 
338   void print(raw_ostream &OS) const override {
339     switch (Kind) {
340     case Register:
341       OS << "<register " << getReg() << " mods: " << Reg.Modifiers << '>';
342       break;
343     case Immediate:
344       if (Imm.Type != AMDGPUOperand::ImmTyNone)
345         OS << getImm();
346       else
347         OS << '<' << getImm() << " mods: " << Imm.Modifiers << '>';
348       break;
349     case Token:
350       OS << '\'' << getToken() << '\'';
351       break;
352     case Expression:
353       OS << "<expr " << *Expr << '>';
354       break;
355     }
356   }
357 
358   static std::unique_ptr<AMDGPUOperand> CreateImm(int64_t Val, SMLoc Loc,
359                                                   enum ImmTy Type = ImmTyNone,
360                                                   bool IsFPImm = false) {
361     auto Op = llvm::make_unique<AMDGPUOperand>(Immediate);
362     Op->Imm.Val = Val;
363     Op->Imm.IsFPImm = IsFPImm;
364     Op->Imm.Type = Type;
365     Op->Imm.Modifiers = 0;
366     Op->StartLoc = Loc;
367     Op->EndLoc = Loc;
368     return Op;
369   }
370 
371   static std::unique_ptr<AMDGPUOperand> CreateToken(StringRef Str, SMLoc Loc,
372                                            bool HasExplicitEncodingSize = true) {
373     auto Res = llvm::make_unique<AMDGPUOperand>(Token);
374     Res->Tok.Data = Str.data();
375     Res->Tok.Length = Str.size();
376     Res->StartLoc = Loc;
377     Res->EndLoc = Loc;
378     return Res;
379   }
380 
381   static std::unique_ptr<AMDGPUOperand> CreateReg(unsigned RegNo, SMLoc S,
382                                                   SMLoc E,
383                                                   const MCRegisterInfo *TRI,
384                                                   const MCSubtargetInfo *STI,
385                                                   bool ForceVOP3) {
386     auto Op = llvm::make_unique<AMDGPUOperand>(Register);
387     Op->Reg.RegNo = RegNo;
388     Op->Reg.TRI = TRI;
389     Op->Reg.STI = STI;
390     Op->Reg.Modifiers = 0;
391     Op->Reg.IsForcedVOP3 = ForceVOP3;
392     Op->StartLoc = S;
393     Op->EndLoc = E;
394     return Op;
395   }
396 
397   static std::unique_ptr<AMDGPUOperand> CreateExpr(const class MCExpr *Expr, SMLoc S) {
398     auto Op = llvm::make_unique<AMDGPUOperand>(Expression);
399     Op->Expr = Expr;
400     Op->StartLoc = S;
401     Op->EndLoc = S;
402     return Op;
403   }
404 
405   bool isDSOffset() const;
406   bool isDSOffset01() const;
407   bool isSWaitCnt() const;
408   bool isMubufOffset() const;
409   bool isSMRDOffset() const;
410   bool isSMRDLiteralOffset() const;
411   bool isDPPCtrl() const;
412 };
413 
414 class AMDGPUAsmParser : public MCTargetAsmParser {
415   const MCInstrInfo &MII;
416   MCAsmParser &Parser;
417 
418   unsigned ForcedEncodingSize;
419 
420   bool isSI() const {
421     return AMDGPU::isSI(getSTI());
422   }
423 
424   bool isCI() const {
425     return AMDGPU::isCI(getSTI());
426   }
427 
428   bool isVI() const {
429     return AMDGPU::isVI(getSTI());
430   }
431 
432   bool hasSGPR102_SGPR103() const {
433     return !isVI();
434   }
435 
436   /// @name Auto-generated Match Functions
437   /// {
438 
439 #define GET_ASSEMBLER_HEADER
440 #include "AMDGPUGenAsmMatcher.inc"
441 
442   /// }
443 
444 private:
445   bool ParseDirectiveMajorMinor(uint32_t &Major, uint32_t &Minor);
446   bool ParseDirectiveHSACodeObjectVersion();
447   bool ParseDirectiveHSACodeObjectISA();
448   bool ParseAMDKernelCodeTValue(StringRef ID, amd_kernel_code_t &Header);
449   bool ParseDirectiveAMDKernelCodeT();
450   bool ParseSectionDirectiveHSAText();
451   bool subtargetHasRegister(const MCRegisterInfo &MRI, unsigned RegNo) const;
452   bool ParseDirectiveAMDGPUHsaKernel();
453   bool ParseDirectiveAMDGPUHsaModuleGlobal();
454   bool ParseDirectiveAMDGPUHsaProgramGlobal();
455   bool ParseSectionDirectiveHSADataGlobalAgent();
456   bool ParseSectionDirectiveHSADataGlobalProgram();
457   bool ParseSectionDirectiveHSARodataReadonlyAgent();
458 
459 public:
460   enum AMDGPUMatchResultTy {
461     Match_PreferE32 = FIRST_TARGET_MATCH_RESULT_TY
462   };
463 
464   AMDGPUAsmParser(const MCSubtargetInfo &STI, MCAsmParser &_Parser,
465                const MCInstrInfo &MII,
466                const MCTargetOptions &Options)
467       : MCTargetAsmParser(Options, STI), MII(MII), Parser(_Parser),
468         ForcedEncodingSize(0) {
469     MCAsmParserExtension::Initialize(Parser);
470 
471     if (getSTI().getFeatureBits().none()) {
472       // Set default features.
473       copySTI().ToggleFeature("SOUTHERN_ISLANDS");
474     }
475 
476     setAvailableFeatures(ComputeAvailableFeatures(getSTI().getFeatureBits()));
477   }
478 
479   AMDGPUTargetStreamer &getTargetStreamer() {
480     MCTargetStreamer &TS = *getParser().getStreamer().getTargetStreamer();
481     return static_cast<AMDGPUTargetStreamer &>(TS);
482   }
483 
484   unsigned getForcedEncodingSize() const {
485     return ForcedEncodingSize;
486   }
487 
488   void setForcedEncodingSize(unsigned Size) {
489     ForcedEncodingSize = Size;
490   }
491 
492   bool isForcedVOP3() const {
493     return ForcedEncodingSize == 64;
494   }
495 
496   std::unique_ptr<AMDGPUOperand> parseRegister();
497   bool ParseRegister(unsigned &RegNo, SMLoc &StartLoc, SMLoc &EndLoc) override;
498   unsigned checkTargetMatchPredicate(MCInst &Inst) override;
499   bool MatchAndEmitInstruction(SMLoc IDLoc, unsigned &Opcode,
500                                OperandVector &Operands, MCStreamer &Out,
501                                uint64_t &ErrorInfo,
502                                bool MatchingInlineAsm) override;
503   bool ParseDirective(AsmToken DirectiveID) override;
504   OperandMatchResultTy parseOperand(OperandVector &Operands, StringRef Mnemonic);
505   bool ParseInstruction(ParseInstructionInfo &Info, StringRef Name,
506                         SMLoc NameLoc, OperandVector &Operands) override;
507 
508   OperandMatchResultTy parseIntWithPrefix(const char *Prefix, int64_t &Int,
509                                           int64_t Default = 0);
510   OperandMatchResultTy parseIntWithPrefix(const char *Prefix,
511                                           OperandVector &Operands,
512                                           enum AMDGPUOperand::ImmTy ImmTy =
513                                                       AMDGPUOperand::ImmTyNone);
514   OperandMatchResultTy parseNamedBit(const char *Name, OperandVector &Operands,
515                                      enum AMDGPUOperand::ImmTy ImmTy =
516                                                       AMDGPUOperand::ImmTyNone);
517   OperandMatchResultTy parseOptionalOps(
518                                    const ArrayRef<OptionalOperand> &OptionalOps,
519                                    OperandVector &Operands);
520 
521 
522   void cvtDSOffset01(MCInst &Inst, const OperandVector &Operands);
523   void cvtDS(MCInst &Inst, const OperandVector &Operands);
524   OperandMatchResultTy parseDSOptionalOps(OperandVector &Operands);
525   OperandMatchResultTy parseDSOff01OptionalOps(OperandVector &Operands);
526   OperandMatchResultTy parseDSOffsetOptional(OperandVector &Operands);
527 
528   bool parseCnt(int64_t &IntVal);
529   OperandMatchResultTy parseSWaitCntOps(OperandVector &Operands);
530   OperandMatchResultTy parseSOppBrTarget(OperandVector &Operands);
531 
532   OperandMatchResultTy parseFlatOptionalOps(OperandVector &Operands);
533   OperandMatchResultTy parseFlatAtomicOptionalOps(OperandVector &Operands);
534   void cvtFlat(MCInst &Inst, const OperandVector &Operands);
535   void cvtFlatAtomic(MCInst &Inst, const OperandVector &Operands);
536 
537   void cvtMubuf(MCInst &Inst, const OperandVector &Operands);
538   OperandMatchResultTy parseOffset(OperandVector &Operands);
539   OperandMatchResultTy parseMubufOptionalOps(OperandVector &Operands);
540   OperandMatchResultTy parseGLC(OperandVector &Operands);
541   OperandMatchResultTy parseSLC(OperandVector &Operands);
542   OperandMatchResultTy parseTFE(OperandVector &Operands);
543 
544   OperandMatchResultTy parseDMask(OperandVector &Operands);
545   OperandMatchResultTy parseUNorm(OperandVector &Operands);
546   OperandMatchResultTy parseDA(OperandVector &Operands);
547   OperandMatchResultTy parseR128(OperandVector &Operands);
548   OperandMatchResultTy parseLWE(OperandVector &Operands);
549 
550   void cvtId(MCInst &Inst, const OperandVector &Operands);
551   void cvtVOP3_2_mod(MCInst &Inst, const OperandVector &Operands);
552   void cvtVOP3_2_nomod(MCInst &Inst, const OperandVector &Operands);
553   void cvtVOP3_only(MCInst &Inst, const OperandVector &Operands);
554   void cvtVOP3(MCInst &Inst, const OperandVector &Operands);
555 
556   void cvtMIMG(MCInst &Inst, const OperandVector &Operands);
557   void cvtMIMGAtomic(MCInst &Inst, const OperandVector &Operands);
558   OperandMatchResultTy parseVOP3OptionalOps(OperandVector &Operands);
559 
560   OperandMatchResultTy parseDPPCtrlOps(OperandVector &Operands);
561   OperandMatchResultTy parseDPPOptionalOps(OperandVector &Operands);
562   void cvtDPP_mod(MCInst &Inst, const OperandVector &Operands);
563   void cvtDPP_nomod(MCInst &Inst, const OperandVector &Operands);
564   void cvtDPP(MCInst &Inst, const OperandVector &Operands, bool HasMods);
565 };
566 
567 struct OptionalOperand {
568   const char *Name;
569   AMDGPUOperand::ImmTy Type;
570   bool IsBit;
571   int64_t Default;
572   bool (*ConvertResult)(int64_t&);
573 };
574 
575 }
576 
577 static int getRegClass(bool IsVgpr, unsigned RegWidth) {
578   if (IsVgpr) {
579     switch (RegWidth) {
580       default: return -1;
581       case 1: return AMDGPU::VGPR_32RegClassID;
582       case 2: return AMDGPU::VReg_64RegClassID;
583       case 3: return AMDGPU::VReg_96RegClassID;
584       case 4: return AMDGPU::VReg_128RegClassID;
585       case 8: return AMDGPU::VReg_256RegClassID;
586       case 16: return AMDGPU::VReg_512RegClassID;
587     }
588   }
589 
590   switch (RegWidth) {
591     default: return -1;
592     case 1: return AMDGPU::SGPR_32RegClassID;
593     case 2: return AMDGPU::SGPR_64RegClassID;
594     case 4: return AMDGPU::SReg_128RegClassID;
595     case 8: return AMDGPU::SReg_256RegClassID;
596     case 16: return AMDGPU::SReg_512RegClassID;
597   }
598 }
599 
600 static unsigned getRegForName(StringRef RegName) {
601 
602   return StringSwitch<unsigned>(RegName)
603     .Case("exec", AMDGPU::EXEC)
604     .Case("vcc", AMDGPU::VCC)
605     .Case("flat_scratch", AMDGPU::FLAT_SCR)
606     .Case("m0", AMDGPU::M0)
607     .Case("scc", AMDGPU::SCC)
608     .Case("flat_scratch_lo", AMDGPU::FLAT_SCR_LO)
609     .Case("flat_scratch_hi", AMDGPU::FLAT_SCR_HI)
610     .Case("vcc_lo", AMDGPU::VCC_LO)
611     .Case("vcc_hi", AMDGPU::VCC_HI)
612     .Case("exec_lo", AMDGPU::EXEC_LO)
613     .Case("exec_hi", AMDGPU::EXEC_HI)
614     .Default(0);
615 }
616 
617 bool AMDGPUAsmParser::ParseRegister(unsigned &RegNo, SMLoc &StartLoc, SMLoc &EndLoc) {
618   auto R = parseRegister();
619   if (!R) return true;
620   assert(R->isReg());
621   RegNo = R->getReg();
622   StartLoc = R->getStartLoc();
623   EndLoc = R->getEndLoc();
624   return false;
625 }
626 
627 std::unique_ptr<AMDGPUOperand> AMDGPUAsmParser::parseRegister() {
628   const AsmToken &Tok = Parser.getTok();
629   SMLoc StartLoc = Tok.getLoc();
630   SMLoc EndLoc = Tok.getEndLoc();
631   const MCRegisterInfo *TRI = getContext().getRegisterInfo();
632 
633   StringRef RegName = Tok.getString();
634   unsigned RegNo = getRegForName(RegName);
635 
636   if (RegNo) {
637     Parser.Lex();
638     if (!subtargetHasRegister(*TRI, RegNo))
639       return nullptr;
640     return AMDGPUOperand::CreateReg(RegNo, StartLoc, EndLoc,
641                                     TRI, &getSTI(), false);
642   }
643 
644   // Match vgprs and sgprs
645   if (RegName[0] != 's' && RegName[0] != 'v')
646     return nullptr;
647 
648   bool IsVgpr = RegName[0] == 'v';
649   unsigned RegWidth;
650   unsigned RegIndexInClass;
651   if (RegName.size() > 1) {
652     // We have a 32-bit register
653     RegWidth = 1;
654     if (RegName.substr(1).getAsInteger(10, RegIndexInClass))
655       return nullptr;
656     Parser.Lex();
657   } else {
658     // We have a register greater than 32-bits.
659 
660     int64_t RegLo, RegHi;
661     Parser.Lex();
662     if (getLexer().isNot(AsmToken::LBrac))
663       return nullptr;
664 
665     Parser.Lex();
666     if (getParser().parseAbsoluteExpression(RegLo))
667       return nullptr;
668 
669     if (getLexer().isNot(AsmToken::Colon))
670       return nullptr;
671 
672     Parser.Lex();
673     if (getParser().parseAbsoluteExpression(RegHi))
674       return nullptr;
675 
676     if (getLexer().isNot(AsmToken::RBrac))
677       return nullptr;
678 
679     Parser.Lex();
680     RegWidth = (RegHi - RegLo) + 1;
681     if (IsVgpr) {
682       // VGPR registers aren't aligned.
683       RegIndexInClass = RegLo;
684     } else {
685       // SGPR registers are aligned.  Max alignment is 4 dwords.
686       unsigned Size = std::min(RegWidth, 4u);
687       if (RegLo % Size != 0)
688         return nullptr;
689 
690       RegIndexInClass = RegLo / Size;
691     }
692   }
693 
694   int RCID = getRegClass(IsVgpr, RegWidth);
695   if (RCID == -1)
696     return nullptr;
697 
698   const MCRegisterClass RC = TRI->getRegClass(RCID);
699   if (RegIndexInClass >= RC.getNumRegs())
700     return nullptr;
701 
702   RegNo = RC.getRegister(RegIndexInClass);
703   if (!subtargetHasRegister(*TRI, RegNo))
704     return nullptr;
705 
706   return AMDGPUOperand::CreateReg(RegNo, StartLoc, EndLoc,
707                                   TRI, &getSTI(), false);
708 }
709 
710 unsigned AMDGPUAsmParser::checkTargetMatchPredicate(MCInst &Inst) {
711 
712   uint64_t TSFlags = MII.get(Inst.getOpcode()).TSFlags;
713 
714   if ((getForcedEncodingSize() == 32 && (TSFlags & SIInstrFlags::VOP3)) ||
715       (getForcedEncodingSize() == 64 && !(TSFlags & SIInstrFlags::VOP3)))
716     return Match_InvalidOperand;
717 
718   if ((TSFlags & SIInstrFlags::VOP3) &&
719       (TSFlags & SIInstrFlags::VOPAsmPrefer32Bit) &&
720       getForcedEncodingSize() != 64)
721     return Match_PreferE32;
722 
723   return Match_Success;
724 }
725 
726 
727 bool AMDGPUAsmParser::MatchAndEmitInstruction(SMLoc IDLoc, unsigned &Opcode,
728                                               OperandVector &Operands,
729                                               MCStreamer &Out,
730                                               uint64_t &ErrorInfo,
731                                               bool MatchingInlineAsm) {
732   MCInst Inst;
733 
734   switch (MatchInstructionImpl(Operands, Inst, ErrorInfo, MatchingInlineAsm)) {
735     default: break;
736     case Match_Success:
737       Inst.setLoc(IDLoc);
738       Out.EmitInstruction(Inst, getSTI());
739       return false;
740     case Match_MissingFeature:
741       return Error(IDLoc, "instruction not supported on this GPU");
742 
743     case Match_MnemonicFail:
744       return Error(IDLoc, "unrecognized instruction mnemonic");
745 
746     case Match_InvalidOperand: {
747       SMLoc ErrorLoc = IDLoc;
748       if (ErrorInfo != ~0ULL) {
749         if (ErrorInfo >= Operands.size()) {
750           return Error(IDLoc, "too few operands for instruction");
751         }
752         ErrorLoc = ((AMDGPUOperand &)*Operands[ErrorInfo]).getStartLoc();
753         if (ErrorLoc == SMLoc())
754           ErrorLoc = IDLoc;
755       }
756       return Error(ErrorLoc, "invalid operand for instruction");
757     }
758     case Match_PreferE32:
759       return Error(IDLoc, "internal error: instruction without _e64 suffix "
760                           "should be encoded as e32");
761   }
762   llvm_unreachable("Implement any new match types added!");
763 }
764 
765 bool AMDGPUAsmParser::ParseDirectiveMajorMinor(uint32_t &Major,
766                                                uint32_t &Minor) {
767   if (getLexer().isNot(AsmToken::Integer))
768     return TokError("invalid major version");
769 
770   Major = getLexer().getTok().getIntVal();
771   Lex();
772 
773   if (getLexer().isNot(AsmToken::Comma))
774     return TokError("minor version number required, comma expected");
775   Lex();
776 
777   if (getLexer().isNot(AsmToken::Integer))
778     return TokError("invalid minor version");
779 
780   Minor = getLexer().getTok().getIntVal();
781   Lex();
782 
783   return false;
784 }
785 
786 bool AMDGPUAsmParser::ParseDirectiveHSACodeObjectVersion() {
787 
788   uint32_t Major;
789   uint32_t Minor;
790 
791   if (ParseDirectiveMajorMinor(Major, Minor))
792     return true;
793 
794   getTargetStreamer().EmitDirectiveHSACodeObjectVersion(Major, Minor);
795   return false;
796 }
797 
798 bool AMDGPUAsmParser::ParseDirectiveHSACodeObjectISA() {
799 
800   uint32_t Major;
801   uint32_t Minor;
802   uint32_t Stepping;
803   StringRef VendorName;
804   StringRef ArchName;
805 
806   // If this directive has no arguments, then use the ISA version for the
807   // targeted GPU.
808   if (getLexer().is(AsmToken::EndOfStatement)) {
809     AMDGPU::IsaVersion Isa = AMDGPU::getIsaVersion(getSTI().getFeatureBits());
810     getTargetStreamer().EmitDirectiveHSACodeObjectISA(Isa.Major, Isa.Minor,
811                                                       Isa.Stepping,
812                                                       "AMD", "AMDGPU");
813     return false;
814   }
815 
816 
817   if (ParseDirectiveMajorMinor(Major, Minor))
818     return true;
819 
820   if (getLexer().isNot(AsmToken::Comma))
821     return TokError("stepping version number required, comma expected");
822   Lex();
823 
824   if (getLexer().isNot(AsmToken::Integer))
825     return TokError("invalid stepping version");
826 
827   Stepping = getLexer().getTok().getIntVal();
828   Lex();
829 
830   if (getLexer().isNot(AsmToken::Comma))
831     return TokError("vendor name required, comma expected");
832   Lex();
833 
834   if (getLexer().isNot(AsmToken::String))
835     return TokError("invalid vendor name");
836 
837   VendorName = getLexer().getTok().getStringContents();
838   Lex();
839 
840   if (getLexer().isNot(AsmToken::Comma))
841     return TokError("arch name required, comma expected");
842   Lex();
843 
844   if (getLexer().isNot(AsmToken::String))
845     return TokError("invalid arch name");
846 
847   ArchName = getLexer().getTok().getStringContents();
848   Lex();
849 
850   getTargetStreamer().EmitDirectiveHSACodeObjectISA(Major, Minor, Stepping,
851                                                     VendorName, ArchName);
852   return false;
853 }
854 
855 bool AMDGPUAsmParser::ParseAMDKernelCodeTValue(StringRef ID,
856                                                amd_kernel_code_t &Header) {
857   SmallString<40> ErrStr;
858   raw_svector_ostream Err(ErrStr);
859   if (!parseAmdKernelCodeField(ID, getLexer(), Header, Err)) {
860     return TokError(Err.str());
861   }
862   Lex();
863   return false;
864 }
865 
866 bool AMDGPUAsmParser::ParseDirectiveAMDKernelCodeT() {
867 
868   amd_kernel_code_t Header;
869   AMDGPU::initDefaultAMDKernelCodeT(Header, getSTI().getFeatureBits());
870 
871   while (true) {
872 
873     if (getLexer().isNot(AsmToken::EndOfStatement))
874       return TokError("amd_kernel_code_t values must begin on a new line");
875 
876     // Lex EndOfStatement.  This is in a while loop, because lexing a comment
877     // will set the current token to EndOfStatement.
878     while(getLexer().is(AsmToken::EndOfStatement))
879       Lex();
880 
881     if (getLexer().isNot(AsmToken::Identifier))
882       return TokError("expected value identifier or .end_amd_kernel_code_t");
883 
884     StringRef ID = getLexer().getTok().getIdentifier();
885     Lex();
886 
887     if (ID == ".end_amd_kernel_code_t")
888       break;
889 
890     if (ParseAMDKernelCodeTValue(ID, Header))
891       return true;
892   }
893 
894   getTargetStreamer().EmitAMDKernelCodeT(Header);
895 
896   return false;
897 }
898 
899 bool AMDGPUAsmParser::ParseSectionDirectiveHSAText() {
900   getParser().getStreamer().SwitchSection(
901       AMDGPU::getHSATextSection(getContext()));
902   return false;
903 }
904 
905 bool AMDGPUAsmParser::ParseDirectiveAMDGPUHsaKernel() {
906   if (getLexer().isNot(AsmToken::Identifier))
907     return TokError("expected symbol name");
908 
909   StringRef KernelName = Parser.getTok().getString();
910 
911   getTargetStreamer().EmitAMDGPUSymbolType(KernelName,
912                                            ELF::STT_AMDGPU_HSA_KERNEL);
913   Lex();
914   return false;
915 }
916 
917 bool AMDGPUAsmParser::ParseDirectiveAMDGPUHsaModuleGlobal() {
918   if (getLexer().isNot(AsmToken::Identifier))
919     return TokError("expected symbol name");
920 
921   StringRef GlobalName = Parser.getTok().getIdentifier();
922 
923   getTargetStreamer().EmitAMDGPUHsaModuleScopeGlobal(GlobalName);
924   Lex();
925   return false;
926 }
927 
928 bool AMDGPUAsmParser::ParseDirectiveAMDGPUHsaProgramGlobal() {
929   if (getLexer().isNot(AsmToken::Identifier))
930     return TokError("expected symbol name");
931 
932   StringRef GlobalName = Parser.getTok().getIdentifier();
933 
934   getTargetStreamer().EmitAMDGPUHsaProgramScopeGlobal(GlobalName);
935   Lex();
936   return false;
937 }
938 
939 bool AMDGPUAsmParser::ParseSectionDirectiveHSADataGlobalAgent() {
940   getParser().getStreamer().SwitchSection(
941       AMDGPU::getHSADataGlobalAgentSection(getContext()));
942   return false;
943 }
944 
945 bool AMDGPUAsmParser::ParseSectionDirectiveHSADataGlobalProgram() {
946   getParser().getStreamer().SwitchSection(
947       AMDGPU::getHSADataGlobalProgramSection(getContext()));
948   return false;
949 }
950 
951 bool AMDGPUAsmParser::ParseSectionDirectiveHSARodataReadonlyAgent() {
952   getParser().getStreamer().SwitchSection(
953       AMDGPU::getHSARodataReadonlyAgentSection(getContext()));
954   return false;
955 }
956 
957 bool AMDGPUAsmParser::ParseDirective(AsmToken DirectiveID) {
958   StringRef IDVal = DirectiveID.getString();
959 
960   if (IDVal == ".hsa_code_object_version")
961     return ParseDirectiveHSACodeObjectVersion();
962 
963   if (IDVal == ".hsa_code_object_isa")
964     return ParseDirectiveHSACodeObjectISA();
965 
966   if (IDVal == ".amd_kernel_code_t")
967     return ParseDirectiveAMDKernelCodeT();
968 
969   if (IDVal == ".hsatext" || IDVal == ".text")
970     return ParseSectionDirectiveHSAText();
971 
972   if (IDVal == ".amdgpu_hsa_kernel")
973     return ParseDirectiveAMDGPUHsaKernel();
974 
975   if (IDVal == ".amdgpu_hsa_module_global")
976     return ParseDirectiveAMDGPUHsaModuleGlobal();
977 
978   if (IDVal == ".amdgpu_hsa_program_global")
979     return ParseDirectiveAMDGPUHsaProgramGlobal();
980 
981   if (IDVal == ".hsadata_global_agent")
982     return ParseSectionDirectiveHSADataGlobalAgent();
983 
984   if (IDVal == ".hsadata_global_program")
985     return ParseSectionDirectiveHSADataGlobalProgram();
986 
987   if (IDVal == ".hsarodata_readonly_agent")
988     return ParseSectionDirectiveHSARodataReadonlyAgent();
989 
990   return true;
991 }
992 
993 bool AMDGPUAsmParser::subtargetHasRegister(const MCRegisterInfo &MRI,
994                                            unsigned RegNo) const {
995   if (isCI())
996     return true;
997 
998   if (isSI()) {
999     // No flat_scr
1000     switch (RegNo) {
1001     case AMDGPU::FLAT_SCR:
1002     case AMDGPU::FLAT_SCR_LO:
1003     case AMDGPU::FLAT_SCR_HI:
1004       return false;
1005     default:
1006       return true;
1007     }
1008   }
1009 
1010   // VI only has 102 SGPRs, so make sure we aren't trying to use the 2 more that
1011   // SI/CI have.
1012   for (MCRegAliasIterator R(AMDGPU::SGPR102_SGPR103, &MRI, true);
1013        R.isValid(); ++R) {
1014     if (*R == RegNo)
1015       return false;
1016   }
1017 
1018   return true;
1019 }
1020 
1021 static bool operandsHaveModifiers(const OperandVector &Operands) {
1022 
1023   for (unsigned i = 0, e = Operands.size(); i != e; ++i) {
1024     const AMDGPUOperand &Op = ((AMDGPUOperand&)*Operands[i]);
1025     if (Op.isRegKind() && Op.hasModifiers())
1026       return true;
1027     if (Op.isImm() && Op.hasModifiers())
1028       return true;
1029     if (Op.isImm() && (Op.getImmTy() == AMDGPUOperand::ImmTyOMod ||
1030                        Op.getImmTy() == AMDGPUOperand::ImmTyClamp))
1031       return true;
1032   }
1033   return false;
1034 }
1035 
1036 AMDGPUAsmParser::OperandMatchResultTy
1037 AMDGPUAsmParser::parseOperand(OperandVector &Operands, StringRef Mnemonic) {
1038 
1039   // Try to parse with a custom parser
1040   OperandMatchResultTy ResTy = MatchOperandParserImpl(Operands, Mnemonic);
1041 
1042   // If we successfully parsed the operand or if there as an error parsing,
1043   // we are done.
1044   //
1045   // If we are parsing after we reach EndOfStatement then this means we
1046   // are appending default values to the Operands list.  This is only done
1047   // by custom parser, so we shouldn't continue on to the generic parsing.
1048   if (ResTy == MatchOperand_Success || ResTy == MatchOperand_ParseFail||
1049       getLexer().is(AsmToken::EndOfStatement))
1050     return ResTy;
1051 
1052   bool Negate = false, Abs = false, Abs2 = false;
1053 
1054   if (getLexer().getKind()== AsmToken::Minus) {
1055     Parser.Lex();
1056     Negate = true;
1057   }
1058 
1059   if (getLexer().getKind() == AsmToken::Identifier && Parser.getTok().getString() == "abs") {
1060     Parser.Lex();
1061     Abs2 = true;
1062     if (getLexer().isNot(AsmToken::LParen)) {
1063       Error(Parser.getTok().getLoc(), "expected left paren after abs");
1064       return MatchOperand_ParseFail;
1065     }
1066     Parser.Lex();
1067   }
1068 
1069   if (getLexer().getKind() == AsmToken::Pipe) {
1070     Parser.Lex();
1071     Abs = true;
1072   }
1073 
1074   switch(getLexer().getKind()) {
1075     case AsmToken::Integer: {
1076       SMLoc S = Parser.getTok().getLoc();
1077       int64_t IntVal;
1078       if (getParser().parseAbsoluteExpression(IntVal))
1079         return MatchOperand_ParseFail;
1080       if (!isInt<32>(IntVal) && !isUInt<32>(IntVal)) {
1081         Error(S, "invalid immediate: only 32-bit values are legal");
1082         return MatchOperand_ParseFail;
1083       }
1084 
1085       if (Negate)
1086         IntVal *= -1;
1087       Operands.push_back(AMDGPUOperand::CreateImm(IntVal, S));
1088       return MatchOperand_Success;
1089     }
1090     case AsmToken::Real: {
1091       // FIXME: We should emit an error if a double precisions floating-point
1092       // value is used.  I'm not sure the best way to detect this.
1093       SMLoc S = Parser.getTok().getLoc();
1094       int64_t IntVal;
1095       if (getParser().parseAbsoluteExpression(IntVal))
1096         return MatchOperand_ParseFail;
1097 
1098       APFloat F((float)BitsToDouble(IntVal));
1099       if (Negate)
1100         F.changeSign();
1101       Operands.push_back(
1102           AMDGPUOperand::CreateImm(F.bitcastToAPInt().getZExtValue(), S));
1103       return MatchOperand_Success;
1104     }
1105     case AsmToken::Identifier: {
1106       if (auto R = parseRegister()) {
1107         unsigned Modifiers = 0;
1108 
1109         if (Negate)
1110           Modifiers |= 0x1;
1111 
1112         if (Abs) {
1113           if (getLexer().getKind() != AsmToken::Pipe)
1114             return MatchOperand_ParseFail;
1115           Parser.Lex();
1116           Modifiers |= 0x2;
1117         }
1118         if (Abs2) {
1119           if (getLexer().isNot(AsmToken::RParen)) {
1120             return MatchOperand_ParseFail;
1121           }
1122           Parser.Lex();
1123           Modifiers |= 0x2;
1124         }
1125         assert(R->isReg());
1126         R->Reg.IsForcedVOP3 = isForcedVOP3();
1127         if (Modifiers) {
1128           R->setModifiers(Modifiers);
1129         }
1130         Operands.push_back(std::move(R));
1131       } else {
1132         ResTy = parseVOP3OptionalOps(Operands);
1133         if (ResTy == MatchOperand_NoMatch) {
1134           const auto &Tok = Parser.getTok();
1135           Operands.push_back(AMDGPUOperand::CreateToken(Tok.getString(),
1136                                                         Tok.getLoc()));
1137           Parser.Lex();
1138         }
1139       }
1140       return MatchOperand_Success;
1141     }
1142     default:
1143       return MatchOperand_NoMatch;
1144   }
1145 }
1146 
1147 bool AMDGPUAsmParser::ParseInstruction(ParseInstructionInfo &Info,
1148                                        StringRef Name,
1149                                        SMLoc NameLoc, OperandVector &Operands) {
1150 
1151   // Clear any forced encodings from the previous instruction.
1152   setForcedEncodingSize(0);
1153 
1154   if (Name.endswith("_e64"))
1155     setForcedEncodingSize(64);
1156   else if (Name.endswith("_e32"))
1157     setForcedEncodingSize(32);
1158 
1159   // Add the instruction mnemonic
1160   Operands.push_back(AMDGPUOperand::CreateToken(Name, NameLoc));
1161 
1162   while (!getLexer().is(AsmToken::EndOfStatement)) {
1163     AMDGPUAsmParser::OperandMatchResultTy Res = parseOperand(Operands, Name);
1164 
1165     // Eat the comma or space if there is one.
1166     if (getLexer().is(AsmToken::Comma))
1167       Parser.Lex();
1168 
1169     switch (Res) {
1170       case MatchOperand_Success: break;
1171       case MatchOperand_ParseFail: return Error(getLexer().getLoc(),
1172                                                 "failed parsing operand.");
1173       case MatchOperand_NoMatch: return Error(getLexer().getLoc(),
1174                                               "not a valid operand.");
1175     }
1176   }
1177 
1178   return false;
1179 }
1180 
1181 //===----------------------------------------------------------------------===//
1182 // Utility functions
1183 //===----------------------------------------------------------------------===//
1184 
1185 AMDGPUAsmParser::OperandMatchResultTy
1186 AMDGPUAsmParser::parseIntWithPrefix(const char *Prefix, int64_t &Int,
1187                                     int64_t Default) {
1188   // We are at the end of the statement, and this is a default argument, so
1189   // use a default value.
1190   if (getLexer().is(AsmToken::EndOfStatement)) {
1191     Int = Default;
1192     return MatchOperand_Success;
1193   }
1194 
1195   switch(getLexer().getKind()) {
1196     default: return MatchOperand_NoMatch;
1197     case AsmToken::Identifier: {
1198       StringRef OffsetName = Parser.getTok().getString();
1199       if (!OffsetName.equals(Prefix))
1200         return MatchOperand_NoMatch;
1201 
1202       Parser.Lex();
1203       if (getLexer().isNot(AsmToken::Colon))
1204         return MatchOperand_ParseFail;
1205 
1206       Parser.Lex();
1207       if (getLexer().isNot(AsmToken::Integer))
1208         return MatchOperand_ParseFail;
1209 
1210       if (getParser().parseAbsoluteExpression(Int))
1211         return MatchOperand_ParseFail;
1212       break;
1213     }
1214   }
1215   return MatchOperand_Success;
1216 }
1217 
1218 AMDGPUAsmParser::OperandMatchResultTy
1219 AMDGPUAsmParser::parseIntWithPrefix(const char *Prefix, OperandVector &Operands,
1220                                     enum AMDGPUOperand::ImmTy ImmTy) {
1221 
1222   SMLoc S = Parser.getTok().getLoc();
1223   int64_t Offset = 0;
1224 
1225   AMDGPUAsmParser::OperandMatchResultTy Res = parseIntWithPrefix(Prefix, Offset);
1226   if (Res != MatchOperand_Success)
1227     return Res;
1228 
1229   Operands.push_back(AMDGPUOperand::CreateImm(Offset, S, ImmTy));
1230   return MatchOperand_Success;
1231 }
1232 
1233 AMDGPUAsmParser::OperandMatchResultTy
1234 AMDGPUAsmParser::parseNamedBit(const char *Name, OperandVector &Operands,
1235                                enum AMDGPUOperand::ImmTy ImmTy) {
1236   int64_t Bit = 0;
1237   SMLoc S = Parser.getTok().getLoc();
1238 
1239   // We are at the end of the statement, and this is a default argument, so
1240   // use a default value.
1241   if (getLexer().isNot(AsmToken::EndOfStatement)) {
1242     switch(getLexer().getKind()) {
1243       case AsmToken::Identifier: {
1244         StringRef Tok = Parser.getTok().getString();
1245         if (Tok == Name) {
1246           Bit = 1;
1247           Parser.Lex();
1248         } else if (Tok.startswith("no") && Tok.endswith(Name)) {
1249           Bit = 0;
1250           Parser.Lex();
1251         } else {
1252           return MatchOperand_NoMatch;
1253         }
1254         break;
1255       }
1256       default:
1257         return MatchOperand_NoMatch;
1258     }
1259   }
1260 
1261   Operands.push_back(AMDGPUOperand::CreateImm(Bit, S, ImmTy));
1262   return MatchOperand_Success;
1263 }
1264 
1265 typedef std::map<enum AMDGPUOperand::ImmTy, unsigned> OptionalImmIndexMap;
1266 
1267 void addOptionalImmOperand(MCInst& Inst, const OperandVector& Operands,
1268                            OptionalImmIndexMap& OptionalIdx,
1269                            enum AMDGPUOperand::ImmTy ImmT, int64_t Default = 0) {
1270   auto i = OptionalIdx.find(ImmT);
1271   if (i != OptionalIdx.end()) {
1272     unsigned Idx = i->second;
1273     ((AMDGPUOperand &)*Operands[Idx]).addImmOperands(Inst, 1);
1274   } else {
1275     Inst.addOperand(MCOperand::createImm(Default));
1276   }
1277 }
1278 
1279 static bool operandsHasOptionalOp(const OperandVector &Operands,
1280                                   const OptionalOperand &OOp) {
1281   for (unsigned i = 0; i < Operands.size(); i++) {
1282     const AMDGPUOperand &ParsedOp = ((const AMDGPUOperand &)*Operands[i]);
1283     if ((ParsedOp.isImm() && ParsedOp.getImmTy() == OOp.Type) ||
1284         (ParsedOp.isToken() && ParsedOp.getToken() == OOp.Name))
1285       return true;
1286 
1287   }
1288   return false;
1289 }
1290 
1291 AMDGPUAsmParser::OperandMatchResultTy
1292 AMDGPUAsmParser::parseOptionalOps(const ArrayRef<OptionalOperand> &OptionalOps,
1293                                    OperandVector &Operands) {
1294   SMLoc S = Parser.getTok().getLoc();
1295   for (const OptionalOperand &Op : OptionalOps) {
1296     if (operandsHasOptionalOp(Operands, Op))
1297       continue;
1298     AMDGPUAsmParser::OperandMatchResultTy Res;
1299     int64_t Value;
1300     if (Op.IsBit) {
1301       Res = parseNamedBit(Op.Name, Operands, Op.Type);
1302       if (Res == MatchOperand_NoMatch)
1303         continue;
1304       return Res;
1305     }
1306 
1307     Res = parseIntWithPrefix(Op.Name, Value, Op.Default);
1308 
1309     if (Res == MatchOperand_NoMatch)
1310       continue;
1311 
1312     if (Res != MatchOperand_Success)
1313       return Res;
1314 
1315     bool DefaultValue = (Value == Op.Default);
1316 
1317     if (Op.ConvertResult && !Op.ConvertResult(Value)) {
1318       return MatchOperand_ParseFail;
1319     }
1320 
1321     if (!DefaultValue) {
1322       Operands.push_back(AMDGPUOperand::CreateImm(Value, S, Op.Type));
1323     }
1324     return MatchOperand_Success;
1325   }
1326   return MatchOperand_NoMatch;
1327 }
1328 
1329 //===----------------------------------------------------------------------===//
1330 // ds
1331 //===----------------------------------------------------------------------===//
1332 
1333 static const OptionalOperand DSOptionalOps [] = {
1334   {"offset",  AMDGPUOperand::ImmTyOffset, false, 0, nullptr},
1335   {"gds",     AMDGPUOperand::ImmTyGDS, true, 0, nullptr}
1336 };
1337 
1338 static const OptionalOperand DSOptionalOpsOff01 [] = {
1339   {"offset0", AMDGPUOperand::ImmTyDSOffset0, false, 0, nullptr},
1340   {"offset1", AMDGPUOperand::ImmTyDSOffset1, false, 0, nullptr},
1341   {"gds",     AMDGPUOperand::ImmTyGDS, true, 0, nullptr}
1342 };
1343 
1344 AMDGPUAsmParser::OperandMatchResultTy
1345 AMDGPUAsmParser::parseDSOptionalOps(OperandVector &Operands) {
1346   return parseOptionalOps(DSOptionalOps, Operands);
1347 }
1348 AMDGPUAsmParser::OperandMatchResultTy
1349 AMDGPUAsmParser::parseDSOff01OptionalOps(OperandVector &Operands) {
1350   return parseOptionalOps(DSOptionalOpsOff01, Operands);
1351 }
1352 
1353 AMDGPUAsmParser::OperandMatchResultTy
1354 AMDGPUAsmParser::parseDSOffsetOptional(OperandVector &Operands) {
1355   SMLoc S = Parser.getTok().getLoc();
1356   AMDGPUAsmParser::OperandMatchResultTy Res =
1357     parseIntWithPrefix("offset", Operands, AMDGPUOperand::ImmTyOffset);
1358   if (Res == MatchOperand_NoMatch) {
1359     Operands.push_back(AMDGPUOperand::CreateImm(0, S,
1360                        AMDGPUOperand::ImmTyOffset));
1361     Res = MatchOperand_Success;
1362   }
1363   return Res;
1364 }
1365 
1366 bool AMDGPUOperand::isDSOffset() const {
1367   return isImm() && isUInt<16>(getImm());
1368 }
1369 
1370 bool AMDGPUOperand::isDSOffset01() const {
1371   return isImm() && isUInt<8>(getImm());
1372 }
1373 
1374 void AMDGPUAsmParser::cvtDSOffset01(MCInst &Inst,
1375                                     const OperandVector &Operands) {
1376 
1377   OptionalImmIndexMap OptionalIdx;
1378 
1379   for (unsigned i = 1, e = Operands.size(); i != e; ++i) {
1380     AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[i]);
1381 
1382     // Add the register arguments
1383     if (Op.isReg()) {
1384       Op.addRegOperands(Inst, 1);
1385       continue;
1386     }
1387 
1388     // Handle optional arguments
1389     OptionalIdx[Op.getImmTy()] = i;
1390   }
1391 
1392   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyDSOffset0);
1393   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyDSOffset1);
1394   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyGDS);
1395 
1396   Inst.addOperand(MCOperand::createReg(AMDGPU::M0)); // m0
1397 }
1398 
1399 void AMDGPUAsmParser::cvtDS(MCInst &Inst, const OperandVector &Operands) {
1400 
1401   std::map<enum AMDGPUOperand::ImmTy, unsigned> OptionalIdx;
1402   bool GDSOnly = false;
1403 
1404   for (unsigned i = 1, e = Operands.size(); i != e; ++i) {
1405     AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[i]);
1406 
1407     // Add the register arguments
1408     if (Op.isReg()) {
1409       Op.addRegOperands(Inst, 1);
1410       continue;
1411     }
1412 
1413     if (Op.isToken() && Op.getToken() == "gds") {
1414       GDSOnly = true;
1415       continue;
1416     }
1417 
1418     // Handle optional arguments
1419     OptionalIdx[Op.getImmTy()] = i;
1420   }
1421 
1422   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyOffset);
1423   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyGDS);
1424 
1425   if (!GDSOnly) {
1426     addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyGDS);
1427   }
1428   Inst.addOperand(MCOperand::createReg(AMDGPU::M0)); // m0
1429 }
1430 
1431 
1432 //===----------------------------------------------------------------------===//
1433 // s_waitcnt
1434 //===----------------------------------------------------------------------===//
1435 
1436 bool AMDGPUAsmParser::parseCnt(int64_t &IntVal) {
1437   StringRef CntName = Parser.getTok().getString();
1438   int64_t CntVal;
1439 
1440   Parser.Lex();
1441   if (getLexer().isNot(AsmToken::LParen))
1442     return true;
1443 
1444   Parser.Lex();
1445   if (getLexer().isNot(AsmToken::Integer))
1446     return true;
1447 
1448   if (getParser().parseAbsoluteExpression(CntVal))
1449     return true;
1450 
1451   if (getLexer().isNot(AsmToken::RParen))
1452     return true;
1453 
1454   Parser.Lex();
1455   if (getLexer().is(AsmToken::Amp) || getLexer().is(AsmToken::Comma))
1456     Parser.Lex();
1457 
1458   int CntShift;
1459   int CntMask;
1460 
1461   if (CntName == "vmcnt") {
1462     CntMask = 0xf;
1463     CntShift = 0;
1464   } else if (CntName == "expcnt") {
1465     CntMask = 0x7;
1466     CntShift = 4;
1467   } else if (CntName == "lgkmcnt") {
1468     CntMask = 0xf;
1469     CntShift = 8;
1470   } else {
1471     return true;
1472   }
1473 
1474   IntVal &= ~(CntMask << CntShift);
1475   IntVal |= (CntVal << CntShift);
1476   return false;
1477 }
1478 
1479 AMDGPUAsmParser::OperandMatchResultTy
1480 AMDGPUAsmParser::parseSWaitCntOps(OperandVector &Operands) {
1481   // Disable all counters by default.
1482   // vmcnt   [3:0]
1483   // expcnt  [6:4]
1484   // lgkmcnt [11:8]
1485   int64_t CntVal = 0xf7f;
1486   SMLoc S = Parser.getTok().getLoc();
1487 
1488   switch(getLexer().getKind()) {
1489     default: return MatchOperand_ParseFail;
1490     case AsmToken::Integer:
1491       // The operand can be an integer value.
1492       if (getParser().parseAbsoluteExpression(CntVal))
1493         return MatchOperand_ParseFail;
1494       break;
1495 
1496     case AsmToken::Identifier:
1497       do {
1498         if (parseCnt(CntVal))
1499           return MatchOperand_ParseFail;
1500       } while(getLexer().isNot(AsmToken::EndOfStatement));
1501       break;
1502   }
1503   Operands.push_back(AMDGPUOperand::CreateImm(CntVal, S));
1504   return MatchOperand_Success;
1505 }
1506 
1507 bool AMDGPUOperand::isSWaitCnt() const {
1508   return isImm();
1509 }
1510 
1511 //===----------------------------------------------------------------------===//
1512 // sopp branch targets
1513 //===----------------------------------------------------------------------===//
1514 
1515 AMDGPUAsmParser::OperandMatchResultTy
1516 AMDGPUAsmParser::parseSOppBrTarget(OperandVector &Operands) {
1517   SMLoc S = Parser.getTok().getLoc();
1518 
1519   switch (getLexer().getKind()) {
1520     default: return MatchOperand_ParseFail;
1521     case AsmToken::Integer: {
1522       int64_t Imm;
1523       if (getParser().parseAbsoluteExpression(Imm))
1524         return MatchOperand_ParseFail;
1525       Operands.push_back(AMDGPUOperand::CreateImm(Imm, S));
1526       return MatchOperand_Success;
1527     }
1528 
1529     case AsmToken::Identifier:
1530       Operands.push_back(AMDGPUOperand::CreateExpr(
1531           MCSymbolRefExpr::create(getContext().getOrCreateSymbol(
1532                                   Parser.getTok().getString()), getContext()), S));
1533       Parser.Lex();
1534       return MatchOperand_Success;
1535   }
1536 }
1537 
1538 //===----------------------------------------------------------------------===//
1539 // flat
1540 //===----------------------------------------------------------------------===//
1541 
1542 static const OptionalOperand FlatOptionalOps [] = {
1543   {"glc",    AMDGPUOperand::ImmTyGLC, true, 0, nullptr},
1544   {"slc",    AMDGPUOperand::ImmTySLC, true, 0, nullptr},
1545   {"tfe",    AMDGPUOperand::ImmTyTFE, true, 0, nullptr}
1546 };
1547 
1548 static const OptionalOperand FlatAtomicOptionalOps [] = {
1549   {"slc",    AMDGPUOperand::ImmTySLC, true, 0, nullptr},
1550   {"tfe",    AMDGPUOperand::ImmTyTFE, true, 0, nullptr}
1551 };
1552 
1553 AMDGPUAsmParser::OperandMatchResultTy
1554 AMDGPUAsmParser::parseFlatOptionalOps(OperandVector &Operands) {
1555   return parseOptionalOps(FlatOptionalOps, Operands);
1556 }
1557 
1558 AMDGPUAsmParser::OperandMatchResultTy
1559 AMDGPUAsmParser::parseFlatAtomicOptionalOps(OperandVector &Operands) {
1560   return parseOptionalOps(FlatAtomicOptionalOps, Operands);
1561 }
1562 
1563 void AMDGPUAsmParser::cvtFlat(MCInst &Inst,
1564                                const OperandVector &Operands) {
1565   OptionalImmIndexMap OptionalIdx;
1566 
1567   for (unsigned i = 1, e = Operands.size(); i != e; ++i) {
1568     AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[i]);
1569 
1570     // Add the register arguments
1571     if (Op.isReg()) {
1572       Op.addRegOperands(Inst, 1);
1573       continue;
1574     }
1575 
1576     OptionalIdx[Op.getImmTy()] = i;
1577   }
1578   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyGLC);
1579   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySLC);
1580   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyTFE);
1581 }
1582 
1583 
1584 void AMDGPUAsmParser::cvtFlatAtomic(MCInst &Inst,
1585                                const OperandVector &Operands) {
1586   OptionalImmIndexMap OptionalIdx;
1587 
1588   for (unsigned i = 1, e = Operands.size(); i != e; ++i) {
1589     AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[i]);
1590 
1591     // Add the register arguments
1592     if (Op.isReg()) {
1593       Op.addRegOperands(Inst, 1);
1594       continue;
1595     }
1596 
1597     // Handle 'glc' token for flat atomics.
1598     if (Op.isToken()) {
1599       continue;
1600     }
1601 
1602     // Handle optional arguments
1603     OptionalIdx[Op.getImmTy()] = i;
1604   }
1605   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySLC);
1606   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyTFE);
1607 }
1608 
1609 //===----------------------------------------------------------------------===//
1610 // mubuf
1611 //===----------------------------------------------------------------------===//
1612 
1613 static const OptionalOperand MubufOptionalOps [] = {
1614   {"offset", AMDGPUOperand::ImmTyOffset, false, 0, nullptr},
1615   {"glc",    AMDGPUOperand::ImmTyGLC, true, 0, nullptr},
1616   {"slc",    AMDGPUOperand::ImmTySLC, true, 0, nullptr},
1617   {"tfe",    AMDGPUOperand::ImmTyTFE, true, 0, nullptr}
1618 };
1619 
1620 AMDGPUAsmParser::OperandMatchResultTy
1621 AMDGPUAsmParser::parseMubufOptionalOps(OperandVector &Operands) {
1622   return parseOptionalOps(MubufOptionalOps, Operands);
1623 }
1624 
1625 AMDGPUAsmParser::OperandMatchResultTy
1626 AMDGPUAsmParser::parseOffset(OperandVector &Operands) {
1627   return parseIntWithPrefix("offset", Operands);
1628 }
1629 
1630 AMDGPUAsmParser::OperandMatchResultTy
1631 AMDGPUAsmParser::parseGLC(OperandVector &Operands) {
1632   return parseNamedBit("glc", Operands);
1633 }
1634 
1635 AMDGPUAsmParser::OperandMatchResultTy
1636 AMDGPUAsmParser::parseSLC(OperandVector &Operands) {
1637   return parseNamedBit("slc", Operands);
1638 }
1639 
1640 AMDGPUAsmParser::OperandMatchResultTy
1641 AMDGPUAsmParser::parseTFE(OperandVector &Operands) {
1642   return parseNamedBit("tfe", Operands);
1643 }
1644 
1645 bool AMDGPUOperand::isMubufOffset() const {
1646   return isImmTy(ImmTyOffset) && isUInt<12>(getImm());
1647 }
1648 
1649 void AMDGPUAsmParser::cvtMubuf(MCInst &Inst,
1650                                const OperandVector &Operands) {
1651   OptionalImmIndexMap OptionalIdx;
1652 
1653   for (unsigned i = 1, e = Operands.size(); i != e; ++i) {
1654     AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[i]);
1655 
1656     // Add the register arguments
1657     if (Op.isReg()) {
1658       Op.addRegOperands(Inst, 1);
1659       continue;
1660     }
1661 
1662     // Handle the case where soffset is an immediate
1663     if (Op.isImm() && Op.getImmTy() == AMDGPUOperand::ImmTyNone) {
1664       Op.addImmOperands(Inst, 1);
1665       continue;
1666     }
1667 
1668     // Handle tokens like 'offen' which are sometimes hard-coded into the
1669     // asm string.  There are no MCInst operands for these.
1670     if (Op.isToken()) {
1671       continue;
1672     }
1673     assert(Op.isImm());
1674 
1675     // Handle optional arguments
1676     OptionalIdx[Op.getImmTy()] = i;
1677   }
1678 
1679   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyOffset);
1680   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyGLC);
1681   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySLC);
1682   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyTFE);
1683 }
1684 
1685 //===----------------------------------------------------------------------===//
1686 // mimg
1687 //===----------------------------------------------------------------------===//
1688 
1689 AMDGPUAsmParser::OperandMatchResultTy
1690 AMDGPUAsmParser::parseDMask(OperandVector &Operands) {
1691   return parseIntWithPrefix("dmask", Operands, AMDGPUOperand::ImmTyDMask);
1692 }
1693 
1694 AMDGPUAsmParser::OperandMatchResultTy
1695 AMDGPUAsmParser::parseUNorm(OperandVector &Operands) {
1696   return parseNamedBit("unorm", Operands, AMDGPUOperand::ImmTyUNorm);
1697 }
1698 
1699 AMDGPUAsmParser::OperandMatchResultTy
1700 AMDGPUAsmParser::parseDA(OperandVector &Operands) {
1701   return parseNamedBit("da", Operands, AMDGPUOperand::ImmTyDA);
1702 }
1703 
1704 AMDGPUAsmParser::OperandMatchResultTy
1705 AMDGPUAsmParser::parseR128(OperandVector &Operands) {
1706   return parseNamedBit("r128", Operands, AMDGPUOperand::ImmTyR128);
1707 }
1708 
1709 AMDGPUAsmParser::OperandMatchResultTy
1710 AMDGPUAsmParser::parseLWE(OperandVector &Operands) {
1711   return parseNamedBit("lwe", Operands, AMDGPUOperand::ImmTyLWE);
1712 }
1713 
1714 //===----------------------------------------------------------------------===//
1715 // smrd
1716 //===----------------------------------------------------------------------===//
1717 
1718 bool AMDGPUOperand::isSMRDOffset() const {
1719 
1720   // FIXME: Support 20-bit offsets on VI.  We need to to pass subtarget
1721   // information here.
1722   return isImm() && isUInt<8>(getImm());
1723 }
1724 
1725 bool AMDGPUOperand::isSMRDLiteralOffset() const {
1726   // 32-bit literals are only supported on CI and we only want to use them
1727   // when the offset is > 8-bits.
1728   return isImm() && !isUInt<8>(getImm()) && isUInt<32>(getImm());
1729 }
1730 
1731 //===----------------------------------------------------------------------===//
1732 // vop3
1733 //===----------------------------------------------------------------------===//
1734 
1735 static bool ConvertOmodMul(int64_t &Mul) {
1736   if (Mul != 1 && Mul != 2 && Mul != 4)
1737     return false;
1738 
1739   Mul >>= 1;
1740   return true;
1741 }
1742 
1743 static bool ConvertOmodDiv(int64_t &Div) {
1744   if (Div == 1) {
1745     Div = 0;
1746     return true;
1747   }
1748 
1749   if (Div == 2) {
1750     Div = 3;
1751     return true;
1752   }
1753 
1754   return false;
1755 }
1756 
1757 static const OptionalOperand VOP3OptionalOps [] = {
1758   {"clamp", AMDGPUOperand::ImmTyClamp, true, 0, nullptr},
1759   {"mul",   AMDGPUOperand::ImmTyOMod, false, 1, ConvertOmodMul},
1760   {"div",   AMDGPUOperand::ImmTyOMod, false, 1, ConvertOmodDiv},
1761 };
1762 
1763 static bool isVOP3(OperandVector &Operands) {
1764   if (operandsHaveModifiers(Operands))
1765     return true;
1766 
1767   if (Operands.size() >= 2) {
1768     AMDGPUOperand &DstOp = ((AMDGPUOperand&)*Operands[1]);
1769 
1770     if (DstOp.isRegClass(AMDGPU::SGPR_64RegClassID))
1771       return true;
1772   }
1773 
1774   if (Operands.size() >= 5)
1775     return true;
1776 
1777   if (Operands.size() > 3) {
1778     AMDGPUOperand &Src1Op = ((AMDGPUOperand&)*Operands[3]);
1779     if (Src1Op.isRegClass(AMDGPU::SReg_32RegClassID) ||
1780         Src1Op.isRegClass(AMDGPU::SReg_64RegClassID))
1781       return true;
1782   }
1783   return false;
1784 }
1785 
1786 AMDGPUAsmParser::OperandMatchResultTy
1787 AMDGPUAsmParser::parseVOP3OptionalOps(OperandVector &Operands) {
1788 
1789   // The value returned by this function may change after parsing
1790   // an operand so store the original value here.
1791   bool HasModifiers = operandsHaveModifiers(Operands);
1792 
1793   bool IsVOP3 = isVOP3(Operands);
1794   if (HasModifiers || IsVOP3 ||
1795       getLexer().isNot(AsmToken::EndOfStatement) ||
1796       getForcedEncodingSize() == 64) {
1797 
1798     AMDGPUAsmParser::OperandMatchResultTy Res =
1799         parseOptionalOps(VOP3OptionalOps, Operands);
1800 
1801     if (!HasModifiers && Res == MatchOperand_Success) {
1802       // We have added a modifier operation, so we need to make sure all
1803       // previous register operands have modifiers
1804       for (unsigned i = 2, e = Operands.size(); i != e; ++i) {
1805         AMDGPUOperand &Op = ((AMDGPUOperand&)*Operands[i]);
1806         if ((Op.isReg() || Op.isImm()) && !Op.hasModifiers())
1807           Op.setModifiers(0);
1808       }
1809     }
1810     return Res;
1811   }
1812   return MatchOperand_NoMatch;
1813 }
1814 
1815 void AMDGPUAsmParser::cvtId(MCInst &Inst, const OperandVector &Operands) {
1816   unsigned I = 1;
1817   const MCInstrDesc &Desc = MII.get(Inst.getOpcode());
1818   for (unsigned J = 0; J < Desc.getNumDefs(); ++J) {
1819     ((AMDGPUOperand &)*Operands[I++]).addRegOperands(Inst, 1);
1820   }
1821   for (unsigned E = Operands.size(); I != E; ++I)
1822     ((AMDGPUOperand &)*Operands[I]).addRegOrImmOperands(Inst, 1);
1823 }
1824 
1825 void AMDGPUAsmParser::cvtVOP3_2_mod(MCInst &Inst, const OperandVector &Operands) {
1826   uint64_t TSFlags = MII.get(Inst.getOpcode()).TSFlags;
1827   if (TSFlags & SIInstrFlags::VOP3) {
1828     cvtVOP3(Inst, Operands);
1829   } else {
1830     cvtId(Inst, Operands);
1831   }
1832 }
1833 
1834 void AMDGPUAsmParser::cvtVOP3_2_nomod(MCInst &Inst, const OperandVector &Operands) {
1835   if (operandsHaveModifiers(Operands)) {
1836     cvtVOP3(Inst, Operands);
1837   } else {
1838     cvtId(Inst, Operands);
1839   }
1840 }
1841 
1842 void AMDGPUAsmParser::cvtVOP3_only(MCInst &Inst, const OperandVector &Operands) {
1843   cvtVOP3(Inst, Operands);
1844 }
1845 
1846 void AMDGPUAsmParser::cvtVOP3(MCInst &Inst, const OperandVector &Operands) {
1847   OptionalImmIndexMap OptionalIdx;
1848   unsigned I = 1;
1849   const MCInstrDesc &Desc = MII.get(Inst.getOpcode());
1850   for (unsigned J = 0; J < Desc.getNumDefs(); ++J) {
1851     ((AMDGPUOperand &)*Operands[I++]).addRegOperands(Inst, 1);
1852   }
1853 
1854   for (unsigned E = Operands.size(); I != E; ++I) {
1855     AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[I]);
1856     if (Op.isRegOrImmWithInputMods()) {
1857       Op.addRegOrImmWithInputModsOperands(Inst, 2);
1858     } else if (Op.isImm()) {
1859       OptionalIdx[Op.getImmTy()] = I;
1860     } else {
1861       assert(false);
1862     }
1863   }
1864 
1865   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyClamp);
1866   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyOMod);
1867 }
1868 
1869 void AMDGPUAsmParser::cvtMIMG(MCInst &Inst, const OperandVector &Operands) {
1870   unsigned I = 1;
1871   const MCInstrDesc &Desc = MII.get(Inst.getOpcode());
1872   for (unsigned J = 0; J < Desc.getNumDefs(); ++J) {
1873     ((AMDGPUOperand &)*Operands[I++]).addRegOperands(Inst, 1);
1874   }
1875 
1876   OptionalImmIndexMap OptionalIdx;
1877 
1878   for (unsigned E = Operands.size(); I != E; ++I) {
1879     AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[I]);
1880 
1881     // Add the register arguments
1882     if (Op.isRegOrImm()) {
1883       Op.addRegOrImmOperands(Inst, 1);
1884       continue;
1885     } else if (Op.isImmModifier()) {
1886       OptionalIdx[Op.getImmTy()] = I;
1887     } else {
1888       assert(false);
1889     }
1890   }
1891 
1892   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyDMask);
1893   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyUNorm);
1894   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyGLC);
1895   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyDA);
1896   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyR128);
1897   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyTFE);
1898   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyLWE);
1899   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySLC);
1900 }
1901 
1902 void AMDGPUAsmParser::cvtMIMGAtomic(MCInst &Inst, const OperandVector &Operands) {
1903   unsigned I = 1;
1904   const MCInstrDesc &Desc = MII.get(Inst.getOpcode());
1905   for (unsigned J = 0; J < Desc.getNumDefs(); ++J) {
1906     ((AMDGPUOperand &)*Operands[I++]).addRegOperands(Inst, 1);
1907   }
1908 
1909   // Add src, same as dst
1910   ((AMDGPUOperand &)*Operands[I]).addRegOperands(Inst, 1);
1911 
1912   OptionalImmIndexMap OptionalIdx;
1913 
1914   for (unsigned E = Operands.size(); I != E; ++I) {
1915     AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[I]);
1916 
1917     // Add the register arguments
1918     if (Op.isRegOrImm()) {
1919       Op.addRegOrImmOperands(Inst, 1);
1920       continue;
1921     } else if (Op.isImmModifier()) {
1922       OptionalIdx[Op.getImmTy()] = I;
1923     } else {
1924       assert(false);
1925     }
1926   }
1927 
1928   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyDMask);
1929   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyUNorm);
1930   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyGLC);
1931   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyDA);
1932   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyR128);
1933   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyTFE);
1934   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyLWE);
1935   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySLC);
1936 }
1937 
1938 //===----------------------------------------------------------------------===//
1939 // dpp
1940 //===----------------------------------------------------------------------===//
1941 
1942 bool AMDGPUOperand::isDPPCtrl() const {
1943   bool result = isImm() && getImmTy() == ImmTyDppCtrl && isUInt<9>(getImm());
1944   if (result) {
1945     int64_t Imm = getImm();
1946     return ((Imm >= 0x000) && (Imm <= 0x0ff)) ||
1947            ((Imm >= 0x101) && (Imm <= 0x10f)) ||
1948            ((Imm >= 0x111) && (Imm <= 0x11f)) ||
1949            ((Imm >= 0x121) && (Imm <= 0x12f)) ||
1950            (Imm == 0x130) ||
1951            (Imm == 0x134) ||
1952            (Imm == 0x138) ||
1953            (Imm == 0x13c) ||
1954            (Imm == 0x140) ||
1955            (Imm == 0x141) ||
1956            (Imm == 0x142) ||
1957            (Imm == 0x143);
1958   }
1959   return false;
1960 }
1961 
1962 AMDGPUAsmParser::OperandMatchResultTy
1963 AMDGPUAsmParser::parseDPPCtrlOps(OperandVector &Operands) {
1964   // ToDo: use same syntax as sp3 for dpp_ctrl
1965   SMLoc S = Parser.getTok().getLoc();
1966   StringRef Prefix;
1967   int64_t Int;
1968 
1969   if (getLexer().getKind() == AsmToken::Identifier) {
1970     Prefix = Parser.getTok().getString();
1971   } else {
1972     return MatchOperand_NoMatch;
1973   }
1974 
1975   if (Prefix == "row_mirror") {
1976     Int = 0x140;
1977   } else if (Prefix == "row_half_mirror") {
1978     Int = 0x141;
1979   } else {
1980     Parser.Lex();
1981     if (getLexer().isNot(AsmToken::Colon))
1982       return MatchOperand_ParseFail;
1983 
1984     if (Prefix == "quad_perm") {
1985       // quad_perm:[%d,%d,%d,%d]
1986       Parser.Lex();
1987       if (getLexer().isNot(AsmToken::LBrac))
1988         return MatchOperand_ParseFail;
1989 
1990       Parser.Lex();
1991       if (getLexer().isNot(AsmToken::Integer))
1992         return MatchOperand_ParseFail;
1993       Int = getLexer().getTok().getIntVal();
1994 
1995       Parser.Lex();
1996       if (getLexer().isNot(AsmToken::Comma))
1997         return MatchOperand_ParseFail;
1998       Parser.Lex();
1999       if (getLexer().isNot(AsmToken::Integer))
2000         return MatchOperand_ParseFail;
2001       Int += (getLexer().getTok().getIntVal() << 2);
2002 
2003       Parser.Lex();
2004       if (getLexer().isNot(AsmToken::Comma))
2005         return MatchOperand_ParseFail;
2006       Parser.Lex();
2007       if (getLexer().isNot(AsmToken::Integer))
2008         return MatchOperand_ParseFail;
2009       Int += (getLexer().getTok().getIntVal() << 4);
2010 
2011       Parser.Lex();
2012       if (getLexer().isNot(AsmToken::Comma))
2013         return MatchOperand_ParseFail;
2014       Parser.Lex();
2015       if (getLexer().isNot(AsmToken::Integer))
2016         return MatchOperand_ParseFail;
2017       Int += (getLexer().getTok().getIntVal() << 6);
2018 
2019       Parser.Lex();
2020       if (getLexer().isNot(AsmToken::RBrac))
2021         return MatchOperand_ParseFail;
2022 
2023     } else {
2024       // sel:%d
2025       Parser.Lex();
2026       if (getLexer().isNot(AsmToken::Integer))
2027         return MatchOperand_ParseFail;
2028       Int = getLexer().getTok().getIntVal();
2029 
2030       if (Prefix == "row_shl") {
2031         Int |= 0x100;
2032       } else if (Prefix == "row_shr") {
2033         Int |= 0x110;
2034       } else if (Prefix == "row_ror") {
2035         Int |= 0x120;
2036       } else if (Prefix == "wave_shl") {
2037         Int = 0x130;
2038       } else if (Prefix == "wave_rol") {
2039         Int = 0x134;
2040       } else if (Prefix == "wave_shr") {
2041         Int = 0x138;
2042       } else if (Prefix == "wave_ror") {
2043         Int = 0x13C;
2044       } else if (Prefix == "row_bcast") {
2045         if (Int == 15) {
2046           Int = 0x142;
2047         } else if (Int == 31) {
2048           Int = 0x143;
2049         }
2050       } else {
2051         return MatchOperand_NoMatch;
2052       }
2053     }
2054   }
2055   Parser.Lex(); // eat last token
2056 
2057   Operands.push_back(AMDGPUOperand::CreateImm(Int, S,
2058                                               AMDGPUOperand::ImmTyDppCtrl));
2059   return MatchOperand_Success;
2060 }
2061 
2062 static const OptionalOperand DPPOptionalOps [] = {
2063   {"row_mask", AMDGPUOperand::ImmTyDppRowMask, false, 0xf, nullptr},
2064   {"bank_mask", AMDGPUOperand::ImmTyDppBankMask, false, 0xf, nullptr},
2065   {"bound_ctrl", AMDGPUOperand::ImmTyDppBoundCtrl, false, -1, nullptr}
2066 };
2067 
2068 AMDGPUAsmParser::OperandMatchResultTy
2069 AMDGPUAsmParser::parseDPPOptionalOps(OperandVector &Operands) {
2070   SMLoc S = Parser.getTok().getLoc();
2071   OperandMatchResultTy Res = parseOptionalOps(DPPOptionalOps, Operands);
2072   // XXX - sp3 use syntax "bound_ctrl:0" to indicate that bound_ctrl bit was set
2073   if (Res == MatchOperand_Success) {
2074     AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands.back());
2075     // If last operand was parsed as bound_ctrl we should replace it with correct value (1)
2076     if (Op.isImmTy(AMDGPUOperand::ImmTyDppBoundCtrl)) {
2077       Operands.pop_back();
2078       Operands.push_back(
2079         AMDGPUOperand::CreateImm(1, S, AMDGPUOperand::ImmTyDppBoundCtrl));
2080         return MatchOperand_Success;
2081     }
2082   }
2083   return Res;
2084 }
2085 
2086 void AMDGPUAsmParser::cvtDPP_mod(MCInst &Inst, const OperandVector &Operands) {
2087   cvtDPP(Inst, Operands, true);
2088 }
2089 
2090 void AMDGPUAsmParser::cvtDPP_nomod(MCInst &Inst, const OperandVector &Operands) {
2091   cvtDPP(Inst, Operands, false);
2092 }
2093 
2094 void AMDGPUAsmParser::cvtDPP(MCInst &Inst, const OperandVector &Operands,
2095                              bool HasMods) {
2096   OptionalImmIndexMap OptionalIdx;
2097 
2098   unsigned I = 1;
2099   const MCInstrDesc &Desc = MII.get(Inst.getOpcode());
2100   for (unsigned J = 0; J < Desc.getNumDefs(); ++J) {
2101     ((AMDGPUOperand &)*Operands[I++]).addRegOperands(Inst, 1);
2102   }
2103 
2104   for (unsigned E = Operands.size(); I != E; ++I) {
2105     AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[I]);
2106     // Add the register arguments
2107     if (!HasMods && Op.isReg()) {
2108       Op.addRegOperands(Inst, 1);
2109     } else if (HasMods && Op.isRegOrImmWithInputMods()) {
2110       Op.addRegOrImmWithInputModsOperands(Inst, 2);
2111     } else if (Op.isDPPCtrl()) {
2112       Op.addImmOperands(Inst, 1);
2113     } else if (Op.isImm()) {
2114       // Handle optional arguments
2115       OptionalIdx[Op.getImmTy()] = I;
2116     } else {
2117       llvm_unreachable("Invalid operand type");
2118     }
2119   }
2120 
2121   // ToDo: fix default values for row_mask and bank_mask
2122   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyDppRowMask, 0xf);
2123   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyDppBankMask, 0xf);
2124   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyDppBoundCtrl);
2125 }
2126 
2127 
2128 /// Force static initialization.
2129 extern "C" void LLVMInitializeAMDGPUAsmParser() {
2130   RegisterMCAsmParser<AMDGPUAsmParser> A(TheAMDGPUTarget);
2131   RegisterMCAsmParser<AMDGPUAsmParser> B(TheGCNTarget);
2132 }
2133 
2134 #define GET_REGISTER_MATCHER
2135 #define GET_MATCHER_IMPLEMENTATION
2136 #include "AMDGPUGenAsmMatcher.inc"
2137