1 //===-- AMDGPUAsmParser.cpp - Parse SI asm to MCInst instructions ---------===//
2 //
3 //                     The LLVM Compiler Infrastructure
4 //
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
7 //
8 //===----------------------------------------------------------------------===//
9 
10 #include "AMDKernelCodeT.h"
11 #include "MCTargetDesc/AMDGPUMCTargetDesc.h"
12 #include "MCTargetDesc/AMDGPUTargetStreamer.h"
13 #include "SIDefines.h"
14 #include "Utils/AMDGPUBaseInfo.h"
15 #include "Utils/AMDKernelCodeTUtils.h"
16 #include "llvm/ADT/APFloat.h"
17 #include "llvm/ADT/STLExtras.h"
18 #include "llvm/ADT/SmallString.h"
19 #include "llvm/ADT/SmallVector.h"
20 #include "llvm/ADT/StringSwitch.h"
21 #include "llvm/ADT/Twine.h"
22 #include "llvm/MC/MCContext.h"
23 #include "llvm/MC/MCExpr.h"
24 #include "llvm/MC/MCInst.h"
25 #include "llvm/MC/MCInstrInfo.h"
26 #include "llvm/MC/MCParser/MCAsmLexer.h"
27 #include "llvm/MC/MCParser/MCAsmParser.h"
28 #include "llvm/MC/MCParser/MCParsedAsmOperand.h"
29 #include "llvm/MC/MCParser/MCTargetAsmParser.h"
30 #include "llvm/MC/MCRegisterInfo.h"
31 #include "llvm/MC/MCStreamer.h"
32 #include "llvm/MC/MCSubtargetInfo.h"
33 #include "llvm/MC/MCSymbolELF.h"
34 #include "llvm/Support/Debug.h"
35 #include "llvm/Support/ELF.h"
36 #include "llvm/Support/SourceMgr.h"
37 #include "llvm/Support/TargetRegistry.h"
38 #include "llvm/Support/raw_ostream.h"
39 
40 using namespace llvm;
41 
42 namespace {
43 
44 struct OptionalOperand;
45 
46 class AMDGPUOperand : public MCParsedAsmOperand {
47   enum KindTy {
48     Token,
49     Immediate,
50     Register,
51     Expression
52   } Kind;
53 
54   SMLoc StartLoc, EndLoc;
55 
56 public:
57   AMDGPUOperand(enum KindTy K) : MCParsedAsmOperand(), Kind(K) {}
58 
59   MCContext *Ctx;
60 
61   enum ImmTy {
62     ImmTyNone,
63     ImmTyDSOffset0,
64     ImmTyDSOffset1,
65     ImmTyGDS,
66     ImmTyOffset,
67     ImmTyGLC,
68     ImmTySLC,
69     ImmTyTFE,
70     ImmTyClamp,
71     ImmTyOMod,
72     ImmTyDppCtrl,
73     ImmTyDppRowMask,
74     ImmTyDppBankMask,
75     ImmTyDppBoundCtrl,
76     ImmTyDMask,
77     ImmTyUNorm,
78     ImmTyDA,
79     ImmTyR128,
80     ImmTyLWE,
81   };
82 
83   struct TokOp {
84     const char *Data;
85     unsigned Length;
86   };
87 
88   struct ImmOp {
89     bool IsFPImm;
90     ImmTy Type;
91     int64_t Val;
92     int Modifiers;
93   };
94 
95   struct RegOp {
96     unsigned RegNo;
97     int Modifiers;
98     const MCRegisterInfo *TRI;
99     const MCSubtargetInfo *STI;
100     bool IsForcedVOP3;
101   };
102 
103   union {
104     TokOp Tok;
105     ImmOp Imm;
106     RegOp Reg;
107     const MCExpr *Expr;
108   };
109 
110   void addImmOperands(MCInst &Inst, unsigned N) const {
111     Inst.addOperand(MCOperand::createImm(getImm()));
112   }
113 
114   StringRef getToken() const {
115     return StringRef(Tok.Data, Tok.Length);
116   }
117 
118   void addRegOperands(MCInst &Inst, unsigned N) const {
119     Inst.addOperand(MCOperand::createReg(AMDGPU::getMCReg(getReg(), *Reg.STI)));
120   }
121 
122   void addRegOrImmOperands(MCInst &Inst, unsigned N) const {
123     if (isRegKind())
124       addRegOperands(Inst, N);
125     else
126       addImmOperands(Inst, N);
127   }
128 
129   void addRegOrImmWithInputModsOperands(MCInst &Inst, unsigned N) const {
130     if (isRegKind()) {
131       Inst.addOperand(MCOperand::createImm(Reg.Modifiers));
132       addRegOperands(Inst, N);
133     } else {
134       Inst.addOperand(MCOperand::createImm(Imm.Modifiers));
135       addImmOperands(Inst, N);
136     }
137   }
138 
139   void addSoppBrTargetOperands(MCInst &Inst, unsigned N) const {
140     if (isImm())
141       addImmOperands(Inst, N);
142     else {
143       assert(isExpr());
144       Inst.addOperand(MCOperand::createExpr(Expr));
145     }
146   }
147 
148   bool defaultTokenHasSuffix() const {
149     StringRef Token(Tok.Data, Tok.Length);
150 
151     return Token.endswith("_e32") || Token.endswith("_e64") ||
152       Token.endswith("_dpp");
153   }
154 
155   bool isToken() const override {
156     return Kind == Token;
157   }
158 
159   bool isImm() const override {
160     return Kind == Immediate;
161   }
162 
163   bool isInlinableImm() const {
164     if (!isImm() || Imm.Type != AMDGPUOperand::ImmTyNone /* Only plain
165       immediates are inlinable (e.g. "clamp" attribute is not) */ )
166       return false;
167     // TODO: We should avoid using host float here. It would be better to
168     // check the float bit values which is what a few other places do.
169     // We've had bot failures before due to weird NaN support on mips hosts.
170     const float F = BitsToFloat(Imm.Val);
171     // TODO: Add 1/(2*pi) for VI
172     return (Imm.Val <= 64 && Imm.Val >= -16) ||
173            (F == 0.0 || F == 0.5 || F == -0.5 || F == 1.0 || F == -1.0 ||
174            F == 2.0 || F == -2.0 || F == 4.0 || F == -4.0);
175   }
176 
177   bool isDSOffset0() const {
178     assert(isImm());
179     return Imm.Type == ImmTyDSOffset0;
180   }
181 
182   bool isDSOffset1() const {
183     assert(isImm());
184     return Imm.Type == ImmTyDSOffset1;
185   }
186 
187   int64_t getImm() const {
188     return Imm.Val;
189   }
190 
191   enum ImmTy getImmTy() const {
192     assert(isImm());
193     return Imm.Type;
194   }
195 
196   bool isRegKind() const {
197     return Kind == Register;
198   }
199 
200   bool isReg() const override {
201     return Kind == Register && Reg.Modifiers == 0;
202   }
203 
204   bool isRegOrImmWithInputMods() const {
205     return Kind == Register || isInlinableImm();
206   }
207 
208   bool isImmTy(ImmTy ImmT) const {
209     return isImm() && Imm.Type == ImmT;
210   }
211 
212   bool isClamp() const {
213     return isImmTy(ImmTyClamp);
214   }
215 
216   bool isOMod() const {
217     return isImmTy(ImmTyOMod);
218   }
219 
220   bool isImmModifier() const {
221     return Kind == Immediate && Imm.Type != ImmTyNone;
222   }
223 
224   bool isDMask() const {
225     return isImmTy(ImmTyDMask);
226   }
227 
228   bool isUNorm() const { return isImmTy(ImmTyUNorm); }
229   bool isDA() const { return isImmTy(ImmTyDA); }
230   bool isR128() const { return isImmTy(ImmTyUNorm); }
231   bool isLWE() const { return isImmTy(ImmTyLWE); }
232 
233   bool isMod() const {
234     return isClamp() || isOMod();
235   }
236 
237   bool isGDS() const { return isImmTy(ImmTyGDS); }
238   bool isGLC() const { return isImmTy(ImmTyGLC); }
239   bool isSLC() const { return isImmTy(ImmTySLC); }
240   bool isTFE() const { return isImmTy(ImmTyTFE); }
241 
242   bool isBankMask() const {
243     return isImmTy(ImmTyDppBankMask);
244   }
245 
246   bool isRowMask() const {
247     return isImmTy(ImmTyDppRowMask);
248   }
249 
250   bool isBoundCtrl() const {
251     return isImmTy(ImmTyDppBoundCtrl);
252   }
253 
254   void setModifiers(unsigned Mods) {
255     assert(isReg() || (isImm() && Imm.Modifiers == 0));
256     if (isReg())
257       Reg.Modifiers = Mods;
258     else
259       Imm.Modifiers = Mods;
260   }
261 
262   bool hasModifiers() const {
263     assert(isRegKind() || isImm());
264     return isRegKind() ? Reg.Modifiers != 0 : Imm.Modifiers != 0;
265   }
266 
267   unsigned getReg() const override {
268     return Reg.RegNo;
269   }
270 
271   bool isRegOrImm() const {
272     return isReg() || isImm();
273   }
274 
275   bool isRegClass(unsigned RCID) const {
276     return isReg() && Reg.TRI->getRegClass(RCID).contains(getReg());
277   }
278 
279   bool isSCSrc32() const {
280     return isInlinableImm() || isRegClass(AMDGPU::SReg_32RegClassID);
281   }
282 
283   bool isSCSrc64() const {
284     return isInlinableImm() || isRegClass(AMDGPU::SReg_64RegClassID);
285   }
286 
287   bool isSSrc32() const {
288     return isImm() || isSCSrc32();
289   }
290 
291   bool isSSrc64() const {
292     // TODO: Find out how SALU supports extension of 32-bit literals to 64 bits.
293     // See isVSrc64().
294     return isImm() || isSCSrc64();
295   }
296 
297   bool isVCSrc32() const {
298     return isInlinableImm() || isRegClass(AMDGPU::VS_32RegClassID);
299   }
300 
301   bool isVCSrc64() const {
302     return isInlinableImm() || isRegClass(AMDGPU::VS_64RegClassID);
303   }
304 
305   bool isVSrc32() const {
306     return isImm() || isVCSrc32();
307   }
308 
309   bool isVSrc64() const {
310     // TODO: Check if the 64-bit value (coming from assembly source) can be
311     // narrowed to 32 bits (in the instruction stream). That require knowledge
312     // of instruction type (unsigned/signed, floating or "untyped"/B64),
313     // see [AMD GCN3 ISA 6.3.1].
314     // TODO: How 64-bit values are formed from 32-bit literals in _B64 insns?
315     return isImm() || isVCSrc64();
316   }
317 
318   bool isMem() const override {
319     return false;
320   }
321 
322   bool isExpr() const {
323     return Kind == Expression;
324   }
325 
326   bool isSoppBrTarget() const {
327     return isExpr() || isImm();
328   }
329 
330   SMLoc getStartLoc() const override {
331     return StartLoc;
332   }
333 
334   SMLoc getEndLoc() const override {
335     return EndLoc;
336   }
337 
338   void print(raw_ostream &OS) const override {
339     switch (Kind) {
340     case Register:
341       OS << "<register " << getReg() << " mods: " << Reg.Modifiers << '>';
342       break;
343     case Immediate:
344       if (Imm.Type != AMDGPUOperand::ImmTyNone)
345         OS << getImm();
346       else
347         OS << '<' << getImm() << " mods: " << Imm.Modifiers << '>';
348       break;
349     case Token:
350       OS << '\'' << getToken() << '\'';
351       break;
352     case Expression:
353       OS << "<expr " << *Expr << '>';
354       break;
355     }
356   }
357 
358   static std::unique_ptr<AMDGPUOperand> CreateImm(int64_t Val, SMLoc Loc,
359                                                   enum ImmTy Type = ImmTyNone,
360                                                   bool IsFPImm = false) {
361     auto Op = llvm::make_unique<AMDGPUOperand>(Immediate);
362     Op->Imm.Val = Val;
363     Op->Imm.IsFPImm = IsFPImm;
364     Op->Imm.Type = Type;
365     Op->Imm.Modifiers = 0;
366     Op->StartLoc = Loc;
367     Op->EndLoc = Loc;
368     return Op;
369   }
370 
371   static std::unique_ptr<AMDGPUOperand> CreateToken(StringRef Str, SMLoc Loc,
372                                            bool HasExplicitEncodingSize = true) {
373     auto Res = llvm::make_unique<AMDGPUOperand>(Token);
374     Res->Tok.Data = Str.data();
375     Res->Tok.Length = Str.size();
376     Res->StartLoc = Loc;
377     Res->EndLoc = Loc;
378     return Res;
379   }
380 
381   static std::unique_ptr<AMDGPUOperand> CreateReg(unsigned RegNo, SMLoc S,
382                                                   SMLoc E,
383                                                   const MCRegisterInfo *TRI,
384                                                   const MCSubtargetInfo *STI,
385                                                   bool ForceVOP3) {
386     auto Op = llvm::make_unique<AMDGPUOperand>(Register);
387     Op->Reg.RegNo = RegNo;
388     Op->Reg.TRI = TRI;
389     Op->Reg.STI = STI;
390     Op->Reg.Modifiers = 0;
391     Op->Reg.IsForcedVOP3 = ForceVOP3;
392     Op->StartLoc = S;
393     Op->EndLoc = E;
394     return Op;
395   }
396 
397   static std::unique_ptr<AMDGPUOperand> CreateExpr(const class MCExpr *Expr, SMLoc S) {
398     auto Op = llvm::make_unique<AMDGPUOperand>(Expression);
399     Op->Expr = Expr;
400     Op->StartLoc = S;
401     Op->EndLoc = S;
402     return Op;
403   }
404 
405   bool isDSOffset() const;
406   bool isDSOffset01() const;
407   bool isSWaitCnt() const;
408   bool isMubufOffset() const;
409   bool isSMRDOffset() const;
410   bool isSMRDLiteralOffset() const;
411   bool isDPPCtrl() const;
412 };
413 
414 class AMDGPUAsmParser : public MCTargetAsmParser {
415   const MCInstrInfo &MII;
416   MCAsmParser &Parser;
417 
418   unsigned ForcedEncodingSize;
419 
420   bool isSI() const {
421     return AMDGPU::isSI(getSTI());
422   }
423 
424   bool isCI() const {
425     return AMDGPU::isCI(getSTI());
426   }
427 
428   bool isVI() const {
429     return AMDGPU::isVI(getSTI());
430   }
431 
432   bool hasSGPR102_SGPR103() const {
433     return !isVI();
434   }
435 
436   /// @name Auto-generated Match Functions
437   /// {
438 
439 #define GET_ASSEMBLER_HEADER
440 #include "AMDGPUGenAsmMatcher.inc"
441 
442   /// }
443 
444 private:
445   bool ParseDirectiveMajorMinor(uint32_t &Major, uint32_t &Minor);
446   bool ParseDirectiveHSACodeObjectVersion();
447   bool ParseDirectiveHSACodeObjectISA();
448   bool ParseAMDKernelCodeTValue(StringRef ID, amd_kernel_code_t &Header);
449   bool ParseDirectiveAMDKernelCodeT();
450   bool ParseSectionDirectiveHSAText();
451   bool subtargetHasRegister(const MCRegisterInfo &MRI, unsigned RegNo) const;
452   bool ParseDirectiveAMDGPUHsaKernel();
453   bool ParseDirectiveAMDGPUHsaModuleGlobal();
454   bool ParseDirectiveAMDGPUHsaProgramGlobal();
455   bool ParseSectionDirectiveHSADataGlobalAgent();
456   bool ParseSectionDirectiveHSADataGlobalProgram();
457   bool ParseSectionDirectiveHSARodataReadonlyAgent();
458 
459 public:
460   enum AMDGPUMatchResultTy {
461     Match_PreferE32 = FIRST_TARGET_MATCH_RESULT_TY
462   };
463 
464   AMDGPUAsmParser(const MCSubtargetInfo &STI, MCAsmParser &_Parser,
465                const MCInstrInfo &MII,
466                const MCTargetOptions &Options)
467       : MCTargetAsmParser(Options, STI), MII(MII), Parser(_Parser),
468         ForcedEncodingSize(0) {
469     MCAsmParserExtension::Initialize(Parser);
470 
471     if (getSTI().getFeatureBits().none()) {
472       // Set default features.
473       copySTI().ToggleFeature("SOUTHERN_ISLANDS");
474     }
475 
476     setAvailableFeatures(ComputeAvailableFeatures(getSTI().getFeatureBits()));
477   }
478 
479   AMDGPUTargetStreamer &getTargetStreamer() {
480     MCTargetStreamer &TS = *getParser().getStreamer().getTargetStreamer();
481     return static_cast<AMDGPUTargetStreamer &>(TS);
482   }
483 
484   unsigned getForcedEncodingSize() const {
485     return ForcedEncodingSize;
486   }
487 
488   void setForcedEncodingSize(unsigned Size) {
489     ForcedEncodingSize = Size;
490   }
491 
492   bool isForcedVOP3() const {
493     return ForcedEncodingSize == 64;
494   }
495 
496   std::unique_ptr<AMDGPUOperand> parseRegister();
497   bool ParseRegister(unsigned &RegNo, SMLoc &StartLoc, SMLoc &EndLoc) override;
498   unsigned checkTargetMatchPredicate(MCInst &Inst) override;
499   bool MatchAndEmitInstruction(SMLoc IDLoc, unsigned &Opcode,
500                                OperandVector &Operands, MCStreamer &Out,
501                                uint64_t &ErrorInfo,
502                                bool MatchingInlineAsm) override;
503   bool ParseDirective(AsmToken DirectiveID) override;
504   OperandMatchResultTy parseOperand(OperandVector &Operands, StringRef Mnemonic);
505   bool ParseInstruction(ParseInstructionInfo &Info, StringRef Name,
506                         SMLoc NameLoc, OperandVector &Operands) override;
507 
508   OperandMatchResultTy parseIntWithPrefix(const char *Prefix, int64_t &Int,
509                                           int64_t Default = 0);
510   OperandMatchResultTy parseIntWithPrefix(const char *Prefix,
511                                           OperandVector &Operands,
512                                           enum AMDGPUOperand::ImmTy ImmTy =
513                                                       AMDGPUOperand::ImmTyNone);
514   OperandMatchResultTy parseNamedBit(const char *Name, OperandVector &Operands,
515                                      enum AMDGPUOperand::ImmTy ImmTy =
516                                                       AMDGPUOperand::ImmTyNone);
517   OperandMatchResultTy parseOptionalOps(
518                                    const ArrayRef<OptionalOperand> &OptionalOps,
519                                    OperandVector &Operands);
520 
521 
522   void cvtDSOffset01(MCInst &Inst, const OperandVector &Operands);
523   void cvtDS(MCInst &Inst, const OperandVector &Operands);
524   OperandMatchResultTy parseDSOptionalOps(OperandVector &Operands);
525   OperandMatchResultTy parseDSOff01OptionalOps(OperandVector &Operands);
526   OperandMatchResultTy parseDSOffsetOptional(OperandVector &Operands);
527 
528   bool parseCnt(int64_t &IntVal);
529   OperandMatchResultTy parseSWaitCntOps(OperandVector &Operands);
530   OperandMatchResultTy parseSOppBrTarget(OperandVector &Operands);
531 
532   OperandMatchResultTy parseFlatOptionalOps(OperandVector &Operands);
533   OperandMatchResultTy parseFlatAtomicOptionalOps(OperandVector &Operands);
534   void cvtFlat(MCInst &Inst, const OperandVector &Operands);
535   void cvtFlatAtomic(MCInst &Inst, const OperandVector &Operands);
536 
537   void cvtMubuf(MCInst &Inst, const OperandVector &Operands);
538   OperandMatchResultTy parseOffset(OperandVector &Operands);
539   OperandMatchResultTy parseMubufOptionalOps(OperandVector &Operands);
540   OperandMatchResultTy parseGLC(OperandVector &Operands);
541   OperandMatchResultTy parseSLC(OperandVector &Operands);
542   OperandMatchResultTy parseTFE(OperandVector &Operands);
543 
544   OperandMatchResultTy parseDMask(OperandVector &Operands);
545   OperandMatchResultTy parseUNorm(OperandVector &Operands);
546   OperandMatchResultTy parseDA(OperandVector &Operands);
547   OperandMatchResultTy parseR128(OperandVector &Operands);
548   OperandMatchResultTy parseLWE(OperandVector &Operands);
549 
550   void cvtId(MCInst &Inst, const OperandVector &Operands);
551   void cvtVOP3_2_mod(MCInst &Inst, const OperandVector &Operands);
552   void cvtVOP3_2_nomod(MCInst &Inst, const OperandVector &Operands);
553   void cvtVOP3_only(MCInst &Inst, const OperandVector &Operands);
554   void cvtVOP3(MCInst &Inst, const OperandVector &Operands);
555 
556   void cvtMIMG(MCInst &Inst, const OperandVector &Operands);
557   void cvtMIMGAtomic(MCInst &Inst, const OperandVector &Operands);
558   OperandMatchResultTy parseVOP3OptionalOps(OperandVector &Operands);
559 
560   OperandMatchResultTy parseDPPCtrlOps(OperandVector &Operands);
561   OperandMatchResultTy parseDPPOptionalOps(OperandVector &Operands);
562   void cvtDPP_mod(MCInst &Inst, const OperandVector &Operands);
563   void cvtDPP_nomod(MCInst &Inst, const OperandVector &Operands);
564   void cvtDPP(MCInst &Inst, const OperandVector &Operands, bool HasMods);
565 };
566 
567 struct OptionalOperand {
568   const char *Name;
569   AMDGPUOperand::ImmTy Type;
570   bool IsBit;
571   int64_t Default;
572   bool (*ConvertResult)(int64_t&);
573 };
574 
575 }
576 
577 enum  RegisterKind { IS_VGPR, IS_SGPR, IS_TTMP };
578 
579 static int getRegClass(RegisterKind Is, unsigned RegWidth) {
580   if (Is == IS_VGPR) {
581     switch (RegWidth) {
582       default: return -1;
583       case 1: return AMDGPU::VGPR_32RegClassID;
584       case 2: return AMDGPU::VReg_64RegClassID;
585       case 3: return AMDGPU::VReg_96RegClassID;
586       case 4: return AMDGPU::VReg_128RegClassID;
587       case 8: return AMDGPU::VReg_256RegClassID;
588       case 16: return AMDGPU::VReg_512RegClassID;
589     }
590   } else if (Is == IS_TTMP) {
591     switch (RegWidth) {
592       default: return -1;
593       case 1: return AMDGPU::TTMP_32RegClassID;
594       case 2: return AMDGPU::TTMP_64RegClassID;
595     }
596   } else if (Is == IS_SGPR) {
597     switch (RegWidth) {
598       default: return -1;
599       case 1: return AMDGPU::SGPR_32RegClassID;
600       case 2: return AMDGPU::SGPR_64RegClassID;
601       case 4: return AMDGPU::SReg_128RegClassID;
602       case 8: return AMDGPU::SReg_256RegClassID;
603       case 16: return AMDGPU::SReg_512RegClassID;
604     }
605   }
606   return -1;
607 }
608 
609 static unsigned getRegForName(StringRef RegName) {
610 
611   return StringSwitch<unsigned>(RegName)
612     .Case("exec", AMDGPU::EXEC)
613     .Case("vcc", AMDGPU::VCC)
614     .Case("flat_scratch", AMDGPU::FLAT_SCR)
615     .Case("m0", AMDGPU::M0)
616     .Case("scc", AMDGPU::SCC)
617     .Case("flat_scratch_lo", AMDGPU::FLAT_SCR_LO)
618     .Case("flat_scratch_hi", AMDGPU::FLAT_SCR_HI)
619     .Case("vcc_lo", AMDGPU::VCC_LO)
620     .Case("vcc_hi", AMDGPU::VCC_HI)
621     .Case("exec_lo", AMDGPU::EXEC_LO)
622     .Case("exec_hi", AMDGPU::EXEC_HI)
623     .Case("tma_lo", AMDGPU::TMA_LO)
624     .Case("tma_hi", AMDGPU::TMA_HI)
625     .Case("tba_lo", AMDGPU::TBA_LO)
626     .Case("tba_hi", AMDGPU::TBA_HI)
627     .Default(0);
628 }
629 
630 bool AMDGPUAsmParser::ParseRegister(unsigned &RegNo, SMLoc &StartLoc, SMLoc &EndLoc) {
631   auto R = parseRegister();
632   if (!R) return true;
633   assert(R->isReg());
634   RegNo = R->getReg();
635   StartLoc = R->getStartLoc();
636   EndLoc = R->getEndLoc();
637   return false;
638 }
639 
640 std::unique_ptr<AMDGPUOperand> AMDGPUAsmParser::parseRegister() {
641   const AsmToken &Tok = Parser.getTok();
642   SMLoc StartLoc = Tok.getLoc();
643   SMLoc EndLoc = Tok.getEndLoc();
644   const MCRegisterInfo *TRI = getContext().getRegisterInfo();
645 
646   StringRef RegName = Tok.getString();
647   unsigned RegNo = getRegForName(RegName);
648 
649   if (RegNo) {
650     Parser.Lex();
651     if (!subtargetHasRegister(*TRI, RegNo))
652       return nullptr;
653     return AMDGPUOperand::CreateReg(RegNo, StartLoc, EndLoc,
654                                     TRI, &getSTI(), false);
655   }
656 
657   // Match vgprs, sgprs and ttmps
658   if (RegName[0] != 's' && RegName[0] != 'v' && !RegName.startswith("ttmp"))
659     return nullptr;
660 
661   const RegisterKind Is = RegName[0] == 'v' ? IS_VGPR : RegName[0] == 's' ? IS_SGPR : IS_TTMP;
662   unsigned RegWidth;
663   unsigned RegIndexInClass;
664   if (RegName.size() > (Is == IS_TTMP ? strlen("ttmp") : 1) ) {
665     // We have a single 32-bit register. Syntax: vXX
666     RegWidth = 1;
667     if (RegName.substr(Is == IS_TTMP ? strlen("ttmp") : 1).getAsInteger(10, RegIndexInClass))
668       return nullptr;
669     Parser.Lex();
670   } else {
671     // We have a register greater than 32-bits (a range of single registers). Syntax: v[XX:YY]
672 
673     int64_t RegLo, RegHi;
674     Parser.Lex();
675     if (getLexer().isNot(AsmToken::LBrac))
676       return nullptr;
677 
678     Parser.Lex();
679     if (getParser().parseAbsoluteExpression(RegLo))
680       return nullptr;
681 
682     if (getLexer().isNot(AsmToken::Colon))
683       return nullptr;
684 
685     Parser.Lex();
686     if (getParser().parseAbsoluteExpression(RegHi))
687       return nullptr;
688 
689     if (getLexer().isNot(AsmToken::RBrac))
690       return nullptr;
691 
692     Parser.Lex();
693     RegWidth = (RegHi - RegLo) + 1;
694     if (Is == IS_VGPR) {
695       // VGPR registers aren't aligned.
696       RegIndexInClass = RegLo;
697     } else {
698       // SGPR and TTMP registers must be are aligned. Max required alignment is 4 dwords.
699       unsigned Size = std::min(RegWidth, 4u);
700       if (RegLo % Size != 0)
701         return nullptr;
702 
703       RegIndexInClass = RegLo / Size;
704     }
705   }
706 
707   int RCID = getRegClass(Is, RegWidth);
708   if (RCID == -1)
709     return nullptr;
710 
711   const MCRegisterClass RC = TRI->getRegClass(RCID);
712   if (RegIndexInClass >= RC.getNumRegs())
713     return nullptr;
714 
715   RegNo = RC.getRegister(RegIndexInClass);
716   if (!subtargetHasRegister(*TRI, RegNo))
717     return nullptr;
718 
719   return AMDGPUOperand::CreateReg(RegNo, StartLoc, EndLoc,
720                                   TRI, &getSTI(), false);
721 }
722 
723 unsigned AMDGPUAsmParser::checkTargetMatchPredicate(MCInst &Inst) {
724 
725   uint64_t TSFlags = MII.get(Inst.getOpcode()).TSFlags;
726 
727   if ((getForcedEncodingSize() == 32 && (TSFlags & SIInstrFlags::VOP3)) ||
728       (getForcedEncodingSize() == 64 && !(TSFlags & SIInstrFlags::VOP3)))
729     return Match_InvalidOperand;
730 
731   if ((TSFlags & SIInstrFlags::VOP3) &&
732       (TSFlags & SIInstrFlags::VOPAsmPrefer32Bit) &&
733       getForcedEncodingSize() != 64)
734     return Match_PreferE32;
735 
736   return Match_Success;
737 }
738 
739 
740 bool AMDGPUAsmParser::MatchAndEmitInstruction(SMLoc IDLoc, unsigned &Opcode,
741                                               OperandVector &Operands,
742                                               MCStreamer &Out,
743                                               uint64_t &ErrorInfo,
744                                               bool MatchingInlineAsm) {
745   MCInst Inst;
746 
747   switch (MatchInstructionImpl(Operands, Inst, ErrorInfo, MatchingInlineAsm)) {
748     default: break;
749     case Match_Success:
750       Inst.setLoc(IDLoc);
751       Out.EmitInstruction(Inst, getSTI());
752       return false;
753     case Match_MissingFeature:
754       return Error(IDLoc, "instruction not supported on this GPU");
755 
756     case Match_MnemonicFail:
757       return Error(IDLoc, "unrecognized instruction mnemonic");
758 
759     case Match_InvalidOperand: {
760       SMLoc ErrorLoc = IDLoc;
761       if (ErrorInfo != ~0ULL) {
762         if (ErrorInfo >= Operands.size()) {
763           return Error(IDLoc, "too few operands for instruction");
764         }
765         ErrorLoc = ((AMDGPUOperand &)*Operands[ErrorInfo]).getStartLoc();
766         if (ErrorLoc == SMLoc())
767           ErrorLoc = IDLoc;
768       }
769       return Error(ErrorLoc, "invalid operand for instruction");
770     }
771     case Match_PreferE32:
772       return Error(IDLoc, "internal error: instruction without _e64 suffix "
773                           "should be encoded as e32");
774   }
775   llvm_unreachable("Implement any new match types added!");
776 }
777 
778 bool AMDGPUAsmParser::ParseDirectiveMajorMinor(uint32_t &Major,
779                                                uint32_t &Minor) {
780   if (getLexer().isNot(AsmToken::Integer))
781     return TokError("invalid major version");
782 
783   Major = getLexer().getTok().getIntVal();
784   Lex();
785 
786   if (getLexer().isNot(AsmToken::Comma))
787     return TokError("minor version number required, comma expected");
788   Lex();
789 
790   if (getLexer().isNot(AsmToken::Integer))
791     return TokError("invalid minor version");
792 
793   Minor = getLexer().getTok().getIntVal();
794   Lex();
795 
796   return false;
797 }
798 
799 bool AMDGPUAsmParser::ParseDirectiveHSACodeObjectVersion() {
800 
801   uint32_t Major;
802   uint32_t Minor;
803 
804   if (ParseDirectiveMajorMinor(Major, Minor))
805     return true;
806 
807   getTargetStreamer().EmitDirectiveHSACodeObjectVersion(Major, Minor);
808   return false;
809 }
810 
811 bool AMDGPUAsmParser::ParseDirectiveHSACodeObjectISA() {
812 
813   uint32_t Major;
814   uint32_t Minor;
815   uint32_t Stepping;
816   StringRef VendorName;
817   StringRef ArchName;
818 
819   // If this directive has no arguments, then use the ISA version for the
820   // targeted GPU.
821   if (getLexer().is(AsmToken::EndOfStatement)) {
822     AMDGPU::IsaVersion Isa = AMDGPU::getIsaVersion(getSTI().getFeatureBits());
823     getTargetStreamer().EmitDirectiveHSACodeObjectISA(Isa.Major, Isa.Minor,
824                                                       Isa.Stepping,
825                                                       "AMD", "AMDGPU");
826     return false;
827   }
828 
829 
830   if (ParseDirectiveMajorMinor(Major, Minor))
831     return true;
832 
833   if (getLexer().isNot(AsmToken::Comma))
834     return TokError("stepping version number required, comma expected");
835   Lex();
836 
837   if (getLexer().isNot(AsmToken::Integer))
838     return TokError("invalid stepping version");
839 
840   Stepping = getLexer().getTok().getIntVal();
841   Lex();
842 
843   if (getLexer().isNot(AsmToken::Comma))
844     return TokError("vendor name required, comma expected");
845   Lex();
846 
847   if (getLexer().isNot(AsmToken::String))
848     return TokError("invalid vendor name");
849 
850   VendorName = getLexer().getTok().getStringContents();
851   Lex();
852 
853   if (getLexer().isNot(AsmToken::Comma))
854     return TokError("arch name required, comma expected");
855   Lex();
856 
857   if (getLexer().isNot(AsmToken::String))
858     return TokError("invalid arch name");
859 
860   ArchName = getLexer().getTok().getStringContents();
861   Lex();
862 
863   getTargetStreamer().EmitDirectiveHSACodeObjectISA(Major, Minor, Stepping,
864                                                     VendorName, ArchName);
865   return false;
866 }
867 
868 bool AMDGPUAsmParser::ParseAMDKernelCodeTValue(StringRef ID,
869                                                amd_kernel_code_t &Header) {
870   SmallString<40> ErrStr;
871   raw_svector_ostream Err(ErrStr);
872   if (!parseAmdKernelCodeField(ID, getLexer(), Header, Err)) {
873     return TokError(Err.str());
874   }
875   Lex();
876   return false;
877 }
878 
879 bool AMDGPUAsmParser::ParseDirectiveAMDKernelCodeT() {
880 
881   amd_kernel_code_t Header;
882   AMDGPU::initDefaultAMDKernelCodeT(Header, getSTI().getFeatureBits());
883 
884   while (true) {
885 
886     if (getLexer().isNot(AsmToken::EndOfStatement))
887       return TokError("amd_kernel_code_t values must begin on a new line");
888 
889     // Lex EndOfStatement.  This is in a while loop, because lexing a comment
890     // will set the current token to EndOfStatement.
891     while(getLexer().is(AsmToken::EndOfStatement))
892       Lex();
893 
894     if (getLexer().isNot(AsmToken::Identifier))
895       return TokError("expected value identifier or .end_amd_kernel_code_t");
896 
897     StringRef ID = getLexer().getTok().getIdentifier();
898     Lex();
899 
900     if (ID == ".end_amd_kernel_code_t")
901       break;
902 
903     if (ParseAMDKernelCodeTValue(ID, Header))
904       return true;
905   }
906 
907   getTargetStreamer().EmitAMDKernelCodeT(Header);
908 
909   return false;
910 }
911 
912 bool AMDGPUAsmParser::ParseSectionDirectiveHSAText() {
913   getParser().getStreamer().SwitchSection(
914       AMDGPU::getHSATextSection(getContext()));
915   return false;
916 }
917 
918 bool AMDGPUAsmParser::ParseDirectiveAMDGPUHsaKernel() {
919   if (getLexer().isNot(AsmToken::Identifier))
920     return TokError("expected symbol name");
921 
922   StringRef KernelName = Parser.getTok().getString();
923 
924   getTargetStreamer().EmitAMDGPUSymbolType(KernelName,
925                                            ELF::STT_AMDGPU_HSA_KERNEL);
926   Lex();
927   return false;
928 }
929 
930 bool AMDGPUAsmParser::ParseDirectiveAMDGPUHsaModuleGlobal() {
931   if (getLexer().isNot(AsmToken::Identifier))
932     return TokError("expected symbol name");
933 
934   StringRef GlobalName = Parser.getTok().getIdentifier();
935 
936   getTargetStreamer().EmitAMDGPUHsaModuleScopeGlobal(GlobalName);
937   Lex();
938   return false;
939 }
940 
941 bool AMDGPUAsmParser::ParseDirectiveAMDGPUHsaProgramGlobal() {
942   if (getLexer().isNot(AsmToken::Identifier))
943     return TokError("expected symbol name");
944 
945   StringRef GlobalName = Parser.getTok().getIdentifier();
946 
947   getTargetStreamer().EmitAMDGPUHsaProgramScopeGlobal(GlobalName);
948   Lex();
949   return false;
950 }
951 
952 bool AMDGPUAsmParser::ParseSectionDirectiveHSADataGlobalAgent() {
953   getParser().getStreamer().SwitchSection(
954       AMDGPU::getHSADataGlobalAgentSection(getContext()));
955   return false;
956 }
957 
958 bool AMDGPUAsmParser::ParseSectionDirectiveHSADataGlobalProgram() {
959   getParser().getStreamer().SwitchSection(
960       AMDGPU::getHSADataGlobalProgramSection(getContext()));
961   return false;
962 }
963 
964 bool AMDGPUAsmParser::ParseSectionDirectiveHSARodataReadonlyAgent() {
965   getParser().getStreamer().SwitchSection(
966       AMDGPU::getHSARodataReadonlyAgentSection(getContext()));
967   return false;
968 }
969 
970 bool AMDGPUAsmParser::ParseDirective(AsmToken DirectiveID) {
971   StringRef IDVal = DirectiveID.getString();
972 
973   if (IDVal == ".hsa_code_object_version")
974     return ParseDirectiveHSACodeObjectVersion();
975 
976   if (IDVal == ".hsa_code_object_isa")
977     return ParseDirectiveHSACodeObjectISA();
978 
979   if (IDVal == ".amd_kernel_code_t")
980     return ParseDirectiveAMDKernelCodeT();
981 
982   if (IDVal == ".hsatext" || IDVal == ".text")
983     return ParseSectionDirectiveHSAText();
984 
985   if (IDVal == ".amdgpu_hsa_kernel")
986     return ParseDirectiveAMDGPUHsaKernel();
987 
988   if (IDVal == ".amdgpu_hsa_module_global")
989     return ParseDirectiveAMDGPUHsaModuleGlobal();
990 
991   if (IDVal == ".amdgpu_hsa_program_global")
992     return ParseDirectiveAMDGPUHsaProgramGlobal();
993 
994   if (IDVal == ".hsadata_global_agent")
995     return ParseSectionDirectiveHSADataGlobalAgent();
996 
997   if (IDVal == ".hsadata_global_program")
998     return ParseSectionDirectiveHSADataGlobalProgram();
999 
1000   if (IDVal == ".hsarodata_readonly_agent")
1001     return ParseSectionDirectiveHSARodataReadonlyAgent();
1002 
1003   return true;
1004 }
1005 
1006 bool AMDGPUAsmParser::subtargetHasRegister(const MCRegisterInfo &MRI,
1007                                            unsigned RegNo) const {
1008   if (isCI())
1009     return true;
1010 
1011   if (isSI()) {
1012     // No flat_scr
1013     switch (RegNo) {
1014     case AMDGPU::FLAT_SCR:
1015     case AMDGPU::FLAT_SCR_LO:
1016     case AMDGPU::FLAT_SCR_HI:
1017       return false;
1018     default:
1019       return true;
1020     }
1021   }
1022 
1023   // VI only has 102 SGPRs, so make sure we aren't trying to use the 2 more that
1024   // SI/CI have.
1025   for (MCRegAliasIterator R(AMDGPU::SGPR102_SGPR103, &MRI, true);
1026        R.isValid(); ++R) {
1027     if (*R == RegNo)
1028       return false;
1029   }
1030 
1031   return true;
1032 }
1033 
1034 static bool operandsHaveModifiers(const OperandVector &Operands) {
1035 
1036   for (unsigned i = 0, e = Operands.size(); i != e; ++i) {
1037     const AMDGPUOperand &Op = ((AMDGPUOperand&)*Operands[i]);
1038     if (Op.isRegKind() && Op.hasModifiers())
1039       return true;
1040     if (Op.isImm() && Op.hasModifiers())
1041       return true;
1042     if (Op.isImm() && (Op.getImmTy() == AMDGPUOperand::ImmTyOMod ||
1043                        Op.getImmTy() == AMDGPUOperand::ImmTyClamp))
1044       return true;
1045   }
1046   return false;
1047 }
1048 
1049 AMDGPUAsmParser::OperandMatchResultTy
1050 AMDGPUAsmParser::parseOperand(OperandVector &Operands, StringRef Mnemonic) {
1051 
1052   // Try to parse with a custom parser
1053   OperandMatchResultTy ResTy = MatchOperandParserImpl(Operands, Mnemonic);
1054 
1055   // If we successfully parsed the operand or if there as an error parsing,
1056   // we are done.
1057   //
1058   // If we are parsing after we reach EndOfStatement then this means we
1059   // are appending default values to the Operands list.  This is only done
1060   // by custom parser, so we shouldn't continue on to the generic parsing.
1061   if (ResTy == MatchOperand_Success || ResTy == MatchOperand_ParseFail||
1062       getLexer().is(AsmToken::EndOfStatement))
1063     return ResTy;
1064 
1065   bool Negate = false, Abs = false, Abs2 = false;
1066 
1067   if (getLexer().getKind()== AsmToken::Minus) {
1068     Parser.Lex();
1069     Negate = true;
1070   }
1071 
1072   if (getLexer().getKind() == AsmToken::Identifier && Parser.getTok().getString() == "abs") {
1073     Parser.Lex();
1074     Abs2 = true;
1075     if (getLexer().isNot(AsmToken::LParen)) {
1076       Error(Parser.getTok().getLoc(), "expected left paren after abs");
1077       return MatchOperand_ParseFail;
1078     }
1079     Parser.Lex();
1080   }
1081 
1082   if (getLexer().getKind() == AsmToken::Pipe) {
1083     Parser.Lex();
1084     Abs = true;
1085   }
1086 
1087   switch(getLexer().getKind()) {
1088     case AsmToken::Integer: {
1089       SMLoc S = Parser.getTok().getLoc();
1090       int64_t IntVal;
1091       if (getParser().parseAbsoluteExpression(IntVal))
1092         return MatchOperand_ParseFail;
1093       if (!isInt<32>(IntVal) && !isUInt<32>(IntVal)) {
1094         Error(S, "invalid immediate: only 32-bit values are legal");
1095         return MatchOperand_ParseFail;
1096       }
1097 
1098       if (Negate)
1099         IntVal *= -1;
1100       Operands.push_back(AMDGPUOperand::CreateImm(IntVal, S));
1101       return MatchOperand_Success;
1102     }
1103     case AsmToken::Real: {
1104       // FIXME: We should emit an error if a double precisions floating-point
1105       // value is used.  I'm not sure the best way to detect this.
1106       SMLoc S = Parser.getTok().getLoc();
1107       int64_t IntVal;
1108       if (getParser().parseAbsoluteExpression(IntVal))
1109         return MatchOperand_ParseFail;
1110 
1111       APFloat F((float)BitsToDouble(IntVal));
1112       if (Negate)
1113         F.changeSign();
1114       Operands.push_back(
1115           AMDGPUOperand::CreateImm(F.bitcastToAPInt().getZExtValue(), S));
1116       return MatchOperand_Success;
1117     }
1118     case AsmToken::Identifier: {
1119       if (auto R = parseRegister()) {
1120         unsigned Modifiers = 0;
1121 
1122         if (Negate)
1123           Modifiers |= 0x1;
1124 
1125         if (Abs) {
1126           if (getLexer().getKind() != AsmToken::Pipe)
1127             return MatchOperand_ParseFail;
1128           Parser.Lex();
1129           Modifiers |= 0x2;
1130         }
1131         if (Abs2) {
1132           if (getLexer().isNot(AsmToken::RParen)) {
1133             return MatchOperand_ParseFail;
1134           }
1135           Parser.Lex();
1136           Modifiers |= 0x2;
1137         }
1138         assert(R->isReg());
1139         R->Reg.IsForcedVOP3 = isForcedVOP3();
1140         if (Modifiers) {
1141           R->setModifiers(Modifiers);
1142         }
1143         Operands.push_back(std::move(R));
1144       } else {
1145         ResTy = parseVOP3OptionalOps(Operands);
1146         if (ResTy == MatchOperand_NoMatch) {
1147           const auto &Tok = Parser.getTok();
1148           Operands.push_back(AMDGPUOperand::CreateToken(Tok.getString(),
1149                                                         Tok.getLoc()));
1150           Parser.Lex();
1151         }
1152       }
1153       return MatchOperand_Success;
1154     }
1155     default:
1156       return MatchOperand_NoMatch;
1157   }
1158 }
1159 
1160 bool AMDGPUAsmParser::ParseInstruction(ParseInstructionInfo &Info,
1161                                        StringRef Name,
1162                                        SMLoc NameLoc, OperandVector &Operands) {
1163 
1164   // Clear any forced encodings from the previous instruction.
1165   setForcedEncodingSize(0);
1166 
1167   if (Name.endswith("_e64"))
1168     setForcedEncodingSize(64);
1169   else if (Name.endswith("_e32"))
1170     setForcedEncodingSize(32);
1171 
1172   // Add the instruction mnemonic
1173   Operands.push_back(AMDGPUOperand::CreateToken(Name, NameLoc));
1174 
1175   while (!getLexer().is(AsmToken::EndOfStatement)) {
1176     AMDGPUAsmParser::OperandMatchResultTy Res = parseOperand(Operands, Name);
1177 
1178     // Eat the comma or space if there is one.
1179     if (getLexer().is(AsmToken::Comma))
1180       Parser.Lex();
1181 
1182     switch (Res) {
1183       case MatchOperand_Success: break;
1184       case MatchOperand_ParseFail: return Error(getLexer().getLoc(),
1185                                                 "failed parsing operand.");
1186       case MatchOperand_NoMatch: return Error(getLexer().getLoc(),
1187                                               "not a valid operand.");
1188     }
1189   }
1190 
1191   return false;
1192 }
1193 
1194 //===----------------------------------------------------------------------===//
1195 // Utility functions
1196 //===----------------------------------------------------------------------===//
1197 
1198 AMDGPUAsmParser::OperandMatchResultTy
1199 AMDGPUAsmParser::parseIntWithPrefix(const char *Prefix, int64_t &Int,
1200                                     int64_t Default) {
1201   // We are at the end of the statement, and this is a default argument, so
1202   // use a default value.
1203   if (getLexer().is(AsmToken::EndOfStatement)) {
1204     Int = Default;
1205     return MatchOperand_Success;
1206   }
1207 
1208   switch(getLexer().getKind()) {
1209     default: return MatchOperand_NoMatch;
1210     case AsmToken::Identifier: {
1211       StringRef OffsetName = Parser.getTok().getString();
1212       if (!OffsetName.equals(Prefix))
1213         return MatchOperand_NoMatch;
1214 
1215       Parser.Lex();
1216       if (getLexer().isNot(AsmToken::Colon))
1217         return MatchOperand_ParseFail;
1218 
1219       Parser.Lex();
1220       if (getLexer().isNot(AsmToken::Integer))
1221         return MatchOperand_ParseFail;
1222 
1223       if (getParser().parseAbsoluteExpression(Int))
1224         return MatchOperand_ParseFail;
1225       break;
1226     }
1227   }
1228   return MatchOperand_Success;
1229 }
1230 
1231 AMDGPUAsmParser::OperandMatchResultTy
1232 AMDGPUAsmParser::parseIntWithPrefix(const char *Prefix, OperandVector &Operands,
1233                                     enum AMDGPUOperand::ImmTy ImmTy) {
1234 
1235   SMLoc S = Parser.getTok().getLoc();
1236   int64_t Offset = 0;
1237 
1238   AMDGPUAsmParser::OperandMatchResultTy Res = parseIntWithPrefix(Prefix, Offset);
1239   if (Res != MatchOperand_Success)
1240     return Res;
1241 
1242   Operands.push_back(AMDGPUOperand::CreateImm(Offset, S, ImmTy));
1243   return MatchOperand_Success;
1244 }
1245 
1246 AMDGPUAsmParser::OperandMatchResultTy
1247 AMDGPUAsmParser::parseNamedBit(const char *Name, OperandVector &Operands,
1248                                enum AMDGPUOperand::ImmTy ImmTy) {
1249   int64_t Bit = 0;
1250   SMLoc S = Parser.getTok().getLoc();
1251 
1252   // We are at the end of the statement, and this is a default argument, so
1253   // use a default value.
1254   if (getLexer().isNot(AsmToken::EndOfStatement)) {
1255     switch(getLexer().getKind()) {
1256       case AsmToken::Identifier: {
1257         StringRef Tok = Parser.getTok().getString();
1258         if (Tok == Name) {
1259           Bit = 1;
1260           Parser.Lex();
1261         } else if (Tok.startswith("no") && Tok.endswith(Name)) {
1262           Bit = 0;
1263           Parser.Lex();
1264         } else {
1265           return MatchOperand_NoMatch;
1266         }
1267         break;
1268       }
1269       default:
1270         return MatchOperand_NoMatch;
1271     }
1272   }
1273 
1274   Operands.push_back(AMDGPUOperand::CreateImm(Bit, S, ImmTy));
1275   return MatchOperand_Success;
1276 }
1277 
1278 typedef std::map<enum AMDGPUOperand::ImmTy, unsigned> OptionalImmIndexMap;
1279 
1280 void addOptionalImmOperand(MCInst& Inst, const OperandVector& Operands,
1281                            OptionalImmIndexMap& OptionalIdx,
1282                            enum AMDGPUOperand::ImmTy ImmT, int64_t Default = 0) {
1283   auto i = OptionalIdx.find(ImmT);
1284   if (i != OptionalIdx.end()) {
1285     unsigned Idx = i->second;
1286     ((AMDGPUOperand &)*Operands[Idx]).addImmOperands(Inst, 1);
1287   } else {
1288     Inst.addOperand(MCOperand::createImm(Default));
1289   }
1290 }
1291 
1292 static bool operandsHasOptionalOp(const OperandVector &Operands,
1293                                   const OptionalOperand &OOp) {
1294   for (unsigned i = 0; i < Operands.size(); i++) {
1295     const AMDGPUOperand &ParsedOp = ((const AMDGPUOperand &)*Operands[i]);
1296     if ((ParsedOp.isImm() && ParsedOp.getImmTy() == OOp.Type) ||
1297         (ParsedOp.isToken() && ParsedOp.getToken() == OOp.Name))
1298       return true;
1299 
1300   }
1301   return false;
1302 }
1303 
1304 AMDGPUAsmParser::OperandMatchResultTy
1305 AMDGPUAsmParser::parseOptionalOps(const ArrayRef<OptionalOperand> &OptionalOps,
1306                                    OperandVector &Operands) {
1307   SMLoc S = Parser.getTok().getLoc();
1308   for (const OptionalOperand &Op : OptionalOps) {
1309     if (operandsHasOptionalOp(Operands, Op))
1310       continue;
1311     AMDGPUAsmParser::OperandMatchResultTy Res;
1312     int64_t Value;
1313     if (Op.IsBit) {
1314       Res = parseNamedBit(Op.Name, Operands, Op.Type);
1315       if (Res == MatchOperand_NoMatch)
1316         continue;
1317       return Res;
1318     }
1319 
1320     Res = parseIntWithPrefix(Op.Name, Value, Op.Default);
1321 
1322     if (Res == MatchOperand_NoMatch)
1323       continue;
1324 
1325     if (Res != MatchOperand_Success)
1326       return Res;
1327 
1328     bool DefaultValue = (Value == Op.Default);
1329 
1330     if (Op.ConvertResult && !Op.ConvertResult(Value)) {
1331       return MatchOperand_ParseFail;
1332     }
1333 
1334     if (!DefaultValue) {
1335       Operands.push_back(AMDGPUOperand::CreateImm(Value, S, Op.Type));
1336     }
1337     return MatchOperand_Success;
1338   }
1339   return MatchOperand_NoMatch;
1340 }
1341 
1342 //===----------------------------------------------------------------------===//
1343 // ds
1344 //===----------------------------------------------------------------------===//
1345 
1346 static const OptionalOperand DSOptionalOps [] = {
1347   {"offset",  AMDGPUOperand::ImmTyOffset, false, 0, nullptr},
1348   {"gds",     AMDGPUOperand::ImmTyGDS, true, 0, nullptr}
1349 };
1350 
1351 static const OptionalOperand DSOptionalOpsOff01 [] = {
1352   {"offset0", AMDGPUOperand::ImmTyDSOffset0, false, 0, nullptr},
1353   {"offset1", AMDGPUOperand::ImmTyDSOffset1, false, 0, nullptr},
1354   {"gds",     AMDGPUOperand::ImmTyGDS, true, 0, nullptr}
1355 };
1356 
1357 AMDGPUAsmParser::OperandMatchResultTy
1358 AMDGPUAsmParser::parseDSOptionalOps(OperandVector &Operands) {
1359   return parseOptionalOps(DSOptionalOps, Operands);
1360 }
1361 AMDGPUAsmParser::OperandMatchResultTy
1362 AMDGPUAsmParser::parseDSOff01OptionalOps(OperandVector &Operands) {
1363   return parseOptionalOps(DSOptionalOpsOff01, Operands);
1364 }
1365 
1366 AMDGPUAsmParser::OperandMatchResultTy
1367 AMDGPUAsmParser::parseDSOffsetOptional(OperandVector &Operands) {
1368   SMLoc S = Parser.getTok().getLoc();
1369   AMDGPUAsmParser::OperandMatchResultTy Res =
1370     parseIntWithPrefix("offset", Operands, AMDGPUOperand::ImmTyOffset);
1371   if (Res == MatchOperand_NoMatch) {
1372     Operands.push_back(AMDGPUOperand::CreateImm(0, S,
1373                        AMDGPUOperand::ImmTyOffset));
1374     Res = MatchOperand_Success;
1375   }
1376   return Res;
1377 }
1378 
1379 bool AMDGPUOperand::isDSOffset() const {
1380   return isImm() && isUInt<16>(getImm());
1381 }
1382 
1383 bool AMDGPUOperand::isDSOffset01() const {
1384   return isImm() && isUInt<8>(getImm());
1385 }
1386 
1387 void AMDGPUAsmParser::cvtDSOffset01(MCInst &Inst,
1388                                     const OperandVector &Operands) {
1389 
1390   OptionalImmIndexMap OptionalIdx;
1391 
1392   for (unsigned i = 1, e = Operands.size(); i != e; ++i) {
1393     AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[i]);
1394 
1395     // Add the register arguments
1396     if (Op.isReg()) {
1397       Op.addRegOperands(Inst, 1);
1398       continue;
1399     }
1400 
1401     // Handle optional arguments
1402     OptionalIdx[Op.getImmTy()] = i;
1403   }
1404 
1405   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyDSOffset0);
1406   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyDSOffset1);
1407   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyGDS);
1408 
1409   Inst.addOperand(MCOperand::createReg(AMDGPU::M0)); // m0
1410 }
1411 
1412 void AMDGPUAsmParser::cvtDS(MCInst &Inst, const OperandVector &Operands) {
1413 
1414   std::map<enum AMDGPUOperand::ImmTy, unsigned> OptionalIdx;
1415   bool GDSOnly = false;
1416 
1417   for (unsigned i = 1, e = Operands.size(); i != e; ++i) {
1418     AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[i]);
1419 
1420     // Add the register arguments
1421     if (Op.isReg()) {
1422       Op.addRegOperands(Inst, 1);
1423       continue;
1424     }
1425 
1426     if (Op.isToken() && Op.getToken() == "gds") {
1427       GDSOnly = true;
1428       continue;
1429     }
1430 
1431     // Handle optional arguments
1432     OptionalIdx[Op.getImmTy()] = i;
1433   }
1434 
1435   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyOffset);
1436   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyGDS);
1437 
1438   if (!GDSOnly) {
1439     addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyGDS);
1440   }
1441   Inst.addOperand(MCOperand::createReg(AMDGPU::M0)); // m0
1442 }
1443 
1444 
1445 //===----------------------------------------------------------------------===//
1446 // s_waitcnt
1447 //===----------------------------------------------------------------------===//
1448 
1449 bool AMDGPUAsmParser::parseCnt(int64_t &IntVal) {
1450   StringRef CntName = Parser.getTok().getString();
1451   int64_t CntVal;
1452 
1453   Parser.Lex();
1454   if (getLexer().isNot(AsmToken::LParen))
1455     return true;
1456 
1457   Parser.Lex();
1458   if (getLexer().isNot(AsmToken::Integer))
1459     return true;
1460 
1461   if (getParser().parseAbsoluteExpression(CntVal))
1462     return true;
1463 
1464   if (getLexer().isNot(AsmToken::RParen))
1465     return true;
1466 
1467   Parser.Lex();
1468   if (getLexer().is(AsmToken::Amp) || getLexer().is(AsmToken::Comma))
1469     Parser.Lex();
1470 
1471   int CntShift;
1472   int CntMask;
1473 
1474   if (CntName == "vmcnt") {
1475     CntMask = 0xf;
1476     CntShift = 0;
1477   } else if (CntName == "expcnt") {
1478     CntMask = 0x7;
1479     CntShift = 4;
1480   } else if (CntName == "lgkmcnt") {
1481     CntMask = 0xf;
1482     CntShift = 8;
1483   } else {
1484     return true;
1485   }
1486 
1487   IntVal &= ~(CntMask << CntShift);
1488   IntVal |= (CntVal << CntShift);
1489   return false;
1490 }
1491 
1492 AMDGPUAsmParser::OperandMatchResultTy
1493 AMDGPUAsmParser::parseSWaitCntOps(OperandVector &Operands) {
1494   // Disable all counters by default.
1495   // vmcnt   [3:0]
1496   // expcnt  [6:4]
1497   // lgkmcnt [11:8]
1498   int64_t CntVal = 0xf7f;
1499   SMLoc S = Parser.getTok().getLoc();
1500 
1501   switch(getLexer().getKind()) {
1502     default: return MatchOperand_ParseFail;
1503     case AsmToken::Integer:
1504       // The operand can be an integer value.
1505       if (getParser().parseAbsoluteExpression(CntVal))
1506         return MatchOperand_ParseFail;
1507       break;
1508 
1509     case AsmToken::Identifier:
1510       do {
1511         if (parseCnt(CntVal))
1512           return MatchOperand_ParseFail;
1513       } while(getLexer().isNot(AsmToken::EndOfStatement));
1514       break;
1515   }
1516   Operands.push_back(AMDGPUOperand::CreateImm(CntVal, S));
1517   return MatchOperand_Success;
1518 }
1519 
1520 bool AMDGPUOperand::isSWaitCnt() const {
1521   return isImm();
1522 }
1523 
1524 //===----------------------------------------------------------------------===//
1525 // sopp branch targets
1526 //===----------------------------------------------------------------------===//
1527 
1528 AMDGPUAsmParser::OperandMatchResultTy
1529 AMDGPUAsmParser::parseSOppBrTarget(OperandVector &Operands) {
1530   SMLoc S = Parser.getTok().getLoc();
1531 
1532   switch (getLexer().getKind()) {
1533     default: return MatchOperand_ParseFail;
1534     case AsmToken::Integer: {
1535       int64_t Imm;
1536       if (getParser().parseAbsoluteExpression(Imm))
1537         return MatchOperand_ParseFail;
1538       Operands.push_back(AMDGPUOperand::CreateImm(Imm, S));
1539       return MatchOperand_Success;
1540     }
1541 
1542     case AsmToken::Identifier:
1543       Operands.push_back(AMDGPUOperand::CreateExpr(
1544           MCSymbolRefExpr::create(getContext().getOrCreateSymbol(
1545                                   Parser.getTok().getString()), getContext()), S));
1546       Parser.Lex();
1547       return MatchOperand_Success;
1548   }
1549 }
1550 
1551 //===----------------------------------------------------------------------===//
1552 // flat
1553 //===----------------------------------------------------------------------===//
1554 
1555 static const OptionalOperand FlatOptionalOps [] = {
1556   {"glc",    AMDGPUOperand::ImmTyGLC, true, 0, nullptr},
1557   {"slc",    AMDGPUOperand::ImmTySLC, true, 0, nullptr},
1558   {"tfe",    AMDGPUOperand::ImmTyTFE, true, 0, nullptr}
1559 };
1560 
1561 static const OptionalOperand FlatAtomicOptionalOps [] = {
1562   {"slc",    AMDGPUOperand::ImmTySLC, true, 0, nullptr},
1563   {"tfe",    AMDGPUOperand::ImmTyTFE, true, 0, nullptr}
1564 };
1565 
1566 AMDGPUAsmParser::OperandMatchResultTy
1567 AMDGPUAsmParser::parseFlatOptionalOps(OperandVector &Operands) {
1568   return parseOptionalOps(FlatOptionalOps, Operands);
1569 }
1570 
1571 AMDGPUAsmParser::OperandMatchResultTy
1572 AMDGPUAsmParser::parseFlatAtomicOptionalOps(OperandVector &Operands) {
1573   return parseOptionalOps(FlatAtomicOptionalOps, Operands);
1574 }
1575 
1576 void AMDGPUAsmParser::cvtFlat(MCInst &Inst,
1577                                const OperandVector &Operands) {
1578   OptionalImmIndexMap OptionalIdx;
1579 
1580   for (unsigned i = 1, e = Operands.size(); i != e; ++i) {
1581     AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[i]);
1582 
1583     // Add the register arguments
1584     if (Op.isReg()) {
1585       Op.addRegOperands(Inst, 1);
1586       continue;
1587     }
1588 
1589     OptionalIdx[Op.getImmTy()] = i;
1590   }
1591   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyGLC);
1592   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySLC);
1593   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyTFE);
1594 }
1595 
1596 
1597 void AMDGPUAsmParser::cvtFlatAtomic(MCInst &Inst,
1598                                const OperandVector &Operands) {
1599   OptionalImmIndexMap OptionalIdx;
1600 
1601   for (unsigned i = 1, e = Operands.size(); i != e; ++i) {
1602     AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[i]);
1603 
1604     // Add the register arguments
1605     if (Op.isReg()) {
1606       Op.addRegOperands(Inst, 1);
1607       continue;
1608     }
1609 
1610     // Handle 'glc' token for flat atomics.
1611     if (Op.isToken()) {
1612       continue;
1613     }
1614 
1615     // Handle optional arguments
1616     OptionalIdx[Op.getImmTy()] = i;
1617   }
1618   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySLC);
1619   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyTFE);
1620 }
1621 
1622 //===----------------------------------------------------------------------===//
1623 // mubuf
1624 //===----------------------------------------------------------------------===//
1625 
1626 static const OptionalOperand MubufOptionalOps [] = {
1627   {"offset", AMDGPUOperand::ImmTyOffset, false, 0, nullptr},
1628   {"glc",    AMDGPUOperand::ImmTyGLC, true, 0, nullptr},
1629   {"slc",    AMDGPUOperand::ImmTySLC, true, 0, nullptr},
1630   {"tfe",    AMDGPUOperand::ImmTyTFE, true, 0, nullptr}
1631 };
1632 
1633 AMDGPUAsmParser::OperandMatchResultTy
1634 AMDGPUAsmParser::parseMubufOptionalOps(OperandVector &Operands) {
1635   return parseOptionalOps(MubufOptionalOps, Operands);
1636 }
1637 
1638 AMDGPUAsmParser::OperandMatchResultTy
1639 AMDGPUAsmParser::parseOffset(OperandVector &Operands) {
1640   return parseIntWithPrefix("offset", Operands);
1641 }
1642 
1643 AMDGPUAsmParser::OperandMatchResultTy
1644 AMDGPUAsmParser::parseGLC(OperandVector &Operands) {
1645   return parseNamedBit("glc", Operands);
1646 }
1647 
1648 AMDGPUAsmParser::OperandMatchResultTy
1649 AMDGPUAsmParser::parseSLC(OperandVector &Operands) {
1650   return parseNamedBit("slc", Operands);
1651 }
1652 
1653 AMDGPUAsmParser::OperandMatchResultTy
1654 AMDGPUAsmParser::parseTFE(OperandVector &Operands) {
1655   return parseNamedBit("tfe", Operands);
1656 }
1657 
1658 bool AMDGPUOperand::isMubufOffset() const {
1659   return isImmTy(ImmTyOffset) && isUInt<12>(getImm());
1660 }
1661 
1662 void AMDGPUAsmParser::cvtMubuf(MCInst &Inst,
1663                                const OperandVector &Operands) {
1664   OptionalImmIndexMap OptionalIdx;
1665 
1666   for (unsigned i = 1, e = Operands.size(); i != e; ++i) {
1667     AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[i]);
1668 
1669     // Add the register arguments
1670     if (Op.isReg()) {
1671       Op.addRegOperands(Inst, 1);
1672       continue;
1673     }
1674 
1675     // Handle the case where soffset is an immediate
1676     if (Op.isImm() && Op.getImmTy() == AMDGPUOperand::ImmTyNone) {
1677       Op.addImmOperands(Inst, 1);
1678       continue;
1679     }
1680 
1681     // Handle tokens like 'offen' which are sometimes hard-coded into the
1682     // asm string.  There are no MCInst operands for these.
1683     if (Op.isToken()) {
1684       continue;
1685     }
1686     assert(Op.isImm());
1687 
1688     // Handle optional arguments
1689     OptionalIdx[Op.getImmTy()] = i;
1690   }
1691 
1692   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyOffset);
1693   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyGLC);
1694   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySLC);
1695   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyTFE);
1696 }
1697 
1698 //===----------------------------------------------------------------------===//
1699 // mimg
1700 //===----------------------------------------------------------------------===//
1701 
1702 AMDGPUAsmParser::OperandMatchResultTy
1703 AMDGPUAsmParser::parseDMask(OperandVector &Operands) {
1704   return parseIntWithPrefix("dmask", Operands, AMDGPUOperand::ImmTyDMask);
1705 }
1706 
1707 AMDGPUAsmParser::OperandMatchResultTy
1708 AMDGPUAsmParser::parseUNorm(OperandVector &Operands) {
1709   return parseNamedBit("unorm", Operands, AMDGPUOperand::ImmTyUNorm);
1710 }
1711 
1712 AMDGPUAsmParser::OperandMatchResultTy
1713 AMDGPUAsmParser::parseDA(OperandVector &Operands) {
1714   return parseNamedBit("da", Operands, AMDGPUOperand::ImmTyDA);
1715 }
1716 
1717 AMDGPUAsmParser::OperandMatchResultTy
1718 AMDGPUAsmParser::parseR128(OperandVector &Operands) {
1719   return parseNamedBit("r128", Operands, AMDGPUOperand::ImmTyR128);
1720 }
1721 
1722 AMDGPUAsmParser::OperandMatchResultTy
1723 AMDGPUAsmParser::parseLWE(OperandVector &Operands) {
1724   return parseNamedBit("lwe", Operands, AMDGPUOperand::ImmTyLWE);
1725 }
1726 
1727 //===----------------------------------------------------------------------===//
1728 // smrd
1729 //===----------------------------------------------------------------------===//
1730 
1731 bool AMDGPUOperand::isSMRDOffset() const {
1732 
1733   // FIXME: Support 20-bit offsets on VI.  We need to to pass subtarget
1734   // information here.
1735   return isImm() && isUInt<8>(getImm());
1736 }
1737 
1738 bool AMDGPUOperand::isSMRDLiteralOffset() const {
1739   // 32-bit literals are only supported on CI and we only want to use them
1740   // when the offset is > 8-bits.
1741   return isImm() && !isUInt<8>(getImm()) && isUInt<32>(getImm());
1742 }
1743 
1744 //===----------------------------------------------------------------------===//
1745 // vop3
1746 //===----------------------------------------------------------------------===//
1747 
1748 static bool ConvertOmodMul(int64_t &Mul) {
1749   if (Mul != 1 && Mul != 2 && Mul != 4)
1750     return false;
1751 
1752   Mul >>= 1;
1753   return true;
1754 }
1755 
1756 static bool ConvertOmodDiv(int64_t &Div) {
1757   if (Div == 1) {
1758     Div = 0;
1759     return true;
1760   }
1761 
1762   if (Div == 2) {
1763     Div = 3;
1764     return true;
1765   }
1766 
1767   return false;
1768 }
1769 
1770 static const OptionalOperand VOP3OptionalOps [] = {
1771   {"clamp", AMDGPUOperand::ImmTyClamp, true, 0, nullptr},
1772   {"mul",   AMDGPUOperand::ImmTyOMod, false, 1, ConvertOmodMul},
1773   {"div",   AMDGPUOperand::ImmTyOMod, false, 1, ConvertOmodDiv},
1774 };
1775 
1776 static bool isVOP3(OperandVector &Operands) {
1777   if (operandsHaveModifiers(Operands))
1778     return true;
1779 
1780   if (Operands.size() >= 2) {
1781     AMDGPUOperand &DstOp = ((AMDGPUOperand&)*Operands[1]);
1782 
1783     if (DstOp.isRegClass(AMDGPU::SGPR_64RegClassID))
1784       return true;
1785   }
1786 
1787   if (Operands.size() >= 5)
1788     return true;
1789 
1790   if (Operands.size() > 3) {
1791     AMDGPUOperand &Src1Op = ((AMDGPUOperand&)*Operands[3]);
1792     if (Src1Op.isRegClass(AMDGPU::SReg_32RegClassID) ||
1793         Src1Op.isRegClass(AMDGPU::SReg_64RegClassID))
1794       return true;
1795   }
1796   return false;
1797 }
1798 
1799 AMDGPUAsmParser::OperandMatchResultTy
1800 AMDGPUAsmParser::parseVOP3OptionalOps(OperandVector &Operands) {
1801 
1802   // The value returned by this function may change after parsing
1803   // an operand so store the original value here.
1804   bool HasModifiers = operandsHaveModifiers(Operands);
1805 
1806   bool IsVOP3 = isVOP3(Operands);
1807   if (HasModifiers || IsVOP3 ||
1808       getLexer().isNot(AsmToken::EndOfStatement) ||
1809       getForcedEncodingSize() == 64) {
1810 
1811     AMDGPUAsmParser::OperandMatchResultTy Res =
1812         parseOptionalOps(VOP3OptionalOps, Operands);
1813 
1814     if (!HasModifiers && Res == MatchOperand_Success) {
1815       // We have added a modifier operation, so we need to make sure all
1816       // previous register operands have modifiers
1817       for (unsigned i = 2, e = Operands.size(); i != e; ++i) {
1818         AMDGPUOperand &Op = ((AMDGPUOperand&)*Operands[i]);
1819         if ((Op.isReg() || Op.isImm()) && !Op.hasModifiers())
1820           Op.setModifiers(0);
1821       }
1822     }
1823     return Res;
1824   }
1825   return MatchOperand_NoMatch;
1826 }
1827 
1828 void AMDGPUAsmParser::cvtId(MCInst &Inst, const OperandVector &Operands) {
1829   unsigned I = 1;
1830   const MCInstrDesc &Desc = MII.get(Inst.getOpcode());
1831   for (unsigned J = 0; J < Desc.getNumDefs(); ++J) {
1832     ((AMDGPUOperand &)*Operands[I++]).addRegOperands(Inst, 1);
1833   }
1834   for (unsigned E = Operands.size(); I != E; ++I)
1835     ((AMDGPUOperand &)*Operands[I]).addRegOrImmOperands(Inst, 1);
1836 }
1837 
1838 void AMDGPUAsmParser::cvtVOP3_2_mod(MCInst &Inst, const OperandVector &Operands) {
1839   uint64_t TSFlags = MII.get(Inst.getOpcode()).TSFlags;
1840   if (TSFlags & SIInstrFlags::VOP3) {
1841     cvtVOP3(Inst, Operands);
1842   } else {
1843     cvtId(Inst, Operands);
1844   }
1845 }
1846 
1847 void AMDGPUAsmParser::cvtVOP3_2_nomod(MCInst &Inst, const OperandVector &Operands) {
1848   if (operandsHaveModifiers(Operands)) {
1849     cvtVOP3(Inst, Operands);
1850   } else {
1851     cvtId(Inst, Operands);
1852   }
1853 }
1854 
1855 void AMDGPUAsmParser::cvtVOP3_only(MCInst &Inst, const OperandVector &Operands) {
1856   cvtVOP3(Inst, Operands);
1857 }
1858 
1859 void AMDGPUAsmParser::cvtVOP3(MCInst &Inst, const OperandVector &Operands) {
1860   OptionalImmIndexMap OptionalIdx;
1861   unsigned I = 1;
1862   const MCInstrDesc &Desc = MII.get(Inst.getOpcode());
1863   for (unsigned J = 0; J < Desc.getNumDefs(); ++J) {
1864     ((AMDGPUOperand &)*Operands[I++]).addRegOperands(Inst, 1);
1865   }
1866 
1867   for (unsigned E = Operands.size(); I != E; ++I) {
1868     AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[I]);
1869     if (Op.isRegOrImmWithInputMods()) {
1870       Op.addRegOrImmWithInputModsOperands(Inst, 2);
1871     } else if (Op.isImm()) {
1872       OptionalIdx[Op.getImmTy()] = I;
1873     } else {
1874       assert(false);
1875     }
1876   }
1877 
1878   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyClamp);
1879   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyOMod);
1880 }
1881 
1882 void AMDGPUAsmParser::cvtMIMG(MCInst &Inst, const OperandVector &Operands) {
1883   unsigned I = 1;
1884   const MCInstrDesc &Desc = MII.get(Inst.getOpcode());
1885   for (unsigned J = 0; J < Desc.getNumDefs(); ++J) {
1886     ((AMDGPUOperand &)*Operands[I++]).addRegOperands(Inst, 1);
1887   }
1888 
1889   OptionalImmIndexMap OptionalIdx;
1890 
1891   for (unsigned E = Operands.size(); I != E; ++I) {
1892     AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[I]);
1893 
1894     // Add the register arguments
1895     if (Op.isRegOrImm()) {
1896       Op.addRegOrImmOperands(Inst, 1);
1897       continue;
1898     } else if (Op.isImmModifier()) {
1899       OptionalIdx[Op.getImmTy()] = I;
1900     } else {
1901       assert(false);
1902     }
1903   }
1904 
1905   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyDMask);
1906   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyUNorm);
1907   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyGLC);
1908   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyDA);
1909   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyR128);
1910   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyTFE);
1911   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyLWE);
1912   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySLC);
1913 }
1914 
1915 void AMDGPUAsmParser::cvtMIMGAtomic(MCInst &Inst, const OperandVector &Operands) {
1916   unsigned I = 1;
1917   const MCInstrDesc &Desc = MII.get(Inst.getOpcode());
1918   for (unsigned J = 0; J < Desc.getNumDefs(); ++J) {
1919     ((AMDGPUOperand &)*Operands[I++]).addRegOperands(Inst, 1);
1920   }
1921 
1922   // Add src, same as dst
1923   ((AMDGPUOperand &)*Operands[I]).addRegOperands(Inst, 1);
1924 
1925   OptionalImmIndexMap OptionalIdx;
1926 
1927   for (unsigned E = Operands.size(); I != E; ++I) {
1928     AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[I]);
1929 
1930     // Add the register arguments
1931     if (Op.isRegOrImm()) {
1932       Op.addRegOrImmOperands(Inst, 1);
1933       continue;
1934     } else if (Op.isImmModifier()) {
1935       OptionalIdx[Op.getImmTy()] = I;
1936     } else {
1937       assert(false);
1938     }
1939   }
1940 
1941   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyDMask);
1942   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyUNorm);
1943   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyGLC);
1944   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyDA);
1945   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyR128);
1946   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyTFE);
1947   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyLWE);
1948   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySLC);
1949 }
1950 
1951 //===----------------------------------------------------------------------===//
1952 // dpp
1953 //===----------------------------------------------------------------------===//
1954 
1955 bool AMDGPUOperand::isDPPCtrl() const {
1956   bool result = isImm() && getImmTy() == ImmTyDppCtrl && isUInt<9>(getImm());
1957   if (result) {
1958     int64_t Imm = getImm();
1959     return ((Imm >= 0x000) && (Imm <= 0x0ff)) ||
1960            ((Imm >= 0x101) && (Imm <= 0x10f)) ||
1961            ((Imm >= 0x111) && (Imm <= 0x11f)) ||
1962            ((Imm >= 0x121) && (Imm <= 0x12f)) ||
1963            (Imm == 0x130) ||
1964            (Imm == 0x134) ||
1965            (Imm == 0x138) ||
1966            (Imm == 0x13c) ||
1967            (Imm == 0x140) ||
1968            (Imm == 0x141) ||
1969            (Imm == 0x142) ||
1970            (Imm == 0x143);
1971   }
1972   return false;
1973 }
1974 
1975 AMDGPUAsmParser::OperandMatchResultTy
1976 AMDGPUAsmParser::parseDPPCtrlOps(OperandVector &Operands) {
1977   // ToDo: use same syntax as sp3 for dpp_ctrl
1978   SMLoc S = Parser.getTok().getLoc();
1979   StringRef Prefix;
1980   int64_t Int;
1981 
1982   if (getLexer().getKind() == AsmToken::Identifier) {
1983     Prefix = Parser.getTok().getString();
1984   } else {
1985     return MatchOperand_NoMatch;
1986   }
1987 
1988   if (Prefix == "row_mirror") {
1989     Int = 0x140;
1990   } else if (Prefix == "row_half_mirror") {
1991     Int = 0x141;
1992   } else {
1993     Parser.Lex();
1994     if (getLexer().isNot(AsmToken::Colon))
1995       return MatchOperand_ParseFail;
1996 
1997     if (Prefix == "quad_perm") {
1998       // quad_perm:[%d,%d,%d,%d]
1999       Parser.Lex();
2000       if (getLexer().isNot(AsmToken::LBrac))
2001         return MatchOperand_ParseFail;
2002 
2003       Parser.Lex();
2004       if (getLexer().isNot(AsmToken::Integer))
2005         return MatchOperand_ParseFail;
2006       Int = getLexer().getTok().getIntVal();
2007 
2008       Parser.Lex();
2009       if (getLexer().isNot(AsmToken::Comma))
2010         return MatchOperand_ParseFail;
2011       Parser.Lex();
2012       if (getLexer().isNot(AsmToken::Integer))
2013         return MatchOperand_ParseFail;
2014       Int += (getLexer().getTok().getIntVal() << 2);
2015 
2016       Parser.Lex();
2017       if (getLexer().isNot(AsmToken::Comma))
2018         return MatchOperand_ParseFail;
2019       Parser.Lex();
2020       if (getLexer().isNot(AsmToken::Integer))
2021         return MatchOperand_ParseFail;
2022       Int += (getLexer().getTok().getIntVal() << 4);
2023 
2024       Parser.Lex();
2025       if (getLexer().isNot(AsmToken::Comma))
2026         return MatchOperand_ParseFail;
2027       Parser.Lex();
2028       if (getLexer().isNot(AsmToken::Integer))
2029         return MatchOperand_ParseFail;
2030       Int += (getLexer().getTok().getIntVal() << 6);
2031 
2032       Parser.Lex();
2033       if (getLexer().isNot(AsmToken::RBrac))
2034         return MatchOperand_ParseFail;
2035 
2036     } else {
2037       // sel:%d
2038       Parser.Lex();
2039       if (getLexer().isNot(AsmToken::Integer))
2040         return MatchOperand_ParseFail;
2041       Int = getLexer().getTok().getIntVal();
2042 
2043       if (Prefix == "row_shl") {
2044         Int |= 0x100;
2045       } else if (Prefix == "row_shr") {
2046         Int |= 0x110;
2047       } else if (Prefix == "row_ror") {
2048         Int |= 0x120;
2049       } else if (Prefix == "wave_shl") {
2050         Int = 0x130;
2051       } else if (Prefix == "wave_rol") {
2052         Int = 0x134;
2053       } else if (Prefix == "wave_shr") {
2054         Int = 0x138;
2055       } else if (Prefix == "wave_ror") {
2056         Int = 0x13C;
2057       } else if (Prefix == "row_bcast") {
2058         if (Int == 15) {
2059           Int = 0x142;
2060         } else if (Int == 31) {
2061           Int = 0x143;
2062         }
2063       } else {
2064         return MatchOperand_NoMatch;
2065       }
2066     }
2067   }
2068   Parser.Lex(); // eat last token
2069 
2070   Operands.push_back(AMDGPUOperand::CreateImm(Int, S,
2071                                               AMDGPUOperand::ImmTyDppCtrl));
2072   return MatchOperand_Success;
2073 }
2074 
2075 static const OptionalOperand DPPOptionalOps [] = {
2076   {"row_mask", AMDGPUOperand::ImmTyDppRowMask, false, 0xf, nullptr},
2077   {"bank_mask", AMDGPUOperand::ImmTyDppBankMask, false, 0xf, nullptr},
2078   {"bound_ctrl", AMDGPUOperand::ImmTyDppBoundCtrl, false, -1, nullptr}
2079 };
2080 
2081 AMDGPUAsmParser::OperandMatchResultTy
2082 AMDGPUAsmParser::parseDPPOptionalOps(OperandVector &Operands) {
2083   SMLoc S = Parser.getTok().getLoc();
2084   OperandMatchResultTy Res = parseOptionalOps(DPPOptionalOps, Operands);
2085   // XXX - sp3 use syntax "bound_ctrl:0" to indicate that bound_ctrl bit was set
2086   if (Res == MatchOperand_Success) {
2087     AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands.back());
2088     // If last operand was parsed as bound_ctrl we should replace it with correct value (1)
2089     if (Op.isImmTy(AMDGPUOperand::ImmTyDppBoundCtrl)) {
2090       Operands.pop_back();
2091       Operands.push_back(
2092         AMDGPUOperand::CreateImm(1, S, AMDGPUOperand::ImmTyDppBoundCtrl));
2093         return MatchOperand_Success;
2094     }
2095   }
2096   return Res;
2097 }
2098 
2099 void AMDGPUAsmParser::cvtDPP_mod(MCInst &Inst, const OperandVector &Operands) {
2100   cvtDPP(Inst, Operands, true);
2101 }
2102 
2103 void AMDGPUAsmParser::cvtDPP_nomod(MCInst &Inst, const OperandVector &Operands) {
2104   cvtDPP(Inst, Operands, false);
2105 }
2106 
2107 void AMDGPUAsmParser::cvtDPP(MCInst &Inst, const OperandVector &Operands,
2108                              bool HasMods) {
2109   OptionalImmIndexMap OptionalIdx;
2110 
2111   unsigned I = 1;
2112   const MCInstrDesc &Desc = MII.get(Inst.getOpcode());
2113   for (unsigned J = 0; J < Desc.getNumDefs(); ++J) {
2114     ((AMDGPUOperand &)*Operands[I++]).addRegOperands(Inst, 1);
2115   }
2116 
2117   for (unsigned E = Operands.size(); I != E; ++I) {
2118     AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[I]);
2119     // Add the register arguments
2120     if (!HasMods && Op.isReg()) {
2121       Op.addRegOperands(Inst, 1);
2122     } else if (HasMods && Op.isRegOrImmWithInputMods()) {
2123       Op.addRegOrImmWithInputModsOperands(Inst, 2);
2124     } else if (Op.isDPPCtrl()) {
2125       Op.addImmOperands(Inst, 1);
2126     } else if (Op.isImm()) {
2127       // Handle optional arguments
2128       OptionalIdx[Op.getImmTy()] = I;
2129     } else {
2130       llvm_unreachable("Invalid operand type");
2131     }
2132   }
2133 
2134   // ToDo: fix default values for row_mask and bank_mask
2135   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyDppRowMask, 0xf);
2136   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyDppBankMask, 0xf);
2137   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyDppBoundCtrl);
2138 }
2139 
2140 
2141 /// Force static initialization.
2142 extern "C" void LLVMInitializeAMDGPUAsmParser() {
2143   RegisterMCAsmParser<AMDGPUAsmParser> A(TheAMDGPUTarget);
2144   RegisterMCAsmParser<AMDGPUAsmParser> B(TheGCNTarget);
2145 }
2146 
2147 #define GET_REGISTER_MATCHER
2148 #define GET_MATCHER_IMPLEMENTATION
2149 #include "AMDGPUGenAsmMatcher.inc"
2150