1 //===-- AMDGPUAsmParser.cpp - Parse SI asm to MCInst instructions ---------===//
2 //
3 //                     The LLVM Compiler Infrastructure
4 //
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
7 //
8 //===----------------------------------------------------------------------===//
9 
10 #include "AMDKernelCodeT.h"
11 #include "MCTargetDesc/AMDGPUMCTargetDesc.h"
12 #include "MCTargetDesc/AMDGPUTargetStreamer.h"
13 #include "SIDefines.h"
14 #include "Utils/AMDGPUBaseInfo.h"
15 #include "Utils/AMDKernelCodeTUtils.h"
16 #include "llvm/ADT/APFloat.h"
17 #include "llvm/ADT/STLExtras.h"
18 #include "llvm/ADT/SmallString.h"
19 #include "llvm/ADT/StringSwitch.h"
20 #include "llvm/ADT/Twine.h"
21 #include "llvm/MC/MCContext.h"
22 #include "llvm/MC/MCExpr.h"
23 #include "llvm/MC/MCInst.h"
24 #include "llvm/MC/MCInstrInfo.h"
25 #include "llvm/MC/MCParser/MCAsmLexer.h"
26 #include "llvm/MC/MCParser/MCAsmParser.h"
27 #include "llvm/MC/MCParser/MCParsedAsmOperand.h"
28 #include "llvm/MC/MCParser/MCTargetAsmParser.h"
29 #include "llvm/MC/MCRegisterInfo.h"
30 #include "llvm/MC/MCStreamer.h"
31 #include "llvm/MC/MCSubtargetInfo.h"
32 #include "llvm/MC/MCSymbolELF.h"
33 #include "llvm/Support/Debug.h"
34 #include "llvm/Support/ELF.h"
35 #include "llvm/Support/SourceMgr.h"
36 #include "llvm/Support/TargetRegistry.h"
37 #include "llvm/Support/raw_ostream.h"
38 
39 using namespace llvm;
40 
41 namespace {
42 
43 struct OptionalOperand;
44 
45 class AMDGPUOperand : public MCParsedAsmOperand {
46   enum KindTy {
47     Token,
48     Immediate,
49     Register,
50     Expression
51   } Kind;
52 
53   SMLoc StartLoc, EndLoc;
54 
55 public:
56   AMDGPUOperand(enum KindTy K) : MCParsedAsmOperand(), Kind(K) {}
57 
58   MCContext *Ctx;
59 
60   enum ImmTy {
61     ImmTyNone,
62     ImmTyDSOffset0,
63     ImmTyDSOffset1,
64     ImmTyGDS,
65     ImmTyOffset,
66     ImmTyGLC,
67     ImmTySLC,
68     ImmTyTFE,
69     ImmTyClamp,
70     ImmTyOMod,
71     ImmTyDppCtrl,
72     ImmTyDppRowMask,
73     ImmTyDppBankMask,
74     ImmTyDppBoundCtrl,
75     ImmTyDMask,
76     ImmTyUNorm,
77     ImmTyDA,
78     ImmTyR128,
79     ImmTyLWE,
80   };
81 
82   struct TokOp {
83     const char *Data;
84     unsigned Length;
85   };
86 
87   struct ImmOp {
88     bool IsFPImm;
89     ImmTy Type;
90     int64_t Val;
91     int Modifiers;
92   };
93 
94   struct RegOp {
95     unsigned RegNo;
96     int Modifiers;
97     const MCRegisterInfo *TRI;
98     const MCSubtargetInfo *STI;
99     bool IsForcedVOP3;
100   };
101 
102   union {
103     TokOp Tok;
104     ImmOp Imm;
105     RegOp Reg;
106     const MCExpr *Expr;
107   };
108 
109   void addImmOperands(MCInst &Inst, unsigned N) const {
110     Inst.addOperand(MCOperand::createImm(getImm()));
111   }
112 
113   StringRef getToken() const {
114     return StringRef(Tok.Data, Tok.Length);
115   }
116 
117   void addRegOperands(MCInst &Inst, unsigned N) const {
118     Inst.addOperand(MCOperand::createReg(AMDGPU::getMCReg(getReg(), *Reg.STI)));
119   }
120 
121   void addRegOrImmOperands(MCInst &Inst, unsigned N) const {
122     if (isRegKind())
123       addRegOperands(Inst, N);
124     else
125       addImmOperands(Inst, N);
126   }
127 
128   void addRegOrImmWithInputModsOperands(MCInst &Inst, unsigned N) const {
129     if (isRegKind()) {
130       Inst.addOperand(MCOperand::createImm(Reg.Modifiers));
131       addRegOperands(Inst, N);
132     } else {
133       Inst.addOperand(MCOperand::createImm(Imm.Modifiers));
134       addImmOperands(Inst, N);
135     }
136   }
137 
138   void addSoppBrTargetOperands(MCInst &Inst, unsigned N) const {
139     if (isImm())
140       addImmOperands(Inst, N);
141     else {
142       assert(isExpr());
143       Inst.addOperand(MCOperand::createExpr(Expr));
144     }
145   }
146 
147   bool defaultTokenHasSuffix() const {
148     StringRef Token(Tok.Data, Tok.Length);
149 
150     return Token.endswith("_e32") || Token.endswith("_e64") ||
151       Token.endswith("_dpp");
152   }
153 
154   bool isToken() const override {
155     return Kind == Token;
156   }
157 
158   bool isImm() const override {
159     return Kind == Immediate;
160   }
161 
162   bool isInlinableImm() const {
163     if (!isImm() || Imm.Type != AMDGPUOperand::ImmTyNone /* Only plain
164       immediates are inlinable (e.g. "clamp" attribute is not) */ )
165       return false;
166     // TODO: We should avoid using host float here. It would be better to
167     // check the float bit values which is what a few other places do.
168     // We've had bot failures before due to weird NaN support on mips hosts.
169     const float F = BitsToFloat(Imm.Val);
170     // TODO: Add 1/(2*pi) for VI
171     return (Imm.Val <= 64 && Imm.Val >= -16) ||
172            (F == 0.0 || F == 0.5 || F == -0.5 || F == 1.0 || F == -1.0 ||
173            F == 2.0 || F == -2.0 || F == 4.0 || F == -4.0);
174   }
175 
176   bool isDSOffset0() const {
177     assert(isImm());
178     return Imm.Type == ImmTyDSOffset0;
179   }
180 
181   bool isDSOffset1() const {
182     assert(isImm());
183     return Imm.Type == ImmTyDSOffset1;
184   }
185 
186   int64_t getImm() const {
187     return Imm.Val;
188   }
189 
190   enum ImmTy getImmTy() const {
191     assert(isImm());
192     return Imm.Type;
193   }
194 
195   bool isRegKind() const {
196     return Kind == Register;
197   }
198 
199   bool isReg() const override {
200     return Kind == Register && Reg.Modifiers == 0;
201   }
202 
203   bool isRegOrImmWithInputMods() const {
204     return Kind == Register || isInlinableImm();
205   }
206 
207   bool isImmTy(ImmTy ImmT) const {
208     return isImm() && Imm.Type == ImmT;
209   }
210 
211   bool isClamp() const {
212     return isImmTy(ImmTyClamp);
213   }
214 
215   bool isOMod() const {
216     return isImmTy(ImmTyOMod);
217   }
218 
219   bool isImmModifier() const {
220     return Kind == Immediate && Imm.Type != ImmTyNone;
221   }
222 
223   bool isDMask() const {
224     return isImmTy(ImmTyDMask);
225   }
226 
227   bool isUNorm() const { return isImmTy(ImmTyUNorm); }
228   bool isDA() const { return isImmTy(ImmTyDA); }
229   bool isR128() const { return isImmTy(ImmTyUNorm); }
230   bool isLWE() const { return isImmTy(ImmTyLWE); }
231 
232   bool isMod() const {
233     return isClamp() || isOMod();
234   }
235 
236   bool isGDS() const { return isImmTy(ImmTyGDS); }
237   bool isGLC() const { return isImmTy(ImmTyGLC); }
238   bool isSLC() const { return isImmTy(ImmTySLC); }
239   bool isTFE() const { return isImmTy(ImmTyTFE); }
240 
241   bool isBankMask() const {
242     return isImmTy(ImmTyDppBankMask);
243   }
244 
245   bool isRowMask() const {
246     return isImmTy(ImmTyDppRowMask);
247   }
248 
249   bool isBoundCtrl() const {
250     return isImmTy(ImmTyDppBoundCtrl);
251   }
252 
253   void setModifiers(unsigned Mods) {
254     assert(isReg() || (isImm() && Imm.Modifiers == 0));
255     if (isReg())
256       Reg.Modifiers = Mods;
257     else
258       Imm.Modifiers = Mods;
259   }
260 
261   bool hasModifiers() const {
262     assert(isRegKind() || isImm());
263     return isRegKind() ? Reg.Modifiers != 0 : Imm.Modifiers != 0;
264   }
265 
266   unsigned getReg() const override {
267     return Reg.RegNo;
268   }
269 
270   bool isRegOrImm() const {
271     return isReg() || isImm();
272   }
273 
274   bool isRegClass(unsigned RCID) const {
275     return isReg() && Reg.TRI->getRegClass(RCID).contains(getReg());
276   }
277 
278   bool isSCSrc32() const {
279     return isInlinableImm() || isRegClass(AMDGPU::SReg_32RegClassID);
280   }
281 
282   bool isSCSrc64() const {
283     return isInlinableImm() || isRegClass(AMDGPU::SReg_64RegClassID);
284   }
285 
286   bool isSSrc32() const {
287     return isImm() || isSCSrc32();
288   }
289 
290   bool isSSrc64() const {
291     // TODO: Find out how SALU supports extension of 32-bit literals to 64 bits.
292     // See isVSrc64().
293     return isImm() || isSCSrc64();
294   }
295 
296   bool isVCSrc32() const {
297     return isInlinableImm() || isRegClass(AMDGPU::VS_32RegClassID);
298   }
299 
300   bool isVCSrc64() const {
301     return isInlinableImm() || isRegClass(AMDGPU::VS_64RegClassID);
302   }
303 
304   bool isVSrc32() const {
305     return isImm() || isVCSrc32();
306   }
307 
308   bool isVSrc64() const {
309     // TODO: Check if the 64-bit value (coming from assembly source) can be
310     // narrowed to 32 bits (in the instruction stream). That require knowledge
311     // of instruction type (unsigned/signed, floating or "untyped"/B64),
312     // see [AMD GCN3 ISA 6.3.1].
313     // TODO: How 64-bit values are formed from 32-bit literals in _B64 insns?
314     return isImm() || isVCSrc64();
315   }
316 
317   bool isMem() const override {
318     return false;
319   }
320 
321   bool isExpr() const {
322     return Kind == Expression;
323   }
324 
325   bool isSoppBrTarget() const {
326     return isExpr() || isImm();
327   }
328 
329   SMLoc getStartLoc() const override {
330     return StartLoc;
331   }
332 
333   SMLoc getEndLoc() const override {
334     return EndLoc;
335   }
336 
337   void print(raw_ostream &OS) const override {
338     switch (Kind) {
339     case Register:
340       OS << "<register " << getReg() << " mods: " << Reg.Modifiers << '>';
341       break;
342     case Immediate:
343       if (Imm.Type != AMDGPUOperand::ImmTyNone)
344         OS << getImm();
345       else
346         OS << '<' << getImm() << " mods: " << Imm.Modifiers << '>';
347       break;
348     case Token:
349       OS << '\'' << getToken() << '\'';
350       break;
351     case Expression:
352       OS << "<expr " << *Expr << '>';
353       break;
354     }
355   }
356 
357   static std::unique_ptr<AMDGPUOperand> CreateImm(int64_t Val, SMLoc Loc,
358                                                   enum ImmTy Type = ImmTyNone,
359                                                   bool IsFPImm = false) {
360     auto Op = llvm::make_unique<AMDGPUOperand>(Immediate);
361     Op->Imm.Val = Val;
362     Op->Imm.IsFPImm = IsFPImm;
363     Op->Imm.Type = Type;
364     Op->Imm.Modifiers = 0;
365     Op->StartLoc = Loc;
366     Op->EndLoc = Loc;
367     return Op;
368   }
369 
370   static std::unique_ptr<AMDGPUOperand> CreateToken(StringRef Str, SMLoc Loc,
371                                            bool HasExplicitEncodingSize = true) {
372     auto Res = llvm::make_unique<AMDGPUOperand>(Token);
373     Res->Tok.Data = Str.data();
374     Res->Tok.Length = Str.size();
375     Res->StartLoc = Loc;
376     Res->EndLoc = Loc;
377     return Res;
378   }
379 
380   static std::unique_ptr<AMDGPUOperand> CreateReg(unsigned RegNo, SMLoc S,
381                                                   SMLoc E,
382                                                   const MCRegisterInfo *TRI,
383                                                   const MCSubtargetInfo *STI,
384                                                   bool ForceVOP3) {
385     auto Op = llvm::make_unique<AMDGPUOperand>(Register);
386     Op->Reg.RegNo = RegNo;
387     Op->Reg.TRI = TRI;
388     Op->Reg.STI = STI;
389     Op->Reg.Modifiers = 0;
390     Op->Reg.IsForcedVOP3 = ForceVOP3;
391     Op->StartLoc = S;
392     Op->EndLoc = E;
393     return Op;
394   }
395 
396   static std::unique_ptr<AMDGPUOperand> CreateExpr(const class MCExpr *Expr, SMLoc S) {
397     auto Op = llvm::make_unique<AMDGPUOperand>(Expression);
398     Op->Expr = Expr;
399     Op->StartLoc = S;
400     Op->EndLoc = S;
401     return Op;
402   }
403 
404   bool isDSOffset() const;
405   bool isDSOffset01() const;
406   bool isSWaitCnt() const;
407   bool isMubufOffset() const;
408   bool isSMRDOffset() const;
409   bool isSMRDLiteralOffset() const;
410   bool isDPPCtrl() const;
411 };
412 
413 class AMDGPUAsmParser : public MCTargetAsmParser {
414   const MCInstrInfo &MII;
415   MCAsmParser &Parser;
416 
417   unsigned ForcedEncodingSize;
418 
419   bool isSI() const {
420     return AMDGPU::isSI(getSTI());
421   }
422 
423   bool isCI() const {
424     return AMDGPU::isCI(getSTI());
425   }
426 
427   bool isVI() const {
428     return AMDGPU::isVI(getSTI());
429   }
430 
431   bool hasSGPR102_SGPR103() const {
432     return !isVI();
433   }
434 
435   /// @name Auto-generated Match Functions
436   /// {
437 
438 #define GET_ASSEMBLER_HEADER
439 #include "AMDGPUGenAsmMatcher.inc"
440 
441   /// }
442 
443 private:
444   bool ParseDirectiveMajorMinor(uint32_t &Major, uint32_t &Minor);
445   bool ParseDirectiveHSACodeObjectVersion();
446   bool ParseDirectiveHSACodeObjectISA();
447   bool ParseAMDKernelCodeTValue(StringRef ID, amd_kernel_code_t &Header);
448   bool ParseDirectiveAMDKernelCodeT();
449   bool ParseSectionDirectiveHSAText();
450   bool subtargetHasRegister(const MCRegisterInfo &MRI, unsigned RegNo) const;
451   bool ParseDirectiveAMDGPUHsaKernel();
452   bool ParseDirectiveAMDGPUHsaModuleGlobal();
453   bool ParseDirectiveAMDGPUHsaProgramGlobal();
454   bool ParseSectionDirectiveHSADataGlobalAgent();
455   bool ParseSectionDirectiveHSADataGlobalProgram();
456   bool ParseSectionDirectiveHSARodataReadonlyAgent();
457 
458 public:
459   enum AMDGPUMatchResultTy {
460     Match_PreferE32 = FIRST_TARGET_MATCH_RESULT_TY
461   };
462 
463   AMDGPUAsmParser(const MCSubtargetInfo &STI, MCAsmParser &_Parser,
464                const MCInstrInfo &MII,
465                const MCTargetOptions &Options)
466       : MCTargetAsmParser(Options, STI), MII(MII), Parser(_Parser),
467         ForcedEncodingSize(0) {
468     MCAsmParserExtension::Initialize(Parser);
469 
470     if (getSTI().getFeatureBits().none()) {
471       // Set default features.
472       copySTI().ToggleFeature("SOUTHERN_ISLANDS");
473     }
474 
475     setAvailableFeatures(ComputeAvailableFeatures(getSTI().getFeatureBits()));
476   }
477 
478   AMDGPUTargetStreamer &getTargetStreamer() {
479     MCTargetStreamer &TS = *getParser().getStreamer().getTargetStreamer();
480     return static_cast<AMDGPUTargetStreamer &>(TS);
481   }
482 
483   unsigned getForcedEncodingSize() const {
484     return ForcedEncodingSize;
485   }
486 
487   void setForcedEncodingSize(unsigned Size) {
488     ForcedEncodingSize = Size;
489   }
490 
491   bool isForcedVOP3() const {
492     return ForcedEncodingSize == 64;
493   }
494 
495   std::unique_ptr<AMDGPUOperand> parseRegister();
496   bool ParseRegister(unsigned &RegNo, SMLoc &StartLoc, SMLoc &EndLoc) override;
497   unsigned checkTargetMatchPredicate(MCInst &Inst) override;
498   bool MatchAndEmitInstruction(SMLoc IDLoc, unsigned &Opcode,
499                                OperandVector &Operands, MCStreamer &Out,
500                                uint64_t &ErrorInfo,
501                                bool MatchingInlineAsm) override;
502   bool ParseDirective(AsmToken DirectiveID) override;
503   OperandMatchResultTy parseOperand(OperandVector &Operands, StringRef Mnemonic);
504   bool ParseInstruction(ParseInstructionInfo &Info, StringRef Name,
505                         SMLoc NameLoc, OperandVector &Operands) override;
506 
507   OperandMatchResultTy parseIntWithPrefix(const char *Prefix, int64_t &Int,
508                                           int64_t Default = 0);
509   OperandMatchResultTy parseIntWithPrefix(const char *Prefix,
510                                           OperandVector &Operands,
511                                           enum AMDGPUOperand::ImmTy ImmTy =
512                                                       AMDGPUOperand::ImmTyNone);
513   OperandMatchResultTy parseNamedBit(const char *Name, OperandVector &Operands,
514                                      enum AMDGPUOperand::ImmTy ImmTy =
515                                                       AMDGPUOperand::ImmTyNone);
516   OperandMatchResultTy parseOptionalOps(
517                                    const ArrayRef<OptionalOperand> &OptionalOps,
518                                    OperandVector &Operands);
519 
520 
521   void cvtDSOffset01(MCInst &Inst, const OperandVector &Operands);
522   void cvtDS(MCInst &Inst, const OperandVector &Operands);
523   OperandMatchResultTy parseDSOptionalOps(OperandVector &Operands);
524   OperandMatchResultTy parseDSOff01OptionalOps(OperandVector &Operands);
525   OperandMatchResultTy parseDSOffsetOptional(OperandVector &Operands);
526 
527   bool parseCnt(int64_t &IntVal);
528   OperandMatchResultTy parseSWaitCntOps(OperandVector &Operands);
529   OperandMatchResultTy parseSOppBrTarget(OperandVector &Operands);
530 
531   OperandMatchResultTy parseFlatOptionalOps(OperandVector &Operands);
532   OperandMatchResultTy parseFlatAtomicOptionalOps(OperandVector &Operands);
533   void cvtFlat(MCInst &Inst, const OperandVector &Operands);
534   void cvtFlatAtomic(MCInst &Inst, const OperandVector &Operands);
535 
536   void cvtMubuf(MCInst &Inst, const OperandVector &Operands);
537   OperandMatchResultTy parseOffset(OperandVector &Operands);
538   OperandMatchResultTy parseMubufOptionalOps(OperandVector &Operands);
539   OperandMatchResultTy parseGLC(OperandVector &Operands);
540   OperandMatchResultTy parseSLC(OperandVector &Operands);
541   OperandMatchResultTy parseTFE(OperandVector &Operands);
542 
543   OperandMatchResultTy parseDMask(OperandVector &Operands);
544   OperandMatchResultTy parseUNorm(OperandVector &Operands);
545   OperandMatchResultTy parseDA(OperandVector &Operands);
546   OperandMatchResultTy parseR128(OperandVector &Operands);
547   OperandMatchResultTy parseLWE(OperandVector &Operands);
548 
549   void cvtId(MCInst &Inst, const OperandVector &Operands);
550   void cvtVOP3_2_mod(MCInst &Inst, const OperandVector &Operands);
551   void cvtVOP3_2_nomod(MCInst &Inst, const OperandVector &Operands);
552   void cvtVOP3_only(MCInst &Inst, const OperandVector &Operands);
553   void cvtVOP3(MCInst &Inst, const OperandVector &Operands);
554 
555   void cvtMIMG(MCInst &Inst, const OperandVector &Operands);
556   void cvtMIMGAtomic(MCInst &Inst, const OperandVector &Operands);
557   OperandMatchResultTy parseVOP3OptionalOps(OperandVector &Operands);
558 
559   OperandMatchResultTy parseDPPCtrlOps(OperandVector &Operands);
560   OperandMatchResultTy parseDPPOptionalOps(OperandVector &Operands);
561   void cvtDPP_mod(MCInst &Inst, const OperandVector &Operands);
562   void cvtDPP_nomod(MCInst &Inst, const OperandVector &Operands);
563   void cvtDPP(MCInst &Inst, const OperandVector &Operands, bool HasMods);
564 };
565 
566 struct OptionalOperand {
567   const char *Name;
568   AMDGPUOperand::ImmTy Type;
569   bool IsBit;
570   int64_t Default;
571   bool (*ConvertResult)(int64_t&);
572 };
573 
574 }
575 
576 enum  RegisterKind { IS_VGPR, IS_SGPR, IS_TTMP };
577 
578 static int getRegClass(RegisterKind Is, unsigned RegWidth) {
579   if (Is == IS_VGPR) {
580     switch (RegWidth) {
581       default: return -1;
582       case 1: return AMDGPU::VGPR_32RegClassID;
583       case 2: return AMDGPU::VReg_64RegClassID;
584       case 3: return AMDGPU::VReg_96RegClassID;
585       case 4: return AMDGPU::VReg_128RegClassID;
586       case 8: return AMDGPU::VReg_256RegClassID;
587       case 16: return AMDGPU::VReg_512RegClassID;
588     }
589   } else if (Is == IS_TTMP) {
590     switch (RegWidth) {
591       default: return -1;
592       case 1: return AMDGPU::TTMP_32RegClassID;
593       case 2: return AMDGPU::TTMP_64RegClassID;
594     }
595   } else if (Is == IS_SGPR) {
596     switch (RegWidth) {
597       default: return -1;
598       case 1: return AMDGPU::SGPR_32RegClassID;
599       case 2: return AMDGPU::SGPR_64RegClassID;
600       case 4: return AMDGPU::SReg_128RegClassID;
601       case 8: return AMDGPU::SReg_256RegClassID;
602       case 16: return AMDGPU::SReg_512RegClassID;
603     }
604   }
605   return -1;
606 }
607 
608 static unsigned getRegForName(StringRef RegName) {
609 
610   return StringSwitch<unsigned>(RegName)
611     .Case("exec", AMDGPU::EXEC)
612     .Case("vcc", AMDGPU::VCC)
613     .Case("flat_scratch", AMDGPU::FLAT_SCR)
614     .Case("m0", AMDGPU::M0)
615     .Case("scc", AMDGPU::SCC)
616     .Case("flat_scratch_lo", AMDGPU::FLAT_SCR_LO)
617     .Case("flat_scratch_hi", AMDGPU::FLAT_SCR_HI)
618     .Case("vcc_lo", AMDGPU::VCC_LO)
619     .Case("vcc_hi", AMDGPU::VCC_HI)
620     .Case("exec_lo", AMDGPU::EXEC_LO)
621     .Case("exec_hi", AMDGPU::EXEC_HI)
622     .Case("tma_lo", AMDGPU::TMA_LO)
623     .Case("tma_hi", AMDGPU::TMA_HI)
624     .Case("tba_lo", AMDGPU::TBA_LO)
625     .Case("tba_hi", AMDGPU::TBA_HI)
626     .Default(0);
627 }
628 
629 bool AMDGPUAsmParser::ParseRegister(unsigned &RegNo, SMLoc &StartLoc, SMLoc &EndLoc) {
630   auto R = parseRegister();
631   if (!R) return true;
632   assert(R->isReg());
633   RegNo = R->getReg();
634   StartLoc = R->getStartLoc();
635   EndLoc = R->getEndLoc();
636   return false;
637 }
638 
639 std::unique_ptr<AMDGPUOperand> AMDGPUAsmParser::parseRegister() {
640   const AsmToken &Tok = Parser.getTok();
641   SMLoc StartLoc = Tok.getLoc();
642   SMLoc EndLoc = Tok.getEndLoc();
643   const MCRegisterInfo *TRI = getContext().getRegisterInfo();
644 
645   StringRef RegName = Tok.getString();
646   unsigned RegNo = getRegForName(RegName);
647 
648   if (RegNo) {
649     Parser.Lex();
650     if (!subtargetHasRegister(*TRI, RegNo))
651       return nullptr;
652     return AMDGPUOperand::CreateReg(RegNo, StartLoc, EndLoc,
653                                     TRI, &getSTI(), false);
654   }
655 
656   // Match vgprs, sgprs and ttmps
657   if (RegName[0] != 's' && RegName[0] != 'v' && !RegName.startswith("ttmp"))
658     return nullptr;
659 
660   const RegisterKind Is = RegName[0] == 'v' ? IS_VGPR : RegName[0] == 's' ? IS_SGPR : IS_TTMP;
661   unsigned RegWidth;
662   unsigned RegIndexInClass;
663   if (RegName.size() > (Is == IS_TTMP ? strlen("ttmp") : 1) ) {
664     // We have a single 32-bit register. Syntax: vXX
665     RegWidth = 1;
666     if (RegName.substr(Is == IS_TTMP ? strlen("ttmp") : 1).getAsInteger(10, RegIndexInClass))
667       return nullptr;
668     Parser.Lex();
669   } else {
670     // We have a register greater than 32-bits (a range of single registers). Syntax: v[XX:YY]
671 
672     int64_t RegLo, RegHi;
673     Parser.Lex();
674     if (getLexer().isNot(AsmToken::LBrac))
675       return nullptr;
676 
677     Parser.Lex();
678     if (getParser().parseAbsoluteExpression(RegLo))
679       return nullptr;
680 
681     if (getLexer().isNot(AsmToken::Colon))
682       return nullptr;
683 
684     Parser.Lex();
685     if (getParser().parseAbsoluteExpression(RegHi))
686       return nullptr;
687 
688     if (getLexer().isNot(AsmToken::RBrac))
689       return nullptr;
690 
691     Parser.Lex();
692     RegWidth = (RegHi - RegLo) + 1;
693     if (Is == IS_VGPR) {
694       // VGPR registers aren't aligned.
695       RegIndexInClass = RegLo;
696     } else {
697       // SGPR and TTMP registers must be are aligned. Max required alignment is 4 dwords.
698       unsigned Size = std::min(RegWidth, 4u);
699       if (RegLo % Size != 0)
700         return nullptr;
701 
702       RegIndexInClass = RegLo / Size;
703     }
704   }
705 
706   int RCID = getRegClass(Is, RegWidth);
707   if (RCID == -1)
708     return nullptr;
709 
710   const MCRegisterClass RC = TRI->getRegClass(RCID);
711   if (RegIndexInClass >= RC.getNumRegs())
712     return nullptr;
713 
714   RegNo = RC.getRegister(RegIndexInClass);
715   if (!subtargetHasRegister(*TRI, RegNo))
716     return nullptr;
717 
718   return AMDGPUOperand::CreateReg(RegNo, StartLoc, EndLoc,
719                                   TRI, &getSTI(), false);
720 }
721 
722 unsigned AMDGPUAsmParser::checkTargetMatchPredicate(MCInst &Inst) {
723 
724   uint64_t TSFlags = MII.get(Inst.getOpcode()).TSFlags;
725 
726   if ((getForcedEncodingSize() == 32 && (TSFlags & SIInstrFlags::VOP3)) ||
727       (getForcedEncodingSize() == 64 && !(TSFlags & SIInstrFlags::VOP3)))
728     return Match_InvalidOperand;
729 
730   if ((TSFlags & SIInstrFlags::VOP3) &&
731       (TSFlags & SIInstrFlags::VOPAsmPrefer32Bit) &&
732       getForcedEncodingSize() != 64)
733     return Match_PreferE32;
734 
735   return Match_Success;
736 }
737 
738 
739 bool AMDGPUAsmParser::MatchAndEmitInstruction(SMLoc IDLoc, unsigned &Opcode,
740                                               OperandVector &Operands,
741                                               MCStreamer &Out,
742                                               uint64_t &ErrorInfo,
743                                               bool MatchingInlineAsm) {
744   MCInst Inst;
745 
746   switch (MatchInstructionImpl(Operands, Inst, ErrorInfo, MatchingInlineAsm)) {
747     default: break;
748     case Match_Success:
749       Inst.setLoc(IDLoc);
750       Out.EmitInstruction(Inst, getSTI());
751       return false;
752     case Match_MissingFeature:
753       return Error(IDLoc, "instruction not supported on this GPU");
754 
755     case Match_MnemonicFail:
756       return Error(IDLoc, "unrecognized instruction mnemonic");
757 
758     case Match_InvalidOperand: {
759       SMLoc ErrorLoc = IDLoc;
760       if (ErrorInfo != ~0ULL) {
761         if (ErrorInfo >= Operands.size()) {
762           return Error(IDLoc, "too few operands for instruction");
763         }
764         ErrorLoc = ((AMDGPUOperand &)*Operands[ErrorInfo]).getStartLoc();
765         if (ErrorLoc == SMLoc())
766           ErrorLoc = IDLoc;
767       }
768       return Error(ErrorLoc, "invalid operand for instruction");
769     }
770     case Match_PreferE32:
771       return Error(IDLoc, "internal error: instruction without _e64 suffix "
772                           "should be encoded as e32");
773   }
774   llvm_unreachable("Implement any new match types added!");
775 }
776 
777 bool AMDGPUAsmParser::ParseDirectiveMajorMinor(uint32_t &Major,
778                                                uint32_t &Minor) {
779   if (getLexer().isNot(AsmToken::Integer))
780     return TokError("invalid major version");
781 
782   Major = getLexer().getTok().getIntVal();
783   Lex();
784 
785   if (getLexer().isNot(AsmToken::Comma))
786     return TokError("minor version number required, comma expected");
787   Lex();
788 
789   if (getLexer().isNot(AsmToken::Integer))
790     return TokError("invalid minor version");
791 
792   Minor = getLexer().getTok().getIntVal();
793   Lex();
794 
795   return false;
796 }
797 
798 bool AMDGPUAsmParser::ParseDirectiveHSACodeObjectVersion() {
799 
800   uint32_t Major;
801   uint32_t Minor;
802 
803   if (ParseDirectiveMajorMinor(Major, Minor))
804     return true;
805 
806   getTargetStreamer().EmitDirectiveHSACodeObjectVersion(Major, Minor);
807   return false;
808 }
809 
810 bool AMDGPUAsmParser::ParseDirectiveHSACodeObjectISA() {
811 
812   uint32_t Major;
813   uint32_t Minor;
814   uint32_t Stepping;
815   StringRef VendorName;
816   StringRef ArchName;
817 
818   // If this directive has no arguments, then use the ISA version for the
819   // targeted GPU.
820   if (getLexer().is(AsmToken::EndOfStatement)) {
821     AMDGPU::IsaVersion Isa = AMDGPU::getIsaVersion(getSTI().getFeatureBits());
822     getTargetStreamer().EmitDirectiveHSACodeObjectISA(Isa.Major, Isa.Minor,
823                                                       Isa.Stepping,
824                                                       "AMD", "AMDGPU");
825     return false;
826   }
827 
828 
829   if (ParseDirectiveMajorMinor(Major, Minor))
830     return true;
831 
832   if (getLexer().isNot(AsmToken::Comma))
833     return TokError("stepping version number required, comma expected");
834   Lex();
835 
836   if (getLexer().isNot(AsmToken::Integer))
837     return TokError("invalid stepping version");
838 
839   Stepping = getLexer().getTok().getIntVal();
840   Lex();
841 
842   if (getLexer().isNot(AsmToken::Comma))
843     return TokError("vendor name required, comma expected");
844   Lex();
845 
846   if (getLexer().isNot(AsmToken::String))
847     return TokError("invalid vendor name");
848 
849   VendorName = getLexer().getTok().getStringContents();
850   Lex();
851 
852   if (getLexer().isNot(AsmToken::Comma))
853     return TokError("arch name required, comma expected");
854   Lex();
855 
856   if (getLexer().isNot(AsmToken::String))
857     return TokError("invalid arch name");
858 
859   ArchName = getLexer().getTok().getStringContents();
860   Lex();
861 
862   getTargetStreamer().EmitDirectiveHSACodeObjectISA(Major, Minor, Stepping,
863                                                     VendorName, ArchName);
864   return false;
865 }
866 
867 bool AMDGPUAsmParser::ParseAMDKernelCodeTValue(StringRef ID,
868                                                amd_kernel_code_t &Header) {
869   SmallString<40> ErrStr;
870   raw_svector_ostream Err(ErrStr);
871   if (!parseAmdKernelCodeField(ID, getLexer(), Header, Err)) {
872     return TokError(Err.str());
873   }
874   Lex();
875   return false;
876 }
877 
878 bool AMDGPUAsmParser::ParseDirectiveAMDKernelCodeT() {
879 
880   amd_kernel_code_t Header;
881   AMDGPU::initDefaultAMDKernelCodeT(Header, getSTI().getFeatureBits());
882 
883   while (true) {
884 
885     if (getLexer().isNot(AsmToken::EndOfStatement))
886       return TokError("amd_kernel_code_t values must begin on a new line");
887 
888     // Lex EndOfStatement.  This is in a while loop, because lexing a comment
889     // will set the current token to EndOfStatement.
890     while(getLexer().is(AsmToken::EndOfStatement))
891       Lex();
892 
893     if (getLexer().isNot(AsmToken::Identifier))
894       return TokError("expected value identifier or .end_amd_kernel_code_t");
895 
896     StringRef ID = getLexer().getTok().getIdentifier();
897     Lex();
898 
899     if (ID == ".end_amd_kernel_code_t")
900       break;
901 
902     if (ParseAMDKernelCodeTValue(ID, Header))
903       return true;
904   }
905 
906   getTargetStreamer().EmitAMDKernelCodeT(Header);
907 
908   return false;
909 }
910 
911 bool AMDGPUAsmParser::ParseSectionDirectiveHSAText() {
912   getParser().getStreamer().SwitchSection(
913       AMDGPU::getHSATextSection(getContext()));
914   return false;
915 }
916 
917 bool AMDGPUAsmParser::ParseDirectiveAMDGPUHsaKernel() {
918   if (getLexer().isNot(AsmToken::Identifier))
919     return TokError("expected symbol name");
920 
921   StringRef KernelName = Parser.getTok().getString();
922 
923   getTargetStreamer().EmitAMDGPUSymbolType(KernelName,
924                                            ELF::STT_AMDGPU_HSA_KERNEL);
925   Lex();
926   return false;
927 }
928 
929 bool AMDGPUAsmParser::ParseDirectiveAMDGPUHsaModuleGlobal() {
930   if (getLexer().isNot(AsmToken::Identifier))
931     return TokError("expected symbol name");
932 
933   StringRef GlobalName = Parser.getTok().getIdentifier();
934 
935   getTargetStreamer().EmitAMDGPUHsaModuleScopeGlobal(GlobalName);
936   Lex();
937   return false;
938 }
939 
940 bool AMDGPUAsmParser::ParseDirectiveAMDGPUHsaProgramGlobal() {
941   if (getLexer().isNot(AsmToken::Identifier))
942     return TokError("expected symbol name");
943 
944   StringRef GlobalName = Parser.getTok().getIdentifier();
945 
946   getTargetStreamer().EmitAMDGPUHsaProgramScopeGlobal(GlobalName);
947   Lex();
948   return false;
949 }
950 
951 bool AMDGPUAsmParser::ParseSectionDirectiveHSADataGlobalAgent() {
952   getParser().getStreamer().SwitchSection(
953       AMDGPU::getHSADataGlobalAgentSection(getContext()));
954   return false;
955 }
956 
957 bool AMDGPUAsmParser::ParseSectionDirectiveHSADataGlobalProgram() {
958   getParser().getStreamer().SwitchSection(
959       AMDGPU::getHSADataGlobalProgramSection(getContext()));
960   return false;
961 }
962 
963 bool AMDGPUAsmParser::ParseSectionDirectiveHSARodataReadonlyAgent() {
964   getParser().getStreamer().SwitchSection(
965       AMDGPU::getHSARodataReadonlyAgentSection(getContext()));
966   return false;
967 }
968 
969 bool AMDGPUAsmParser::ParseDirective(AsmToken DirectiveID) {
970   StringRef IDVal = DirectiveID.getString();
971 
972   if (IDVal == ".hsa_code_object_version")
973     return ParseDirectiveHSACodeObjectVersion();
974 
975   if (IDVal == ".hsa_code_object_isa")
976     return ParseDirectiveHSACodeObjectISA();
977 
978   if (IDVal == ".amd_kernel_code_t")
979     return ParseDirectiveAMDKernelCodeT();
980 
981   if (IDVal == ".hsatext" || IDVal == ".text")
982     return ParseSectionDirectiveHSAText();
983 
984   if (IDVal == ".amdgpu_hsa_kernel")
985     return ParseDirectiveAMDGPUHsaKernel();
986 
987   if (IDVal == ".amdgpu_hsa_module_global")
988     return ParseDirectiveAMDGPUHsaModuleGlobal();
989 
990   if (IDVal == ".amdgpu_hsa_program_global")
991     return ParseDirectiveAMDGPUHsaProgramGlobal();
992 
993   if (IDVal == ".hsadata_global_agent")
994     return ParseSectionDirectiveHSADataGlobalAgent();
995 
996   if (IDVal == ".hsadata_global_program")
997     return ParseSectionDirectiveHSADataGlobalProgram();
998 
999   if (IDVal == ".hsarodata_readonly_agent")
1000     return ParseSectionDirectiveHSARodataReadonlyAgent();
1001 
1002   return true;
1003 }
1004 
1005 bool AMDGPUAsmParser::subtargetHasRegister(const MCRegisterInfo &MRI,
1006                                            unsigned RegNo) const {
1007   if (isCI())
1008     return true;
1009 
1010   if (isSI()) {
1011     // No flat_scr
1012     switch (RegNo) {
1013     case AMDGPU::FLAT_SCR:
1014     case AMDGPU::FLAT_SCR_LO:
1015     case AMDGPU::FLAT_SCR_HI:
1016       return false;
1017     default:
1018       return true;
1019     }
1020   }
1021 
1022   // VI only has 102 SGPRs, so make sure we aren't trying to use the 2 more that
1023   // SI/CI have.
1024   for (MCRegAliasIterator R(AMDGPU::SGPR102_SGPR103, &MRI, true);
1025        R.isValid(); ++R) {
1026     if (*R == RegNo)
1027       return false;
1028   }
1029 
1030   return true;
1031 }
1032 
1033 static bool operandsHaveModifiers(const OperandVector &Operands) {
1034 
1035   for (unsigned i = 0, e = Operands.size(); i != e; ++i) {
1036     const AMDGPUOperand &Op = ((AMDGPUOperand&)*Operands[i]);
1037     if (Op.isRegKind() && Op.hasModifiers())
1038       return true;
1039     if (Op.isImm() && Op.hasModifiers())
1040       return true;
1041     if (Op.isImm() && (Op.getImmTy() == AMDGPUOperand::ImmTyOMod ||
1042                        Op.getImmTy() == AMDGPUOperand::ImmTyClamp))
1043       return true;
1044   }
1045   return false;
1046 }
1047 
1048 AMDGPUAsmParser::OperandMatchResultTy
1049 AMDGPUAsmParser::parseOperand(OperandVector &Operands, StringRef Mnemonic) {
1050 
1051   // Try to parse with a custom parser
1052   OperandMatchResultTy ResTy = MatchOperandParserImpl(Operands, Mnemonic);
1053 
1054   // If we successfully parsed the operand or if there as an error parsing,
1055   // we are done.
1056   //
1057   // If we are parsing after we reach EndOfStatement then this means we
1058   // are appending default values to the Operands list.  This is only done
1059   // by custom parser, so we shouldn't continue on to the generic parsing.
1060   if (ResTy == MatchOperand_Success || ResTy == MatchOperand_ParseFail||
1061       getLexer().is(AsmToken::EndOfStatement))
1062     return ResTy;
1063 
1064   bool Negate = false, Abs = false, Abs2 = false;
1065 
1066   if (getLexer().getKind()== AsmToken::Minus) {
1067     Parser.Lex();
1068     Negate = true;
1069   }
1070 
1071   if (getLexer().getKind() == AsmToken::Identifier && Parser.getTok().getString() == "abs") {
1072     Parser.Lex();
1073     Abs2 = true;
1074     if (getLexer().isNot(AsmToken::LParen)) {
1075       Error(Parser.getTok().getLoc(), "expected left paren after abs");
1076       return MatchOperand_ParseFail;
1077     }
1078     Parser.Lex();
1079   }
1080 
1081   if (getLexer().getKind() == AsmToken::Pipe) {
1082     Parser.Lex();
1083     Abs = true;
1084   }
1085 
1086   switch(getLexer().getKind()) {
1087     case AsmToken::Integer: {
1088       SMLoc S = Parser.getTok().getLoc();
1089       int64_t IntVal;
1090       if (getParser().parseAbsoluteExpression(IntVal))
1091         return MatchOperand_ParseFail;
1092       if (!isInt<32>(IntVal) && !isUInt<32>(IntVal)) {
1093         Error(S, "invalid immediate: only 32-bit values are legal");
1094         return MatchOperand_ParseFail;
1095       }
1096 
1097       if (Negate)
1098         IntVal *= -1;
1099       Operands.push_back(AMDGPUOperand::CreateImm(IntVal, S));
1100       return MatchOperand_Success;
1101     }
1102     case AsmToken::Real: {
1103       // FIXME: We should emit an error if a double precisions floating-point
1104       // value is used.  I'm not sure the best way to detect this.
1105       SMLoc S = Parser.getTok().getLoc();
1106       int64_t IntVal;
1107       if (getParser().parseAbsoluteExpression(IntVal))
1108         return MatchOperand_ParseFail;
1109 
1110       APFloat F((float)BitsToDouble(IntVal));
1111       if (Negate)
1112         F.changeSign();
1113       Operands.push_back(
1114           AMDGPUOperand::CreateImm(F.bitcastToAPInt().getZExtValue(), S));
1115       return MatchOperand_Success;
1116     }
1117     case AsmToken::Identifier: {
1118       if (auto R = parseRegister()) {
1119         unsigned Modifiers = 0;
1120 
1121         if (Negate)
1122           Modifiers |= 0x1;
1123 
1124         if (Abs) {
1125           if (getLexer().getKind() != AsmToken::Pipe)
1126             return MatchOperand_ParseFail;
1127           Parser.Lex();
1128           Modifiers |= 0x2;
1129         }
1130         if (Abs2) {
1131           if (getLexer().isNot(AsmToken::RParen)) {
1132             return MatchOperand_ParseFail;
1133           }
1134           Parser.Lex();
1135           Modifiers |= 0x2;
1136         }
1137         assert(R->isReg());
1138         R->Reg.IsForcedVOP3 = isForcedVOP3();
1139         if (Modifiers) {
1140           R->setModifiers(Modifiers);
1141         }
1142         Operands.push_back(std::move(R));
1143       } else {
1144         ResTy = parseVOP3OptionalOps(Operands);
1145         if (ResTy == MatchOperand_NoMatch) {
1146           const auto &Tok = Parser.getTok();
1147           Operands.push_back(AMDGPUOperand::CreateToken(Tok.getString(),
1148                                                         Tok.getLoc()));
1149           Parser.Lex();
1150         }
1151       }
1152       return MatchOperand_Success;
1153     }
1154     default:
1155       return MatchOperand_NoMatch;
1156   }
1157 }
1158 
1159 bool AMDGPUAsmParser::ParseInstruction(ParseInstructionInfo &Info,
1160                                        StringRef Name,
1161                                        SMLoc NameLoc, OperandVector &Operands) {
1162 
1163   // Clear any forced encodings from the previous instruction.
1164   setForcedEncodingSize(0);
1165 
1166   if (Name.endswith("_e64"))
1167     setForcedEncodingSize(64);
1168   else if (Name.endswith("_e32"))
1169     setForcedEncodingSize(32);
1170 
1171   // Add the instruction mnemonic
1172   Operands.push_back(AMDGPUOperand::CreateToken(Name, NameLoc));
1173 
1174   while (!getLexer().is(AsmToken::EndOfStatement)) {
1175     AMDGPUAsmParser::OperandMatchResultTy Res = parseOperand(Operands, Name);
1176 
1177     // Eat the comma or space if there is one.
1178     if (getLexer().is(AsmToken::Comma))
1179       Parser.Lex();
1180 
1181     switch (Res) {
1182       case MatchOperand_Success: break;
1183       case MatchOperand_ParseFail: return Error(getLexer().getLoc(),
1184                                                 "failed parsing operand.");
1185       case MatchOperand_NoMatch: return Error(getLexer().getLoc(),
1186                                               "not a valid operand.");
1187     }
1188   }
1189 
1190   return false;
1191 }
1192 
1193 //===----------------------------------------------------------------------===//
1194 // Utility functions
1195 //===----------------------------------------------------------------------===//
1196 
1197 AMDGPUAsmParser::OperandMatchResultTy
1198 AMDGPUAsmParser::parseIntWithPrefix(const char *Prefix, int64_t &Int,
1199                                     int64_t Default) {
1200   // We are at the end of the statement, and this is a default argument, so
1201   // use a default value.
1202   if (getLexer().is(AsmToken::EndOfStatement)) {
1203     Int = Default;
1204     return MatchOperand_Success;
1205   }
1206 
1207   switch(getLexer().getKind()) {
1208     default: return MatchOperand_NoMatch;
1209     case AsmToken::Identifier: {
1210       StringRef OffsetName = Parser.getTok().getString();
1211       if (!OffsetName.equals(Prefix))
1212         return MatchOperand_NoMatch;
1213 
1214       Parser.Lex();
1215       if (getLexer().isNot(AsmToken::Colon))
1216         return MatchOperand_ParseFail;
1217 
1218       Parser.Lex();
1219       if (getLexer().isNot(AsmToken::Integer))
1220         return MatchOperand_ParseFail;
1221 
1222       if (getParser().parseAbsoluteExpression(Int))
1223         return MatchOperand_ParseFail;
1224       break;
1225     }
1226   }
1227   return MatchOperand_Success;
1228 }
1229 
1230 AMDGPUAsmParser::OperandMatchResultTy
1231 AMDGPUAsmParser::parseIntWithPrefix(const char *Prefix, OperandVector &Operands,
1232                                     enum AMDGPUOperand::ImmTy ImmTy) {
1233 
1234   SMLoc S = Parser.getTok().getLoc();
1235   int64_t Offset = 0;
1236 
1237   AMDGPUAsmParser::OperandMatchResultTy Res = parseIntWithPrefix(Prefix, Offset);
1238   if (Res != MatchOperand_Success)
1239     return Res;
1240 
1241   Operands.push_back(AMDGPUOperand::CreateImm(Offset, S, ImmTy));
1242   return MatchOperand_Success;
1243 }
1244 
1245 AMDGPUAsmParser::OperandMatchResultTy
1246 AMDGPUAsmParser::parseNamedBit(const char *Name, OperandVector &Operands,
1247                                enum AMDGPUOperand::ImmTy ImmTy) {
1248   int64_t Bit = 0;
1249   SMLoc S = Parser.getTok().getLoc();
1250 
1251   // We are at the end of the statement, and this is a default argument, so
1252   // use a default value.
1253   if (getLexer().isNot(AsmToken::EndOfStatement)) {
1254     switch(getLexer().getKind()) {
1255       case AsmToken::Identifier: {
1256         StringRef Tok = Parser.getTok().getString();
1257         if (Tok == Name) {
1258           Bit = 1;
1259           Parser.Lex();
1260         } else if (Tok.startswith("no") && Tok.endswith(Name)) {
1261           Bit = 0;
1262           Parser.Lex();
1263         } else {
1264           return MatchOperand_NoMatch;
1265         }
1266         break;
1267       }
1268       default:
1269         return MatchOperand_NoMatch;
1270     }
1271   }
1272 
1273   Operands.push_back(AMDGPUOperand::CreateImm(Bit, S, ImmTy));
1274   return MatchOperand_Success;
1275 }
1276 
1277 typedef std::map<enum AMDGPUOperand::ImmTy, unsigned> OptionalImmIndexMap;
1278 
1279 void addOptionalImmOperand(MCInst& Inst, const OperandVector& Operands,
1280                            OptionalImmIndexMap& OptionalIdx,
1281                            enum AMDGPUOperand::ImmTy ImmT, int64_t Default = 0) {
1282   auto i = OptionalIdx.find(ImmT);
1283   if (i != OptionalIdx.end()) {
1284     unsigned Idx = i->second;
1285     ((AMDGPUOperand &)*Operands[Idx]).addImmOperands(Inst, 1);
1286   } else {
1287     Inst.addOperand(MCOperand::createImm(Default));
1288   }
1289 }
1290 
1291 static bool operandsHasOptionalOp(const OperandVector &Operands,
1292                                   const OptionalOperand &OOp) {
1293   for (unsigned i = 0; i < Operands.size(); i++) {
1294     const AMDGPUOperand &ParsedOp = ((const AMDGPUOperand &)*Operands[i]);
1295     if ((ParsedOp.isImm() && ParsedOp.getImmTy() == OOp.Type) ||
1296         (ParsedOp.isToken() && ParsedOp.getToken() == OOp.Name))
1297       return true;
1298 
1299   }
1300   return false;
1301 }
1302 
1303 AMDGPUAsmParser::OperandMatchResultTy
1304 AMDGPUAsmParser::parseOptionalOps(const ArrayRef<OptionalOperand> &OptionalOps,
1305                                    OperandVector &Operands) {
1306   SMLoc S = Parser.getTok().getLoc();
1307   for (const OptionalOperand &Op : OptionalOps) {
1308     if (operandsHasOptionalOp(Operands, Op))
1309       continue;
1310     AMDGPUAsmParser::OperandMatchResultTy Res;
1311     int64_t Value;
1312     if (Op.IsBit) {
1313       Res = parseNamedBit(Op.Name, Operands, Op.Type);
1314       if (Res == MatchOperand_NoMatch)
1315         continue;
1316       return Res;
1317     }
1318 
1319     Res = parseIntWithPrefix(Op.Name, Value, Op.Default);
1320 
1321     if (Res == MatchOperand_NoMatch)
1322       continue;
1323 
1324     if (Res != MatchOperand_Success)
1325       return Res;
1326 
1327     bool DefaultValue = (Value == Op.Default);
1328 
1329     if (Op.ConvertResult && !Op.ConvertResult(Value)) {
1330       return MatchOperand_ParseFail;
1331     }
1332 
1333     if (!DefaultValue) {
1334       Operands.push_back(AMDGPUOperand::CreateImm(Value, S, Op.Type));
1335     }
1336     return MatchOperand_Success;
1337   }
1338   return MatchOperand_NoMatch;
1339 }
1340 
1341 //===----------------------------------------------------------------------===//
1342 // ds
1343 //===----------------------------------------------------------------------===//
1344 
1345 static const OptionalOperand DSOptionalOps [] = {
1346   {"offset",  AMDGPUOperand::ImmTyOffset, false, 0, nullptr},
1347   {"gds",     AMDGPUOperand::ImmTyGDS, true, 0, nullptr}
1348 };
1349 
1350 static const OptionalOperand DSOptionalOpsOff01 [] = {
1351   {"offset0", AMDGPUOperand::ImmTyDSOffset0, false, 0, nullptr},
1352   {"offset1", AMDGPUOperand::ImmTyDSOffset1, false, 0, nullptr},
1353   {"gds",     AMDGPUOperand::ImmTyGDS, true, 0, nullptr}
1354 };
1355 
1356 AMDGPUAsmParser::OperandMatchResultTy
1357 AMDGPUAsmParser::parseDSOptionalOps(OperandVector &Operands) {
1358   return parseOptionalOps(DSOptionalOps, Operands);
1359 }
1360 AMDGPUAsmParser::OperandMatchResultTy
1361 AMDGPUAsmParser::parseDSOff01OptionalOps(OperandVector &Operands) {
1362   return parseOptionalOps(DSOptionalOpsOff01, Operands);
1363 }
1364 
1365 AMDGPUAsmParser::OperandMatchResultTy
1366 AMDGPUAsmParser::parseDSOffsetOptional(OperandVector &Operands) {
1367   SMLoc S = Parser.getTok().getLoc();
1368   AMDGPUAsmParser::OperandMatchResultTy Res =
1369     parseIntWithPrefix("offset", Operands, AMDGPUOperand::ImmTyOffset);
1370   if (Res == MatchOperand_NoMatch) {
1371     Operands.push_back(AMDGPUOperand::CreateImm(0, S,
1372                        AMDGPUOperand::ImmTyOffset));
1373     Res = MatchOperand_Success;
1374   }
1375   return Res;
1376 }
1377 
1378 bool AMDGPUOperand::isDSOffset() const {
1379   return isImm() && isUInt<16>(getImm());
1380 }
1381 
1382 bool AMDGPUOperand::isDSOffset01() const {
1383   return isImm() && isUInt<8>(getImm());
1384 }
1385 
1386 void AMDGPUAsmParser::cvtDSOffset01(MCInst &Inst,
1387                                     const OperandVector &Operands) {
1388 
1389   OptionalImmIndexMap OptionalIdx;
1390 
1391   for (unsigned i = 1, e = Operands.size(); i != e; ++i) {
1392     AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[i]);
1393 
1394     // Add the register arguments
1395     if (Op.isReg()) {
1396       Op.addRegOperands(Inst, 1);
1397       continue;
1398     }
1399 
1400     // Handle optional arguments
1401     OptionalIdx[Op.getImmTy()] = i;
1402   }
1403 
1404   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyDSOffset0);
1405   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyDSOffset1);
1406   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyGDS);
1407 
1408   Inst.addOperand(MCOperand::createReg(AMDGPU::M0)); // m0
1409 }
1410 
1411 void AMDGPUAsmParser::cvtDS(MCInst &Inst, const OperandVector &Operands) {
1412 
1413   std::map<enum AMDGPUOperand::ImmTy, unsigned> OptionalIdx;
1414   bool GDSOnly = false;
1415 
1416   for (unsigned i = 1, e = Operands.size(); i != e; ++i) {
1417     AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[i]);
1418 
1419     // Add the register arguments
1420     if (Op.isReg()) {
1421       Op.addRegOperands(Inst, 1);
1422       continue;
1423     }
1424 
1425     if (Op.isToken() && Op.getToken() == "gds") {
1426       GDSOnly = true;
1427       continue;
1428     }
1429 
1430     // Handle optional arguments
1431     OptionalIdx[Op.getImmTy()] = i;
1432   }
1433 
1434   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyOffset);
1435   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyGDS);
1436 
1437   if (!GDSOnly) {
1438     addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyGDS);
1439   }
1440   Inst.addOperand(MCOperand::createReg(AMDGPU::M0)); // m0
1441 }
1442 
1443 
1444 //===----------------------------------------------------------------------===//
1445 // s_waitcnt
1446 //===----------------------------------------------------------------------===//
1447 
1448 bool AMDGPUAsmParser::parseCnt(int64_t &IntVal) {
1449   StringRef CntName = Parser.getTok().getString();
1450   int64_t CntVal;
1451 
1452   Parser.Lex();
1453   if (getLexer().isNot(AsmToken::LParen))
1454     return true;
1455 
1456   Parser.Lex();
1457   if (getLexer().isNot(AsmToken::Integer))
1458     return true;
1459 
1460   if (getParser().parseAbsoluteExpression(CntVal))
1461     return true;
1462 
1463   if (getLexer().isNot(AsmToken::RParen))
1464     return true;
1465 
1466   Parser.Lex();
1467   if (getLexer().is(AsmToken::Amp) || getLexer().is(AsmToken::Comma))
1468     Parser.Lex();
1469 
1470   int CntShift;
1471   int CntMask;
1472 
1473   if (CntName == "vmcnt") {
1474     CntMask = 0xf;
1475     CntShift = 0;
1476   } else if (CntName == "expcnt") {
1477     CntMask = 0x7;
1478     CntShift = 4;
1479   } else if (CntName == "lgkmcnt") {
1480     CntMask = 0xf;
1481     CntShift = 8;
1482   } else {
1483     return true;
1484   }
1485 
1486   IntVal &= ~(CntMask << CntShift);
1487   IntVal |= (CntVal << CntShift);
1488   return false;
1489 }
1490 
1491 AMDGPUAsmParser::OperandMatchResultTy
1492 AMDGPUAsmParser::parseSWaitCntOps(OperandVector &Operands) {
1493   // Disable all counters by default.
1494   // vmcnt   [3:0]
1495   // expcnt  [6:4]
1496   // lgkmcnt [11:8]
1497   int64_t CntVal = 0xf7f;
1498   SMLoc S = Parser.getTok().getLoc();
1499 
1500   switch(getLexer().getKind()) {
1501     default: return MatchOperand_ParseFail;
1502     case AsmToken::Integer:
1503       // The operand can be an integer value.
1504       if (getParser().parseAbsoluteExpression(CntVal))
1505         return MatchOperand_ParseFail;
1506       break;
1507 
1508     case AsmToken::Identifier:
1509       do {
1510         if (parseCnt(CntVal))
1511           return MatchOperand_ParseFail;
1512       } while(getLexer().isNot(AsmToken::EndOfStatement));
1513       break;
1514   }
1515   Operands.push_back(AMDGPUOperand::CreateImm(CntVal, S));
1516   return MatchOperand_Success;
1517 }
1518 
1519 bool AMDGPUOperand::isSWaitCnt() const {
1520   return isImm();
1521 }
1522 
1523 //===----------------------------------------------------------------------===//
1524 // sopp branch targets
1525 //===----------------------------------------------------------------------===//
1526 
1527 AMDGPUAsmParser::OperandMatchResultTy
1528 AMDGPUAsmParser::parseSOppBrTarget(OperandVector &Operands) {
1529   SMLoc S = Parser.getTok().getLoc();
1530 
1531   switch (getLexer().getKind()) {
1532     default: return MatchOperand_ParseFail;
1533     case AsmToken::Integer: {
1534       int64_t Imm;
1535       if (getParser().parseAbsoluteExpression(Imm))
1536         return MatchOperand_ParseFail;
1537       Operands.push_back(AMDGPUOperand::CreateImm(Imm, S));
1538       return MatchOperand_Success;
1539     }
1540 
1541     case AsmToken::Identifier:
1542       Operands.push_back(AMDGPUOperand::CreateExpr(
1543           MCSymbolRefExpr::create(getContext().getOrCreateSymbol(
1544                                   Parser.getTok().getString()), getContext()), S));
1545       Parser.Lex();
1546       return MatchOperand_Success;
1547   }
1548 }
1549 
1550 //===----------------------------------------------------------------------===//
1551 // flat
1552 //===----------------------------------------------------------------------===//
1553 
1554 static const OptionalOperand FlatOptionalOps [] = {
1555   {"glc",    AMDGPUOperand::ImmTyGLC, true, 0, nullptr},
1556   {"slc",    AMDGPUOperand::ImmTySLC, true, 0, nullptr},
1557   {"tfe",    AMDGPUOperand::ImmTyTFE, true, 0, nullptr}
1558 };
1559 
1560 static const OptionalOperand FlatAtomicOptionalOps [] = {
1561   {"slc",    AMDGPUOperand::ImmTySLC, true, 0, nullptr},
1562   {"tfe",    AMDGPUOperand::ImmTyTFE, true, 0, nullptr}
1563 };
1564 
1565 AMDGPUAsmParser::OperandMatchResultTy
1566 AMDGPUAsmParser::parseFlatOptionalOps(OperandVector &Operands) {
1567   return parseOptionalOps(FlatOptionalOps, Operands);
1568 }
1569 
1570 AMDGPUAsmParser::OperandMatchResultTy
1571 AMDGPUAsmParser::parseFlatAtomicOptionalOps(OperandVector &Operands) {
1572   return parseOptionalOps(FlatAtomicOptionalOps, Operands);
1573 }
1574 
1575 void AMDGPUAsmParser::cvtFlat(MCInst &Inst,
1576                                const OperandVector &Operands) {
1577   OptionalImmIndexMap OptionalIdx;
1578 
1579   for (unsigned i = 1, e = Operands.size(); i != e; ++i) {
1580     AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[i]);
1581 
1582     // Add the register arguments
1583     if (Op.isReg()) {
1584       Op.addRegOperands(Inst, 1);
1585       continue;
1586     }
1587 
1588     OptionalIdx[Op.getImmTy()] = i;
1589   }
1590   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyGLC);
1591   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySLC);
1592   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyTFE);
1593 }
1594 
1595 
1596 void AMDGPUAsmParser::cvtFlatAtomic(MCInst &Inst,
1597                                const OperandVector &Operands) {
1598   OptionalImmIndexMap OptionalIdx;
1599 
1600   for (unsigned i = 1, e = Operands.size(); i != e; ++i) {
1601     AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[i]);
1602 
1603     // Add the register arguments
1604     if (Op.isReg()) {
1605       Op.addRegOperands(Inst, 1);
1606       continue;
1607     }
1608 
1609     // Handle 'glc' token for flat atomics.
1610     if (Op.isToken()) {
1611       continue;
1612     }
1613 
1614     // Handle optional arguments
1615     OptionalIdx[Op.getImmTy()] = i;
1616   }
1617   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySLC);
1618   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyTFE);
1619 }
1620 
1621 //===----------------------------------------------------------------------===//
1622 // mubuf
1623 //===----------------------------------------------------------------------===//
1624 
1625 static const OptionalOperand MubufOptionalOps [] = {
1626   {"offset", AMDGPUOperand::ImmTyOffset, false, 0, nullptr},
1627   {"glc",    AMDGPUOperand::ImmTyGLC, true, 0, nullptr},
1628   {"slc",    AMDGPUOperand::ImmTySLC, true, 0, nullptr},
1629   {"tfe",    AMDGPUOperand::ImmTyTFE, true, 0, nullptr}
1630 };
1631 
1632 AMDGPUAsmParser::OperandMatchResultTy
1633 AMDGPUAsmParser::parseMubufOptionalOps(OperandVector &Operands) {
1634   return parseOptionalOps(MubufOptionalOps, Operands);
1635 }
1636 
1637 AMDGPUAsmParser::OperandMatchResultTy
1638 AMDGPUAsmParser::parseOffset(OperandVector &Operands) {
1639   return parseIntWithPrefix("offset", Operands);
1640 }
1641 
1642 AMDGPUAsmParser::OperandMatchResultTy
1643 AMDGPUAsmParser::parseGLC(OperandVector &Operands) {
1644   return parseNamedBit("glc", Operands);
1645 }
1646 
1647 AMDGPUAsmParser::OperandMatchResultTy
1648 AMDGPUAsmParser::parseSLC(OperandVector &Operands) {
1649   return parseNamedBit("slc", Operands);
1650 }
1651 
1652 AMDGPUAsmParser::OperandMatchResultTy
1653 AMDGPUAsmParser::parseTFE(OperandVector &Operands) {
1654   return parseNamedBit("tfe", Operands);
1655 }
1656 
1657 bool AMDGPUOperand::isMubufOffset() const {
1658   return isImmTy(ImmTyOffset) && isUInt<12>(getImm());
1659 }
1660 
1661 void AMDGPUAsmParser::cvtMubuf(MCInst &Inst,
1662                                const OperandVector &Operands) {
1663   OptionalImmIndexMap OptionalIdx;
1664 
1665   for (unsigned i = 1, e = Operands.size(); i != e; ++i) {
1666     AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[i]);
1667 
1668     // Add the register arguments
1669     if (Op.isReg()) {
1670       Op.addRegOperands(Inst, 1);
1671       continue;
1672     }
1673 
1674     // Handle the case where soffset is an immediate
1675     if (Op.isImm() && Op.getImmTy() == AMDGPUOperand::ImmTyNone) {
1676       Op.addImmOperands(Inst, 1);
1677       continue;
1678     }
1679 
1680     // Handle tokens like 'offen' which are sometimes hard-coded into the
1681     // asm string.  There are no MCInst operands for these.
1682     if (Op.isToken()) {
1683       continue;
1684     }
1685     assert(Op.isImm());
1686 
1687     // Handle optional arguments
1688     OptionalIdx[Op.getImmTy()] = i;
1689   }
1690 
1691   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyOffset);
1692   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyGLC);
1693   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySLC);
1694   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyTFE);
1695 }
1696 
1697 //===----------------------------------------------------------------------===//
1698 // mimg
1699 //===----------------------------------------------------------------------===//
1700 
1701 AMDGPUAsmParser::OperandMatchResultTy
1702 AMDGPUAsmParser::parseDMask(OperandVector &Operands) {
1703   return parseIntWithPrefix("dmask", Operands, AMDGPUOperand::ImmTyDMask);
1704 }
1705 
1706 AMDGPUAsmParser::OperandMatchResultTy
1707 AMDGPUAsmParser::parseUNorm(OperandVector &Operands) {
1708   return parseNamedBit("unorm", Operands, AMDGPUOperand::ImmTyUNorm);
1709 }
1710 
1711 AMDGPUAsmParser::OperandMatchResultTy
1712 AMDGPUAsmParser::parseDA(OperandVector &Operands) {
1713   return parseNamedBit("da", Operands, AMDGPUOperand::ImmTyDA);
1714 }
1715 
1716 AMDGPUAsmParser::OperandMatchResultTy
1717 AMDGPUAsmParser::parseR128(OperandVector &Operands) {
1718   return parseNamedBit("r128", Operands, AMDGPUOperand::ImmTyR128);
1719 }
1720 
1721 AMDGPUAsmParser::OperandMatchResultTy
1722 AMDGPUAsmParser::parseLWE(OperandVector &Operands) {
1723   return parseNamedBit("lwe", Operands, AMDGPUOperand::ImmTyLWE);
1724 }
1725 
1726 //===----------------------------------------------------------------------===//
1727 // smrd
1728 //===----------------------------------------------------------------------===//
1729 
1730 bool AMDGPUOperand::isSMRDOffset() const {
1731 
1732   // FIXME: Support 20-bit offsets on VI.  We need to to pass subtarget
1733   // information here.
1734   return isImm() && isUInt<8>(getImm());
1735 }
1736 
1737 bool AMDGPUOperand::isSMRDLiteralOffset() const {
1738   // 32-bit literals are only supported on CI and we only want to use them
1739   // when the offset is > 8-bits.
1740   return isImm() && !isUInt<8>(getImm()) && isUInt<32>(getImm());
1741 }
1742 
1743 //===----------------------------------------------------------------------===//
1744 // vop3
1745 //===----------------------------------------------------------------------===//
1746 
1747 static bool ConvertOmodMul(int64_t &Mul) {
1748   if (Mul != 1 && Mul != 2 && Mul != 4)
1749     return false;
1750 
1751   Mul >>= 1;
1752   return true;
1753 }
1754 
1755 static bool ConvertOmodDiv(int64_t &Div) {
1756   if (Div == 1) {
1757     Div = 0;
1758     return true;
1759   }
1760 
1761   if (Div == 2) {
1762     Div = 3;
1763     return true;
1764   }
1765 
1766   return false;
1767 }
1768 
1769 static const OptionalOperand VOP3OptionalOps [] = {
1770   {"clamp", AMDGPUOperand::ImmTyClamp, true, 0, nullptr},
1771   {"mul",   AMDGPUOperand::ImmTyOMod, false, 1, ConvertOmodMul},
1772   {"div",   AMDGPUOperand::ImmTyOMod, false, 1, ConvertOmodDiv},
1773 };
1774 
1775 static bool isVOP3(OperandVector &Operands) {
1776   if (operandsHaveModifiers(Operands))
1777     return true;
1778 
1779   if (Operands.size() >= 2) {
1780     AMDGPUOperand &DstOp = ((AMDGPUOperand&)*Operands[1]);
1781 
1782     if (DstOp.isRegClass(AMDGPU::SGPR_64RegClassID))
1783       return true;
1784   }
1785 
1786   if (Operands.size() >= 5)
1787     return true;
1788 
1789   if (Operands.size() > 3) {
1790     AMDGPUOperand &Src1Op = ((AMDGPUOperand&)*Operands[3]);
1791     if (Src1Op.isRegClass(AMDGPU::SReg_32RegClassID) ||
1792         Src1Op.isRegClass(AMDGPU::SReg_64RegClassID))
1793       return true;
1794   }
1795   return false;
1796 }
1797 
1798 AMDGPUAsmParser::OperandMatchResultTy
1799 AMDGPUAsmParser::parseVOP3OptionalOps(OperandVector &Operands) {
1800 
1801   // The value returned by this function may change after parsing
1802   // an operand so store the original value here.
1803   bool HasModifiers = operandsHaveModifiers(Operands);
1804 
1805   bool IsVOP3 = isVOP3(Operands);
1806   if (HasModifiers || IsVOP3 ||
1807       getLexer().isNot(AsmToken::EndOfStatement) ||
1808       getForcedEncodingSize() == 64) {
1809 
1810     AMDGPUAsmParser::OperandMatchResultTy Res =
1811         parseOptionalOps(VOP3OptionalOps, Operands);
1812 
1813     if (!HasModifiers && Res == MatchOperand_Success) {
1814       // We have added a modifier operation, so we need to make sure all
1815       // previous register operands have modifiers
1816       for (unsigned i = 2, e = Operands.size(); i != e; ++i) {
1817         AMDGPUOperand &Op = ((AMDGPUOperand&)*Operands[i]);
1818         if ((Op.isReg() || Op.isImm()) && !Op.hasModifiers())
1819           Op.setModifiers(0);
1820       }
1821     }
1822     return Res;
1823   }
1824   return MatchOperand_NoMatch;
1825 }
1826 
1827 void AMDGPUAsmParser::cvtId(MCInst &Inst, const OperandVector &Operands) {
1828   unsigned I = 1;
1829   const MCInstrDesc &Desc = MII.get(Inst.getOpcode());
1830   for (unsigned J = 0; J < Desc.getNumDefs(); ++J) {
1831     ((AMDGPUOperand &)*Operands[I++]).addRegOperands(Inst, 1);
1832   }
1833   for (unsigned E = Operands.size(); I != E; ++I)
1834     ((AMDGPUOperand &)*Operands[I]).addRegOrImmOperands(Inst, 1);
1835 }
1836 
1837 void AMDGPUAsmParser::cvtVOP3_2_mod(MCInst &Inst, const OperandVector &Operands) {
1838   uint64_t TSFlags = MII.get(Inst.getOpcode()).TSFlags;
1839   if (TSFlags & SIInstrFlags::VOP3) {
1840     cvtVOP3(Inst, Operands);
1841   } else {
1842     cvtId(Inst, Operands);
1843   }
1844 }
1845 
1846 void AMDGPUAsmParser::cvtVOP3_2_nomod(MCInst &Inst, const OperandVector &Operands) {
1847   if (operandsHaveModifiers(Operands)) {
1848     cvtVOP3(Inst, Operands);
1849   } else {
1850     cvtId(Inst, Operands);
1851   }
1852 }
1853 
1854 void AMDGPUAsmParser::cvtVOP3_only(MCInst &Inst, const OperandVector &Operands) {
1855   cvtVOP3(Inst, Operands);
1856 }
1857 
1858 void AMDGPUAsmParser::cvtVOP3(MCInst &Inst, const OperandVector &Operands) {
1859   OptionalImmIndexMap OptionalIdx;
1860   unsigned I = 1;
1861   const MCInstrDesc &Desc = MII.get(Inst.getOpcode());
1862   for (unsigned J = 0; J < Desc.getNumDefs(); ++J) {
1863     ((AMDGPUOperand &)*Operands[I++]).addRegOperands(Inst, 1);
1864   }
1865 
1866   for (unsigned E = Operands.size(); I != E; ++I) {
1867     AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[I]);
1868     if (Op.isRegOrImmWithInputMods()) {
1869       Op.addRegOrImmWithInputModsOperands(Inst, 2);
1870     } else if (Op.isImm()) {
1871       OptionalIdx[Op.getImmTy()] = I;
1872     } else {
1873       assert(false);
1874     }
1875   }
1876 
1877   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyClamp);
1878   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyOMod);
1879 }
1880 
1881 void AMDGPUAsmParser::cvtMIMG(MCInst &Inst, const OperandVector &Operands) {
1882   unsigned I = 1;
1883   const MCInstrDesc &Desc = MII.get(Inst.getOpcode());
1884   for (unsigned J = 0; J < Desc.getNumDefs(); ++J) {
1885     ((AMDGPUOperand &)*Operands[I++]).addRegOperands(Inst, 1);
1886   }
1887 
1888   OptionalImmIndexMap OptionalIdx;
1889 
1890   for (unsigned E = Operands.size(); I != E; ++I) {
1891     AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[I]);
1892 
1893     // Add the register arguments
1894     if (Op.isRegOrImm()) {
1895       Op.addRegOrImmOperands(Inst, 1);
1896       continue;
1897     } else if (Op.isImmModifier()) {
1898       OptionalIdx[Op.getImmTy()] = I;
1899     } else {
1900       assert(false);
1901     }
1902   }
1903 
1904   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyDMask);
1905   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyUNorm);
1906   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyGLC);
1907   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyDA);
1908   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyR128);
1909   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyTFE);
1910   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyLWE);
1911   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySLC);
1912 }
1913 
1914 void AMDGPUAsmParser::cvtMIMGAtomic(MCInst &Inst, const OperandVector &Operands) {
1915   unsigned I = 1;
1916   const MCInstrDesc &Desc = MII.get(Inst.getOpcode());
1917   for (unsigned J = 0; J < Desc.getNumDefs(); ++J) {
1918     ((AMDGPUOperand &)*Operands[I++]).addRegOperands(Inst, 1);
1919   }
1920 
1921   // Add src, same as dst
1922   ((AMDGPUOperand &)*Operands[I]).addRegOperands(Inst, 1);
1923 
1924   OptionalImmIndexMap OptionalIdx;
1925 
1926   for (unsigned E = Operands.size(); I != E; ++I) {
1927     AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[I]);
1928 
1929     // Add the register arguments
1930     if (Op.isRegOrImm()) {
1931       Op.addRegOrImmOperands(Inst, 1);
1932       continue;
1933     } else if (Op.isImmModifier()) {
1934       OptionalIdx[Op.getImmTy()] = I;
1935     } else {
1936       assert(false);
1937     }
1938   }
1939 
1940   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyDMask);
1941   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyUNorm);
1942   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyGLC);
1943   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyDA);
1944   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyR128);
1945   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyTFE);
1946   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyLWE);
1947   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySLC);
1948 }
1949 
1950 //===----------------------------------------------------------------------===//
1951 // dpp
1952 //===----------------------------------------------------------------------===//
1953 
1954 bool AMDGPUOperand::isDPPCtrl() const {
1955   bool result = isImm() && getImmTy() == ImmTyDppCtrl && isUInt<9>(getImm());
1956   if (result) {
1957     int64_t Imm = getImm();
1958     return ((Imm >= 0x000) && (Imm <= 0x0ff)) ||
1959            ((Imm >= 0x101) && (Imm <= 0x10f)) ||
1960            ((Imm >= 0x111) && (Imm <= 0x11f)) ||
1961            ((Imm >= 0x121) && (Imm <= 0x12f)) ||
1962            (Imm == 0x130) ||
1963            (Imm == 0x134) ||
1964            (Imm == 0x138) ||
1965            (Imm == 0x13c) ||
1966            (Imm == 0x140) ||
1967            (Imm == 0x141) ||
1968            (Imm == 0x142) ||
1969            (Imm == 0x143);
1970   }
1971   return false;
1972 }
1973 
1974 AMDGPUAsmParser::OperandMatchResultTy
1975 AMDGPUAsmParser::parseDPPCtrlOps(OperandVector &Operands) {
1976   // ToDo: use same syntax as sp3 for dpp_ctrl
1977   SMLoc S = Parser.getTok().getLoc();
1978   StringRef Prefix;
1979   int64_t Int;
1980 
1981   if (getLexer().getKind() == AsmToken::Identifier) {
1982     Prefix = Parser.getTok().getString();
1983   } else {
1984     return MatchOperand_NoMatch;
1985   }
1986 
1987   if (Prefix == "row_mirror") {
1988     Int = 0x140;
1989   } else if (Prefix == "row_half_mirror") {
1990     Int = 0x141;
1991   } else {
1992     Parser.Lex();
1993     if (getLexer().isNot(AsmToken::Colon))
1994       return MatchOperand_ParseFail;
1995 
1996     if (Prefix == "quad_perm") {
1997       // quad_perm:[%d,%d,%d,%d]
1998       Parser.Lex();
1999       if (getLexer().isNot(AsmToken::LBrac))
2000         return MatchOperand_ParseFail;
2001 
2002       Parser.Lex();
2003       if (getLexer().isNot(AsmToken::Integer))
2004         return MatchOperand_ParseFail;
2005       Int = getLexer().getTok().getIntVal();
2006 
2007       Parser.Lex();
2008       if (getLexer().isNot(AsmToken::Comma))
2009         return MatchOperand_ParseFail;
2010       Parser.Lex();
2011       if (getLexer().isNot(AsmToken::Integer))
2012         return MatchOperand_ParseFail;
2013       Int += (getLexer().getTok().getIntVal() << 2);
2014 
2015       Parser.Lex();
2016       if (getLexer().isNot(AsmToken::Comma))
2017         return MatchOperand_ParseFail;
2018       Parser.Lex();
2019       if (getLexer().isNot(AsmToken::Integer))
2020         return MatchOperand_ParseFail;
2021       Int += (getLexer().getTok().getIntVal() << 4);
2022 
2023       Parser.Lex();
2024       if (getLexer().isNot(AsmToken::Comma))
2025         return MatchOperand_ParseFail;
2026       Parser.Lex();
2027       if (getLexer().isNot(AsmToken::Integer))
2028         return MatchOperand_ParseFail;
2029       Int += (getLexer().getTok().getIntVal() << 6);
2030 
2031       Parser.Lex();
2032       if (getLexer().isNot(AsmToken::RBrac))
2033         return MatchOperand_ParseFail;
2034 
2035     } else {
2036       // sel:%d
2037       Parser.Lex();
2038       if (getLexer().isNot(AsmToken::Integer))
2039         return MatchOperand_ParseFail;
2040       Int = getLexer().getTok().getIntVal();
2041 
2042       if (Prefix == "row_shl") {
2043         Int |= 0x100;
2044       } else if (Prefix == "row_shr") {
2045         Int |= 0x110;
2046       } else if (Prefix == "row_ror") {
2047         Int |= 0x120;
2048       } else if (Prefix == "wave_shl") {
2049         Int = 0x130;
2050       } else if (Prefix == "wave_rol") {
2051         Int = 0x134;
2052       } else if (Prefix == "wave_shr") {
2053         Int = 0x138;
2054       } else if (Prefix == "wave_ror") {
2055         Int = 0x13C;
2056       } else if (Prefix == "row_bcast") {
2057         if (Int == 15) {
2058           Int = 0x142;
2059         } else if (Int == 31) {
2060           Int = 0x143;
2061         }
2062       } else {
2063         return MatchOperand_NoMatch;
2064       }
2065     }
2066   }
2067   Parser.Lex(); // eat last token
2068 
2069   Operands.push_back(AMDGPUOperand::CreateImm(Int, S,
2070                                               AMDGPUOperand::ImmTyDppCtrl));
2071   return MatchOperand_Success;
2072 }
2073 
2074 static const OptionalOperand DPPOptionalOps [] = {
2075   {"row_mask", AMDGPUOperand::ImmTyDppRowMask, false, 0xf, nullptr},
2076   {"bank_mask", AMDGPUOperand::ImmTyDppBankMask, false, 0xf, nullptr},
2077   {"bound_ctrl", AMDGPUOperand::ImmTyDppBoundCtrl, false, -1, nullptr}
2078 };
2079 
2080 AMDGPUAsmParser::OperandMatchResultTy
2081 AMDGPUAsmParser::parseDPPOptionalOps(OperandVector &Operands) {
2082   SMLoc S = Parser.getTok().getLoc();
2083   OperandMatchResultTy Res = parseOptionalOps(DPPOptionalOps, Operands);
2084   // XXX - sp3 use syntax "bound_ctrl:0" to indicate that bound_ctrl bit was set
2085   if (Res == MatchOperand_Success) {
2086     AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands.back());
2087     // If last operand was parsed as bound_ctrl we should replace it with correct value (1)
2088     if (Op.isImmTy(AMDGPUOperand::ImmTyDppBoundCtrl)) {
2089       Operands.pop_back();
2090       Operands.push_back(
2091         AMDGPUOperand::CreateImm(1, S, AMDGPUOperand::ImmTyDppBoundCtrl));
2092         return MatchOperand_Success;
2093     }
2094   }
2095   return Res;
2096 }
2097 
2098 void AMDGPUAsmParser::cvtDPP_mod(MCInst &Inst, const OperandVector &Operands) {
2099   cvtDPP(Inst, Operands, true);
2100 }
2101 
2102 void AMDGPUAsmParser::cvtDPP_nomod(MCInst &Inst, const OperandVector &Operands) {
2103   cvtDPP(Inst, Operands, false);
2104 }
2105 
2106 void AMDGPUAsmParser::cvtDPP(MCInst &Inst, const OperandVector &Operands,
2107                              bool HasMods) {
2108   OptionalImmIndexMap OptionalIdx;
2109 
2110   unsigned I = 1;
2111   const MCInstrDesc &Desc = MII.get(Inst.getOpcode());
2112   for (unsigned J = 0; J < Desc.getNumDefs(); ++J) {
2113     ((AMDGPUOperand &)*Operands[I++]).addRegOperands(Inst, 1);
2114   }
2115 
2116   for (unsigned E = Operands.size(); I != E; ++I) {
2117     AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[I]);
2118     // Add the register arguments
2119     if (!HasMods && Op.isReg()) {
2120       Op.addRegOperands(Inst, 1);
2121     } else if (HasMods && Op.isRegOrImmWithInputMods()) {
2122       Op.addRegOrImmWithInputModsOperands(Inst, 2);
2123     } else if (Op.isDPPCtrl()) {
2124       Op.addImmOperands(Inst, 1);
2125     } else if (Op.isImm()) {
2126       // Handle optional arguments
2127       OptionalIdx[Op.getImmTy()] = I;
2128     } else {
2129       llvm_unreachable("Invalid operand type");
2130     }
2131   }
2132 
2133   // ToDo: fix default values for row_mask and bank_mask
2134   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyDppRowMask, 0xf);
2135   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyDppBankMask, 0xf);
2136   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyDppBoundCtrl);
2137 }
2138 
2139 
2140 /// Force static initialization.
2141 extern "C" void LLVMInitializeAMDGPUAsmParser() {
2142   RegisterMCAsmParser<AMDGPUAsmParser> A(TheAMDGPUTarget);
2143   RegisterMCAsmParser<AMDGPUAsmParser> B(TheGCNTarget);
2144 }
2145 
2146 #define GET_REGISTER_MATCHER
2147 #define GET_MATCHER_IMPLEMENTATION
2148 #include "AMDGPUGenAsmMatcher.inc"
2149