1 //===-- AMDGPUAsmParser.cpp - Parse SI asm to MCInst instructions ---------===//
2 //
3 //                     The LLVM Compiler Infrastructure
4 //
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
7 //
8 //===----------------------------------------------------------------------===//
9 
10 #include "AMDKernelCodeT.h"
11 #include "MCTargetDesc/AMDGPUMCTargetDesc.h"
12 #include "MCTargetDesc/AMDGPUTargetStreamer.h"
13 #include "SIDefines.h"
14 #include "Utils/AMDGPUBaseInfo.h"
15 #include "Utils/AMDKernelCodeTUtils.h"
16 #include "llvm/ADT/APFloat.h"
17 #include "llvm/ADT/STLExtras.h"
18 #include "llvm/ADT/SmallBitVector.h"
19 #include "llvm/ADT/SmallString.h"
20 #include "llvm/ADT/StringSwitch.h"
21 #include "llvm/ADT/Twine.h"
22 #include "llvm/MC/MCContext.h"
23 #include "llvm/MC/MCExpr.h"
24 #include "llvm/MC/MCInst.h"
25 #include "llvm/MC/MCInstrInfo.h"
26 #include "llvm/MC/MCParser/MCAsmLexer.h"
27 #include "llvm/MC/MCParser/MCAsmParser.h"
28 #include "llvm/MC/MCParser/MCParsedAsmOperand.h"
29 #include "llvm/MC/MCParser/MCTargetAsmParser.h"
30 #include "llvm/MC/MCRegisterInfo.h"
31 #include "llvm/MC/MCStreamer.h"
32 #include "llvm/MC/MCSubtargetInfo.h"
33 #include "llvm/MC/MCSymbolELF.h"
34 #include "llvm/Support/Debug.h"
35 #include "llvm/Support/ELF.h"
36 #include "llvm/Support/SourceMgr.h"
37 #include "llvm/Support/TargetRegistry.h"
38 #include "llvm/Support/raw_ostream.h"
39 
40 // FIXME ODR: Move this to some common place for AsmParser and InstPrinter
41 namespace llvm {
42 namespace AMDGPU {
43 namespace SendMsg {
44 
45 // This must be in sync with llvm::AMDGPU::SendMsg::Id enum members.
46 static
47 const char* const IdSymbolic[] = {
48   nullptr,
49   "MSG_INTERRUPT",
50   "MSG_GS",
51   "MSG_GS_DONE",
52   nullptr,
53   nullptr,
54   nullptr,
55   nullptr,
56   nullptr,
57   nullptr,
58   nullptr,
59   nullptr,
60   nullptr,
61   nullptr,
62   nullptr,
63   "MSG_SYSMSG"
64 };
65 
66 // These two must be in sync with llvm::AMDGPU::SendMsg::Op enum members.
67 static
68 const char* const OpSysSymbolic[] = {
69   nullptr,
70   "SYSMSG_OP_ECC_ERR_INTERRUPT",
71   "SYSMSG_OP_REG_RD",
72   "SYSMSG_OP_HOST_TRAP_ACK",
73   "SYSMSG_OP_TTRACE_PC"
74 };
75 
76 static
77 const char* const OpGsSymbolic[] = {
78   "GS_OP_NOP",
79   "GS_OP_CUT",
80   "GS_OP_EMIT",
81   "GS_OP_EMIT_CUT"
82 };
83 
84 } // namespace SendMsg
85 } // namespace AMDGPU
86 } // namespace llvm
87 
88 using namespace llvm;
89 
90 namespace {
91 
92 struct OptionalOperand;
93 
94 enum RegisterKind { IS_UNKNOWN, IS_VGPR, IS_SGPR, IS_TTMP, IS_SPECIAL };
95 
96 class AMDGPUOperand : public MCParsedAsmOperand {
97   enum KindTy {
98     Token,
99     Immediate,
100     Register,
101     Expression
102   } Kind;
103 
104   SMLoc StartLoc, EndLoc;
105 
106 public:
107   AMDGPUOperand(enum KindTy K) : MCParsedAsmOperand(), Kind(K) {}
108 
109   MCContext *Ctx;
110 
111   typedef std::unique_ptr<AMDGPUOperand> Ptr;
112 
113   enum ImmTy {
114     ImmTyNone,
115     ImmTyGDS,
116     ImmTyOffen,
117     ImmTyIdxen,
118     ImmTyAddr64,
119     ImmTyOffset,
120     ImmTyOffset0,
121     ImmTyOffset1,
122     ImmTyGLC,
123     ImmTySLC,
124     ImmTyTFE,
125     ImmTyClampSI,
126     ImmTyOModSI,
127     ImmTyDppCtrl,
128     ImmTyDppRowMask,
129     ImmTyDppBankMask,
130     ImmTyDppBoundCtrl,
131     ImmTySdwaSel,
132     ImmTySdwaDstUnused,
133     ImmTyDMask,
134     ImmTyUNorm,
135     ImmTyDA,
136     ImmTyR128,
137     ImmTyLWE,
138     ImmTyHwreg,
139     ImmTySendMsg,
140   };
141 
142   struct TokOp {
143     const char *Data;
144     unsigned Length;
145   };
146 
147   struct ImmOp {
148     bool IsFPImm;
149     ImmTy Type;
150     int64_t Val;
151     int Modifiers;
152   };
153 
154   struct RegOp {
155     unsigned RegNo;
156     int Modifiers;
157     const MCRegisterInfo *TRI;
158     const MCSubtargetInfo *STI;
159     bool IsForcedVOP3;
160   };
161 
162   union {
163     TokOp Tok;
164     ImmOp Imm;
165     RegOp Reg;
166     const MCExpr *Expr;
167   };
168 
169   void addImmOperands(MCInst &Inst, unsigned N, bool ApplyModifiers = true) const {
170     if (Imm.Type == ImmTyNone && ApplyModifiers && Imm.Modifiers != 0) {
171       // Apply modifiers to immediate value
172       int64_t Val = Imm.Val;
173       bool Negate = Imm.Modifiers & 0x1;
174       bool Abs = Imm.Modifiers & 0x2;
175       if (Imm.IsFPImm) {
176         APFloat F(BitsToFloat(Val));
177         if (Abs) {
178           F.clearSign();
179         }
180         if (Negate) {
181           F.changeSign();
182         }
183         Val = F.bitcastToAPInt().getZExtValue();
184       } else {
185         Val = Abs ? std::abs(Val) : Val;
186         Val = Negate ? -Val : Val;
187       }
188       Inst.addOperand(MCOperand::createImm(Val));
189     } else {
190       Inst.addOperand(MCOperand::createImm(getImm()));
191     }
192   }
193 
194   StringRef getToken() const {
195     return StringRef(Tok.Data, Tok.Length);
196   }
197 
198   void addRegOperands(MCInst &Inst, unsigned N) const {
199     Inst.addOperand(MCOperand::createReg(AMDGPU::getMCReg(getReg(), *Reg.STI)));
200   }
201 
202   void addRegOrImmOperands(MCInst &Inst, unsigned N) const {
203     if (isRegKind())
204       addRegOperands(Inst, N);
205     else
206       addImmOperands(Inst, N);
207   }
208 
209   void addRegOrImmWithInputModsOperands(MCInst &Inst, unsigned N) const {
210     if (isRegKind()) {
211       Inst.addOperand(MCOperand::createImm(Reg.Modifiers));
212       addRegOperands(Inst, N);
213     } else {
214       Inst.addOperand(MCOperand::createImm(Imm.Modifiers));
215       addImmOperands(Inst, N, false);
216     }
217   }
218 
219   void addSoppBrTargetOperands(MCInst &Inst, unsigned N) const {
220     if (isImm())
221       addImmOperands(Inst, N);
222     else {
223       assert(isExpr());
224       Inst.addOperand(MCOperand::createExpr(Expr));
225     }
226   }
227 
228   bool isToken() const override {
229     return Kind == Token;
230   }
231 
232   bool isImm() const override {
233     return Kind == Immediate;
234   }
235 
236   bool isInlinableImm() const {
237     if (!isImm() || Imm.Type != AMDGPUOperand::ImmTyNone /* Only plain
238       immediates are inlinable (e.g. "clamp" attribute is not) */ )
239       return false;
240     // TODO: We should avoid using host float here. It would be better to
241     // check the float bit values which is what a few other places do.
242     // We've had bot failures before due to weird NaN support on mips hosts.
243     const float F = BitsToFloat(Imm.Val);
244     // TODO: Add 1/(2*pi) for VI
245     return (Imm.Val <= 64 && Imm.Val >= -16) ||
246            (F == 0.0 || F == 0.5 || F == -0.5 || F == 1.0 || F == -1.0 ||
247            F == 2.0 || F == -2.0 || F == 4.0 || F == -4.0);
248   }
249 
250   int64_t getImm() const {
251     return Imm.Val;
252   }
253 
254   enum ImmTy getImmTy() const {
255     assert(isImm());
256     return Imm.Type;
257   }
258 
259   bool isRegKind() const {
260     return Kind == Register;
261   }
262 
263   bool isReg() const override {
264     return Kind == Register && Reg.Modifiers == 0;
265   }
266 
267   bool isRegOrImmWithInputMods() const {
268     return Kind == Register || isInlinableImm();
269   }
270 
271   bool isImmTy(ImmTy ImmT) const {
272     return isImm() && Imm.Type == ImmT;
273   }
274 
275   bool isClampSI() const {
276     return isImmTy(ImmTyClampSI);
277   }
278 
279   bool isOModSI() const {
280     return isImmTy(ImmTyOModSI);
281   }
282 
283   bool isImmModifier() const {
284     return Kind == Immediate && Imm.Type != ImmTyNone;
285   }
286 
287   bool isDMask() const {
288     return isImmTy(ImmTyDMask);
289   }
290 
291   bool isUNorm() const { return isImmTy(ImmTyUNorm); }
292   bool isDA() const { return isImmTy(ImmTyDA); }
293   bool isR128() const { return isImmTy(ImmTyUNorm); }
294   bool isLWE() const { return isImmTy(ImmTyLWE); }
295 
296   bool isMod() const {
297     return isClampSI() || isOModSI();
298   }
299 
300   bool isOffen() const { return isImmTy(ImmTyOffen); }
301   bool isIdxen() const { return isImmTy(ImmTyIdxen); }
302   bool isAddr64() const { return isImmTy(ImmTyAddr64); }
303   bool isOffset() const { return isImmTy(ImmTyOffset) && isUInt<16>(getImm()); }
304   bool isOffset0() const { return isImmTy(ImmTyOffset0) && isUInt<16>(getImm()); }
305   bool isOffset1() const { return isImmTy(ImmTyOffset1) && isUInt<8>(getImm()); }
306   bool isGDS() const { return isImmTy(ImmTyGDS); }
307   bool isGLC() const { return isImmTy(ImmTyGLC); }
308   bool isSLC() const { return isImmTy(ImmTySLC); }
309   bool isTFE() const { return isImmTy(ImmTyTFE); }
310 
311   bool isBankMask() const {
312     return isImmTy(ImmTyDppBankMask);
313   }
314 
315   bool isRowMask() const {
316     return isImmTy(ImmTyDppRowMask);
317   }
318 
319   bool isBoundCtrl() const {
320     return isImmTy(ImmTyDppBoundCtrl);
321   }
322 
323   bool isSDWASel() const {
324     return isImmTy(ImmTySdwaSel);
325   }
326 
327   bool isSDWADstUnused() const {
328     return isImmTy(ImmTySdwaDstUnused);
329   }
330 
331   void setModifiers(unsigned Mods) {
332     assert(isReg() || (isImm() && Imm.Modifiers == 0));
333     if (isReg())
334       Reg.Modifiers = Mods;
335     else
336       Imm.Modifiers = Mods;
337   }
338 
339   bool hasModifiers() const {
340     assert(isRegKind() || isImm());
341     return isRegKind() ? Reg.Modifiers != 0 : Imm.Modifiers != 0;
342   }
343 
344   unsigned getReg() const override {
345     return Reg.RegNo;
346   }
347 
348   bool isRegOrImm() const {
349     return isReg() || isImm();
350   }
351 
352   bool isRegClass(unsigned RCID) const {
353     return isReg() && Reg.TRI->getRegClass(RCID).contains(getReg());
354   }
355 
356   bool isSCSrc32() const {
357     return isInlinableImm() || isRegClass(AMDGPU::SReg_32RegClassID);
358   }
359 
360   bool isSCSrc64() const {
361     return isInlinableImm() || isRegClass(AMDGPU::SReg_64RegClassID);
362   }
363 
364   bool isSSrc32() const {
365     return isImm() || isSCSrc32();
366   }
367 
368   bool isSSrc64() const {
369     // TODO: Find out how SALU supports extension of 32-bit literals to 64 bits.
370     // See isVSrc64().
371     return isImm() || isSCSrc64();
372   }
373 
374   bool isVCSrc32() const {
375     return isInlinableImm() || isRegClass(AMDGPU::VS_32RegClassID);
376   }
377 
378   bool isVCSrc64() const {
379     return isInlinableImm() || isRegClass(AMDGPU::VS_64RegClassID);
380   }
381 
382   bool isVSrc32() const {
383     return isImm() || isVCSrc32();
384   }
385 
386   bool isVSrc64() const {
387     // TODO: Check if the 64-bit value (coming from assembly source) can be
388     // narrowed to 32 bits (in the instruction stream). That require knowledge
389     // of instruction type (unsigned/signed, floating or "untyped"/B64),
390     // see [AMD GCN3 ISA 6.3.1].
391     // TODO: How 64-bit values are formed from 32-bit literals in _B64 insns?
392     return isImm() || isVCSrc64();
393   }
394 
395   bool isMem() const override {
396     return false;
397   }
398 
399   bool isExpr() const {
400     return Kind == Expression;
401   }
402 
403   bool isSoppBrTarget() const {
404     return isExpr() || isImm();
405   }
406 
407   SMLoc getStartLoc() const override {
408     return StartLoc;
409   }
410 
411   SMLoc getEndLoc() const override {
412     return EndLoc;
413   }
414 
415   void printImmTy(raw_ostream& OS, ImmTy Type) const {
416     switch (Type) {
417     case ImmTyNone: OS << "None"; break;
418     case ImmTyGDS: OS << "GDS"; break;
419     case ImmTyOffen: OS << "Offen"; break;
420     case ImmTyIdxen: OS << "Idxen"; break;
421     case ImmTyAddr64: OS << "Addr64"; break;
422     case ImmTyOffset: OS << "Offset"; break;
423     case ImmTyOffset0: OS << "Offset0"; break;
424     case ImmTyOffset1: OS << "Offset1"; break;
425     case ImmTyGLC: OS << "GLC"; break;
426     case ImmTySLC: OS << "SLC"; break;
427     case ImmTyTFE: OS << "TFE"; break;
428     case ImmTyClampSI: OS << "ClampSI"; break;
429     case ImmTyOModSI: OS << "OModSI"; break;
430     case ImmTyDppCtrl: OS << "DppCtrl"; break;
431     case ImmTyDppRowMask: OS << "DppRowMask"; break;
432     case ImmTyDppBankMask: OS << "DppBankMask"; break;
433     case ImmTyDppBoundCtrl: OS << "DppBoundCtrl"; break;
434     case ImmTySdwaSel: OS << "SdwaSel"; break;
435     case ImmTySdwaDstUnused: OS << "SdwaDstUnused"; break;
436     case ImmTyDMask: OS << "DMask"; break;
437     case ImmTyUNorm: OS << "UNorm"; break;
438     case ImmTyDA: OS << "DA"; break;
439     case ImmTyR128: OS << "R128"; break;
440     case ImmTyLWE: OS << "LWE"; break;
441     case ImmTyHwreg: OS << "Hwreg"; break;
442     case ImmTySendMsg: OS << "SendMsg"; break;
443     }
444   }
445 
446   void print(raw_ostream &OS) const override {
447     switch (Kind) {
448     case Register:
449       OS << "<register " << getReg() << " mods: " << Reg.Modifiers << '>';
450       break;
451     case Immediate:
452       OS << '<' << getImm();
453       if (getImmTy() != ImmTyNone) {
454         OS << " type: "; printImmTy(OS, getImmTy());
455       }
456       OS << " mods: " << Imm.Modifiers << '>';
457       break;
458     case Token:
459       OS << '\'' << getToken() << '\'';
460       break;
461     case Expression:
462       OS << "<expr " << *Expr << '>';
463       break;
464     }
465   }
466 
467   static AMDGPUOperand::Ptr CreateImm(int64_t Val, SMLoc Loc,
468                                       enum ImmTy Type = ImmTyNone,
469                                       bool IsFPImm = false) {
470     auto Op = llvm::make_unique<AMDGPUOperand>(Immediate);
471     Op->Imm.Val = Val;
472     Op->Imm.IsFPImm = IsFPImm;
473     Op->Imm.Type = Type;
474     Op->Imm.Modifiers = 0;
475     Op->StartLoc = Loc;
476     Op->EndLoc = Loc;
477     return Op;
478   }
479 
480   static AMDGPUOperand::Ptr CreateToken(StringRef Str, SMLoc Loc,
481                                         bool HasExplicitEncodingSize = true) {
482     auto Res = llvm::make_unique<AMDGPUOperand>(Token);
483     Res->Tok.Data = Str.data();
484     Res->Tok.Length = Str.size();
485     Res->StartLoc = Loc;
486     Res->EndLoc = Loc;
487     return Res;
488   }
489 
490   static AMDGPUOperand::Ptr CreateReg(unsigned RegNo, SMLoc S,
491                                       SMLoc E,
492                                       const MCRegisterInfo *TRI,
493                                       const MCSubtargetInfo *STI,
494                                       bool ForceVOP3) {
495     auto Op = llvm::make_unique<AMDGPUOperand>(Register);
496     Op->Reg.RegNo = RegNo;
497     Op->Reg.TRI = TRI;
498     Op->Reg.STI = STI;
499     Op->Reg.Modifiers = 0;
500     Op->Reg.IsForcedVOP3 = ForceVOP3;
501     Op->StartLoc = S;
502     Op->EndLoc = E;
503     return Op;
504   }
505 
506   static AMDGPUOperand::Ptr CreateExpr(const class MCExpr *Expr, SMLoc S) {
507     auto Op = llvm::make_unique<AMDGPUOperand>(Expression);
508     Op->Expr = Expr;
509     Op->StartLoc = S;
510     Op->EndLoc = S;
511     return Op;
512   }
513 
514   bool isSWaitCnt() const;
515   bool isHwreg() const;
516   bool isSendMsg() const;
517   bool isMubufOffset() const;
518   bool isSMRDOffset() const;
519   bool isSMRDLiteralOffset() const;
520   bool isDPPCtrl() const;
521 };
522 
523 class AMDGPUAsmParser : public MCTargetAsmParser {
524   const MCInstrInfo &MII;
525   MCAsmParser &Parser;
526 
527   unsigned ForcedEncodingSize;
528 
529   bool isSI() const {
530     return AMDGPU::isSI(getSTI());
531   }
532 
533   bool isCI() const {
534     return AMDGPU::isCI(getSTI());
535   }
536 
537   bool isVI() const {
538     return AMDGPU::isVI(getSTI());
539   }
540 
541   bool hasSGPR102_SGPR103() const {
542     return !isVI();
543   }
544 
545   /// @name Auto-generated Match Functions
546   /// {
547 
548 #define GET_ASSEMBLER_HEADER
549 #include "AMDGPUGenAsmMatcher.inc"
550 
551   /// }
552 
553 private:
554   bool ParseDirectiveMajorMinor(uint32_t &Major, uint32_t &Minor);
555   bool ParseDirectiveHSACodeObjectVersion();
556   bool ParseDirectiveHSACodeObjectISA();
557   bool ParseAMDKernelCodeTValue(StringRef ID, amd_kernel_code_t &Header);
558   bool ParseDirectiveAMDKernelCodeT();
559   bool ParseSectionDirectiveHSAText();
560   bool subtargetHasRegister(const MCRegisterInfo &MRI, unsigned RegNo) const;
561   bool ParseDirectiveAMDGPUHsaKernel();
562   bool ParseDirectiveAMDGPUHsaModuleGlobal();
563   bool ParseDirectiveAMDGPUHsaProgramGlobal();
564   bool ParseSectionDirectiveHSADataGlobalAgent();
565   bool ParseSectionDirectiveHSADataGlobalProgram();
566   bool ParseSectionDirectiveHSARodataReadonlyAgent();
567   bool AddNextRegisterToList(unsigned& Reg, unsigned& RegWidth, RegisterKind RegKind, unsigned Reg1, unsigned RegNum);
568   bool ParseAMDGPURegister(RegisterKind& RegKind, unsigned& Reg, unsigned& RegNum, unsigned& RegWidth);
569   void cvtMubufImpl(MCInst &Inst, const OperandVector &Operands, bool IsAtomic, bool IsAtomicReturn);
570 
571 public:
572   enum AMDGPUMatchResultTy {
573     Match_PreferE32 = FIRST_TARGET_MATCH_RESULT_TY
574   };
575 
576   AMDGPUAsmParser(const MCSubtargetInfo &STI, MCAsmParser &_Parser,
577                const MCInstrInfo &MII,
578                const MCTargetOptions &Options)
579       : MCTargetAsmParser(Options, STI), MII(MII), Parser(_Parser),
580         ForcedEncodingSize(0) {
581     MCAsmParserExtension::Initialize(Parser);
582 
583     if (getSTI().getFeatureBits().none()) {
584       // Set default features.
585       copySTI().ToggleFeature("SOUTHERN_ISLANDS");
586     }
587 
588     setAvailableFeatures(ComputeAvailableFeatures(getSTI().getFeatureBits()));
589   }
590 
591   AMDGPUTargetStreamer &getTargetStreamer() {
592     MCTargetStreamer &TS = *getParser().getStreamer().getTargetStreamer();
593     return static_cast<AMDGPUTargetStreamer &>(TS);
594   }
595 
596   unsigned getForcedEncodingSize() const {
597     return ForcedEncodingSize;
598   }
599 
600   void setForcedEncodingSize(unsigned Size) {
601     ForcedEncodingSize = Size;
602   }
603 
604   bool isForcedVOP3() const {
605     return ForcedEncodingSize == 64;
606   }
607 
608   std::unique_ptr<AMDGPUOperand> parseRegister();
609   bool ParseRegister(unsigned &RegNo, SMLoc &StartLoc, SMLoc &EndLoc) override;
610   unsigned checkTargetMatchPredicate(MCInst &Inst) override;
611   unsigned validateTargetOperandClass(MCParsedAsmOperand &Op,
612                                       unsigned Kind) override;
613   bool MatchAndEmitInstruction(SMLoc IDLoc, unsigned &Opcode,
614                                OperandVector &Operands, MCStreamer &Out,
615                                uint64_t &ErrorInfo,
616                                bool MatchingInlineAsm) override;
617   bool ParseDirective(AsmToken DirectiveID) override;
618   OperandMatchResultTy parseOperand(OperandVector &Operands, StringRef Mnemonic);
619   bool ParseInstruction(ParseInstructionInfo &Info, StringRef Name,
620                         SMLoc NameLoc, OperandVector &Operands) override;
621 
622   OperandMatchResultTy parseIntWithPrefix(const char *Prefix, int64_t &Int);
623   OperandMatchResultTy parseIntWithPrefix(const char *Prefix,
624                                           OperandVector &Operands,
625                                           enum AMDGPUOperand::ImmTy ImmTy = AMDGPUOperand::ImmTyNone,
626                                           bool (*ConvertResult)(int64_t&) = 0);
627   OperandMatchResultTy parseNamedBit(const char *Name, OperandVector &Operands,
628                                      enum AMDGPUOperand::ImmTy ImmTy = AMDGPUOperand::ImmTyNone);
629   OperandMatchResultTy parseStringWithPrefix(const char *Prefix, StringRef &Value);
630 
631   OperandMatchResultTy parseImm(OperandVector &Operands);
632   OperandMatchResultTy parseRegOrImm(OperandVector &Operands);
633   OperandMatchResultTy parseRegOrImmWithInputMods(OperandVector &Operands);
634 
635   void cvtDSOffset01(MCInst &Inst, const OperandVector &Operands);
636   void cvtDS(MCInst &Inst, const OperandVector &Operands);
637 
638   bool parseCnt(int64_t &IntVal);
639   OperandMatchResultTy parseSWaitCntOps(OperandVector &Operands);
640   bool parseHwregOperand(int64_t &HwRegCode, int64_t &Offset, int64_t &Width, bool &IsIdentifier);
641   OperandMatchResultTy parseHwreg(OperandVector &Operands);
642 
643 private:
644   struct OperandInfoTy {
645     int64_t Id;
646     bool IsSymbolic;
647     OperandInfoTy(int64_t Id_) : Id(Id_), IsSymbolic(false) { }
648   };
649 
650   bool parseSendMsg(OperandInfoTy &Msg, OperandInfoTy &Operation, int64_t &StreamId);
651 
652 public:
653   OperandMatchResultTy parseOptionalOperand(OperandVector &Operands);
654 
655   OperandMatchResultTy parseSendMsgOp(OperandVector &Operands);
656   OperandMatchResultTy parseSOppBrTarget(OperandVector &Operands);
657   AMDGPUOperand::Ptr defaultHwreg() const;
658 
659   void cvtMubuf(MCInst &Inst, const OperandVector &Operands) { cvtMubufImpl(Inst, Operands, false, false); }
660   void cvtMubufAtomic(MCInst &Inst, const OperandVector &Operands) { cvtMubufImpl(Inst, Operands, true, false); }
661   void cvtMubufAtomicReturn(MCInst &Inst, const OperandVector &Operands) { cvtMubufImpl(Inst, Operands, true, true); }
662   AMDGPUOperand::Ptr defaultMubufOffset() const;
663   AMDGPUOperand::Ptr defaultGLC() const;
664   AMDGPUOperand::Ptr defaultSLC() const;
665   AMDGPUOperand::Ptr defaultTFE() const;
666 
667   AMDGPUOperand::Ptr defaultDMask() const;
668   AMDGPUOperand::Ptr defaultUNorm() const;
669   AMDGPUOperand::Ptr defaultDA() const;
670   AMDGPUOperand::Ptr defaultR128() const;
671   AMDGPUOperand::Ptr defaultLWE() const;
672   AMDGPUOperand::Ptr defaultSMRDOffset() const;
673   AMDGPUOperand::Ptr defaultSMRDLiteralOffset() const;
674 
675   AMDGPUOperand::Ptr defaultClampSI() const;
676   AMDGPUOperand::Ptr defaultOModSI() const;
677 
678   OperandMatchResultTy parseOModOperand(OperandVector &Operands);
679 
680   void cvtId(MCInst &Inst, const OperandVector &Operands);
681   void cvtVOP3_2_mod(MCInst &Inst, const OperandVector &Operands);
682   void cvtVOP3(MCInst &Inst, const OperandVector &Operands);
683 
684   void cvtMIMG(MCInst &Inst, const OperandVector &Operands);
685   void cvtMIMGAtomic(MCInst &Inst, const OperandVector &Operands);
686 
687   OperandMatchResultTy parseDPPCtrl(OperandVector &Operands);
688   AMDGPUOperand::Ptr defaultRowMask() const;
689   AMDGPUOperand::Ptr defaultBankMask() const;
690   AMDGPUOperand::Ptr defaultBoundCtrl() const;
691   void cvtDPP(MCInst &Inst, const OperandVector &Operands);
692 
693   OperandMatchResultTy parseSDWASel(OperandVector &Operands);
694   OperandMatchResultTy parseSDWADstUnused(OperandVector &Operands);
695   AMDGPUOperand::Ptr defaultSDWASel() const;
696   AMDGPUOperand::Ptr defaultSDWADstUnused() const;
697 };
698 
699 struct OptionalOperand {
700   const char *Name;
701   AMDGPUOperand::ImmTy Type;
702   bool IsBit;
703   bool (*ConvertResult)(int64_t&);
704 };
705 
706 }
707 
708 static int getRegClass(RegisterKind Is, unsigned RegWidth) {
709   if (Is == IS_VGPR) {
710     switch (RegWidth) {
711       default: return -1;
712       case 1: return AMDGPU::VGPR_32RegClassID;
713       case 2: return AMDGPU::VReg_64RegClassID;
714       case 3: return AMDGPU::VReg_96RegClassID;
715       case 4: return AMDGPU::VReg_128RegClassID;
716       case 8: return AMDGPU::VReg_256RegClassID;
717       case 16: return AMDGPU::VReg_512RegClassID;
718     }
719   } else if (Is == IS_TTMP) {
720     switch (RegWidth) {
721       default: return -1;
722       case 1: return AMDGPU::TTMP_32RegClassID;
723       case 2: return AMDGPU::TTMP_64RegClassID;
724       case 4: return AMDGPU::TTMP_128RegClassID;
725     }
726   } else if (Is == IS_SGPR) {
727     switch (RegWidth) {
728       default: return -1;
729       case 1: return AMDGPU::SGPR_32RegClassID;
730       case 2: return AMDGPU::SGPR_64RegClassID;
731       case 4: return AMDGPU::SGPR_128RegClassID;
732       case 8: return AMDGPU::SReg_256RegClassID;
733       case 16: return AMDGPU::SReg_512RegClassID;
734     }
735   }
736   return -1;
737 }
738 
739 static unsigned getSpecialRegForName(StringRef RegName) {
740   return StringSwitch<unsigned>(RegName)
741     .Case("exec", AMDGPU::EXEC)
742     .Case("vcc", AMDGPU::VCC)
743     .Case("flat_scratch", AMDGPU::FLAT_SCR)
744     .Case("m0", AMDGPU::M0)
745     .Case("scc", AMDGPU::SCC)
746     .Case("tba", AMDGPU::TBA)
747     .Case("tma", AMDGPU::TMA)
748     .Case("flat_scratch_lo", AMDGPU::FLAT_SCR_LO)
749     .Case("flat_scratch_hi", AMDGPU::FLAT_SCR_HI)
750     .Case("vcc_lo", AMDGPU::VCC_LO)
751     .Case("vcc_hi", AMDGPU::VCC_HI)
752     .Case("exec_lo", AMDGPU::EXEC_LO)
753     .Case("exec_hi", AMDGPU::EXEC_HI)
754     .Case("tma_lo", AMDGPU::TMA_LO)
755     .Case("tma_hi", AMDGPU::TMA_HI)
756     .Case("tba_lo", AMDGPU::TBA_LO)
757     .Case("tba_hi", AMDGPU::TBA_HI)
758     .Default(0);
759 }
760 
761 bool AMDGPUAsmParser::ParseRegister(unsigned &RegNo, SMLoc &StartLoc, SMLoc &EndLoc) {
762   auto R = parseRegister();
763   if (!R) return true;
764   assert(R->isReg());
765   RegNo = R->getReg();
766   StartLoc = R->getStartLoc();
767   EndLoc = R->getEndLoc();
768   return false;
769 }
770 
771 bool AMDGPUAsmParser::AddNextRegisterToList(unsigned& Reg, unsigned& RegWidth, RegisterKind RegKind, unsigned Reg1, unsigned RegNum)
772 {
773   switch (RegKind) {
774   case IS_SPECIAL:
775     if (Reg == AMDGPU::EXEC_LO && Reg1 == AMDGPU::EXEC_HI) { Reg = AMDGPU::EXEC; RegWidth = 2; return true; }
776     if (Reg == AMDGPU::FLAT_SCR_LO && Reg1 == AMDGPU::FLAT_SCR_HI) { Reg = AMDGPU::FLAT_SCR; RegWidth = 2; return true; }
777     if (Reg == AMDGPU::VCC_LO && Reg1 == AMDGPU::VCC_HI) { Reg = AMDGPU::VCC; RegWidth = 2; return true; }
778     if (Reg == AMDGPU::TBA_LO && Reg1 == AMDGPU::TBA_HI) { Reg = AMDGPU::TBA; RegWidth = 2; return true; }
779     if (Reg == AMDGPU::TMA_LO && Reg1 == AMDGPU::TMA_HI) { Reg = AMDGPU::TMA; RegWidth = 2; return true; }
780     return false;
781   case IS_VGPR:
782   case IS_SGPR:
783   case IS_TTMP:
784     if (Reg1 != Reg + RegWidth) { return false; }
785     RegWidth++;
786     return true;
787   default:
788     assert(false); return false;
789   }
790 }
791 
792 bool AMDGPUAsmParser::ParseAMDGPURegister(RegisterKind& RegKind, unsigned& Reg, unsigned& RegNum, unsigned& RegWidth)
793 {
794   const MCRegisterInfo *TRI = getContext().getRegisterInfo();
795   if (getLexer().is(AsmToken::Identifier)) {
796     StringRef RegName = Parser.getTok().getString();
797     if ((Reg = getSpecialRegForName(RegName))) {
798       Parser.Lex();
799       RegKind = IS_SPECIAL;
800     } else {
801       unsigned RegNumIndex = 0;
802       if (RegName[0] == 'v') { RegNumIndex = 1; RegKind = IS_VGPR; }
803       else if (RegName[0] == 's') { RegNumIndex = 1; RegKind = IS_SGPR; }
804       else if (RegName.startswith("ttmp")) { RegNumIndex = strlen("ttmp"); RegKind = IS_TTMP; }
805       else { return false; }
806       if (RegName.size() > RegNumIndex) {
807         // Single 32-bit register: vXX.
808         if (RegName.substr(RegNumIndex).getAsInteger(10, RegNum)) { return false; }
809         Parser.Lex();
810         RegWidth = 1;
811       } else {
812         // Range of registers: v[XX:YY].
813         Parser.Lex();
814         int64_t RegLo, RegHi;
815         if (getLexer().isNot(AsmToken::LBrac)) { return false; }
816         Parser.Lex();
817 
818         if (getParser().parseAbsoluteExpression(RegLo)) { return false; }
819 
820         if (getLexer().isNot(AsmToken::Colon)) { return false; }
821         Parser.Lex();
822 
823         if (getParser().parseAbsoluteExpression(RegHi)) { return false; }
824 
825         if (getLexer().isNot(AsmToken::RBrac)) { return false; }
826         Parser.Lex();
827 
828         RegNum = (unsigned) RegLo;
829         RegWidth = (RegHi - RegLo) + 1;
830       }
831     }
832   } else if (getLexer().is(AsmToken::LBrac)) {
833     // List of consecutive registers: [s0,s1,s2,s3]
834     Parser.Lex();
835     if (!ParseAMDGPURegister(RegKind, Reg, RegNum, RegWidth)) { return false; }
836     if (RegWidth != 1) { return false; }
837     RegisterKind RegKind1;
838     unsigned Reg1, RegNum1, RegWidth1;
839     do {
840       if (getLexer().is(AsmToken::Comma)) {
841         Parser.Lex();
842       } else if (getLexer().is(AsmToken::RBrac)) {
843         Parser.Lex();
844         break;
845       } else if (ParseAMDGPURegister(RegKind1, Reg1, RegNum1, RegWidth1)) {
846         if (RegWidth1 != 1) { return false; }
847         if (RegKind1 != RegKind) { return false; }
848         if (!AddNextRegisterToList(Reg, RegWidth, RegKind1, Reg1, RegNum1)) { return false; }
849       } else {
850         return false;
851       }
852     } while (true);
853   } else {
854     return false;
855   }
856   switch (RegKind) {
857   case IS_SPECIAL:
858     RegNum = 0;
859     RegWidth = 1;
860     break;
861   case IS_VGPR:
862   case IS_SGPR:
863   case IS_TTMP:
864   {
865     unsigned Size = 1;
866     if (RegKind == IS_SGPR || RegKind == IS_TTMP) {
867       // SGPR and TTMP registers must be are aligned. Max required alignment is 4 dwords.
868       Size = std::min(RegWidth, 4u);
869     }
870     if (RegNum % Size != 0) { return false; }
871     RegNum = RegNum / Size;
872     int RCID = getRegClass(RegKind, RegWidth);
873     if (RCID == -1) { return false; }
874     const MCRegisterClass RC = TRI->getRegClass(RCID);
875     if (RegNum >= RC.getNumRegs()) { return false; }
876     Reg = RC.getRegister(RegNum);
877     break;
878   }
879 
880   default:
881     assert(false); return false;
882   }
883 
884   if (!subtargetHasRegister(*TRI, Reg)) { return false; }
885   return true;
886 }
887 
888 std::unique_ptr<AMDGPUOperand> AMDGPUAsmParser::parseRegister() {
889   const auto &Tok = Parser.getTok();
890   SMLoc StartLoc = Tok.getLoc();
891   SMLoc EndLoc = Tok.getEndLoc();
892   const MCRegisterInfo *TRI = getContext().getRegisterInfo();
893 
894   RegisterKind RegKind;
895   unsigned Reg, RegNum, RegWidth;
896 
897   if (!ParseAMDGPURegister(RegKind, Reg, RegNum, RegWidth)) {
898     return nullptr;
899   }
900   return AMDGPUOperand::CreateReg(Reg, StartLoc, EndLoc,
901                                   TRI, &getSTI(), false);
902 }
903 
904 AMDGPUAsmParser::OperandMatchResultTy
905 AMDGPUAsmParser::parseImm(OperandVector &Operands) {
906   bool Minus = false;
907   if (getLexer().getKind() == AsmToken::Minus) {
908     Minus = true;
909     Parser.Lex();
910   }
911 
912   SMLoc S = Parser.getTok().getLoc();
913   switch(getLexer().getKind()) {
914   case AsmToken::Integer: {
915     int64_t IntVal;
916     if (getParser().parseAbsoluteExpression(IntVal))
917       return MatchOperand_ParseFail;
918     if (!isInt<32>(IntVal) && !isUInt<32>(IntVal)) {
919       Error(S, "invalid immediate: only 32-bit values are legal");
920       return MatchOperand_ParseFail;
921     }
922 
923     if (Minus)
924       IntVal *= -1;
925     Operands.push_back(AMDGPUOperand::CreateImm(IntVal, S));
926     return MatchOperand_Success;
927   }
928   case AsmToken::Real: {
929     // FIXME: We should emit an error if a double precisions floating-point
930     // value is used.  I'm not sure the best way to detect this.
931     int64_t IntVal;
932     if (getParser().parseAbsoluteExpression(IntVal))
933       return MatchOperand_ParseFail;
934 
935     APFloat F((float)BitsToDouble(IntVal));
936     if (Minus)
937       F.changeSign();
938     Operands.push_back(
939         AMDGPUOperand::CreateImm(F.bitcastToAPInt().getZExtValue(), S,
940                                  AMDGPUOperand::ImmTyNone, true));
941     return MatchOperand_Success;
942   }
943   default:
944     return Minus ? MatchOperand_ParseFail : MatchOperand_NoMatch;
945   }
946 }
947 
948 AMDGPUAsmParser::OperandMatchResultTy
949 AMDGPUAsmParser::parseRegOrImm(OperandVector &Operands) {
950   auto res = parseImm(Operands);
951   if (res != MatchOperand_NoMatch) {
952     return res;
953   }
954 
955   if (auto R = parseRegister()) {
956     assert(R->isReg());
957     R->Reg.IsForcedVOP3 = isForcedVOP3();
958     Operands.push_back(std::move(R));
959     return MatchOperand_Success;
960   }
961   return MatchOperand_ParseFail;
962 }
963 
964 AMDGPUAsmParser::OperandMatchResultTy
965 AMDGPUAsmParser::parseRegOrImmWithInputMods(OperandVector &Operands) {
966   // XXX: During parsing we can't determine if minus sign means
967   // negate-modifier or negative immediate value.
968   // By default we suppose it is modifier.
969   bool Negate = false, Abs = false, Abs2 = false;
970 
971   if (getLexer().getKind()== AsmToken::Minus) {
972     Parser.Lex();
973     Negate = true;
974   }
975 
976   if (getLexer().getKind() == AsmToken::Identifier && Parser.getTok().getString() == "abs") {
977     Parser.Lex();
978     Abs2 = true;
979     if (getLexer().isNot(AsmToken::LParen)) {
980       Error(Parser.getTok().getLoc(), "expected left paren after abs");
981       return MatchOperand_ParseFail;
982     }
983     Parser.Lex();
984   }
985 
986   if (getLexer().getKind() == AsmToken::Pipe) {
987     if (Abs2) {
988       Error(Parser.getTok().getLoc(), "expected register or immediate");
989       return MatchOperand_ParseFail;
990     }
991     Parser.Lex();
992     Abs = true;
993   }
994 
995   auto Res = parseRegOrImm(Operands);
996   if (Res != MatchOperand_Success) {
997     return Res;
998   }
999 
1000   unsigned Modifiers = 0;
1001   if (Negate) {
1002     Modifiers |= 0x1;
1003   }
1004   if (Abs) {
1005     if (getLexer().getKind() != AsmToken::Pipe) {
1006       Error(Parser.getTok().getLoc(), "expected vertical bar");
1007       return MatchOperand_ParseFail;
1008     }
1009     Parser.Lex();
1010     Modifiers |= 0x2;
1011   }
1012   if (Abs2) {
1013     if (getLexer().isNot(AsmToken::RParen)) {
1014       Error(Parser.getTok().getLoc(), "expected closing parentheses");
1015       return MatchOperand_ParseFail;
1016     }
1017     Parser.Lex();
1018     Modifiers |= 0x2;
1019   }
1020 
1021   if (Modifiers) {
1022     AMDGPUOperand &Op = static_cast<AMDGPUOperand &>(*Operands.back());
1023     Op.setModifiers(Modifiers);
1024   }
1025   return MatchOperand_Success;
1026 }
1027 
1028 
1029 unsigned AMDGPUAsmParser::checkTargetMatchPredicate(MCInst &Inst) {
1030 
1031   uint64_t TSFlags = MII.get(Inst.getOpcode()).TSFlags;
1032 
1033   if ((getForcedEncodingSize() == 32 && (TSFlags & SIInstrFlags::VOP3)) ||
1034       (getForcedEncodingSize() == 64 && !(TSFlags & SIInstrFlags::VOP3)))
1035     return Match_InvalidOperand;
1036 
1037   if ((TSFlags & SIInstrFlags::VOP3) &&
1038       (TSFlags & SIInstrFlags::VOPAsmPrefer32Bit) &&
1039       getForcedEncodingSize() != 64)
1040     return Match_PreferE32;
1041 
1042   return Match_Success;
1043 }
1044 
1045 bool AMDGPUAsmParser::MatchAndEmitInstruction(SMLoc IDLoc, unsigned &Opcode,
1046                                               OperandVector &Operands,
1047                                               MCStreamer &Out,
1048                                               uint64_t &ErrorInfo,
1049                                               bool MatchingInlineAsm) {
1050   MCInst Inst;
1051 
1052   switch (MatchInstructionImpl(Operands, Inst, ErrorInfo, MatchingInlineAsm)) {
1053     default: break;
1054     case Match_Success:
1055       Inst.setLoc(IDLoc);
1056       Out.EmitInstruction(Inst, getSTI());
1057       return false;
1058     case Match_MissingFeature:
1059       return Error(IDLoc, "instruction not supported on this GPU");
1060 
1061     case Match_MnemonicFail:
1062       return Error(IDLoc, "unrecognized instruction mnemonic");
1063 
1064     case Match_InvalidOperand: {
1065       SMLoc ErrorLoc = IDLoc;
1066       if (ErrorInfo != ~0ULL) {
1067         if (ErrorInfo >= Operands.size()) {
1068           return Error(IDLoc, "too few operands for instruction");
1069         }
1070         ErrorLoc = ((AMDGPUOperand &)*Operands[ErrorInfo]).getStartLoc();
1071         if (ErrorLoc == SMLoc())
1072           ErrorLoc = IDLoc;
1073       }
1074       return Error(ErrorLoc, "invalid operand for instruction");
1075     }
1076     case Match_PreferE32:
1077       return Error(IDLoc, "internal error: instruction without _e64 suffix "
1078                           "should be encoded as e32");
1079   }
1080   llvm_unreachable("Implement any new match types added!");
1081 }
1082 
1083 bool AMDGPUAsmParser::ParseDirectiveMajorMinor(uint32_t &Major,
1084                                                uint32_t &Minor) {
1085   if (getLexer().isNot(AsmToken::Integer))
1086     return TokError("invalid major version");
1087 
1088   Major = getLexer().getTok().getIntVal();
1089   Lex();
1090 
1091   if (getLexer().isNot(AsmToken::Comma))
1092     return TokError("minor version number required, comma expected");
1093   Lex();
1094 
1095   if (getLexer().isNot(AsmToken::Integer))
1096     return TokError("invalid minor version");
1097 
1098   Minor = getLexer().getTok().getIntVal();
1099   Lex();
1100 
1101   return false;
1102 }
1103 
1104 bool AMDGPUAsmParser::ParseDirectiveHSACodeObjectVersion() {
1105 
1106   uint32_t Major;
1107   uint32_t Minor;
1108 
1109   if (ParseDirectiveMajorMinor(Major, Minor))
1110     return true;
1111 
1112   getTargetStreamer().EmitDirectiveHSACodeObjectVersion(Major, Minor);
1113   return false;
1114 }
1115 
1116 bool AMDGPUAsmParser::ParseDirectiveHSACodeObjectISA() {
1117 
1118   uint32_t Major;
1119   uint32_t Minor;
1120   uint32_t Stepping;
1121   StringRef VendorName;
1122   StringRef ArchName;
1123 
1124   // If this directive has no arguments, then use the ISA version for the
1125   // targeted GPU.
1126   if (getLexer().is(AsmToken::EndOfStatement)) {
1127     AMDGPU::IsaVersion Isa = AMDGPU::getIsaVersion(getSTI().getFeatureBits());
1128     getTargetStreamer().EmitDirectiveHSACodeObjectISA(Isa.Major, Isa.Minor,
1129                                                       Isa.Stepping,
1130                                                       "AMD", "AMDGPU");
1131     return false;
1132   }
1133 
1134 
1135   if (ParseDirectiveMajorMinor(Major, Minor))
1136     return true;
1137 
1138   if (getLexer().isNot(AsmToken::Comma))
1139     return TokError("stepping version number required, comma expected");
1140   Lex();
1141 
1142   if (getLexer().isNot(AsmToken::Integer))
1143     return TokError("invalid stepping version");
1144 
1145   Stepping = getLexer().getTok().getIntVal();
1146   Lex();
1147 
1148   if (getLexer().isNot(AsmToken::Comma))
1149     return TokError("vendor name required, comma expected");
1150   Lex();
1151 
1152   if (getLexer().isNot(AsmToken::String))
1153     return TokError("invalid vendor name");
1154 
1155   VendorName = getLexer().getTok().getStringContents();
1156   Lex();
1157 
1158   if (getLexer().isNot(AsmToken::Comma))
1159     return TokError("arch name required, comma expected");
1160   Lex();
1161 
1162   if (getLexer().isNot(AsmToken::String))
1163     return TokError("invalid arch name");
1164 
1165   ArchName = getLexer().getTok().getStringContents();
1166   Lex();
1167 
1168   getTargetStreamer().EmitDirectiveHSACodeObjectISA(Major, Minor, Stepping,
1169                                                     VendorName, ArchName);
1170   return false;
1171 }
1172 
1173 bool AMDGPUAsmParser::ParseAMDKernelCodeTValue(StringRef ID,
1174                                                amd_kernel_code_t &Header) {
1175   SmallString<40> ErrStr;
1176   raw_svector_ostream Err(ErrStr);
1177   if (!parseAmdKernelCodeField(ID, getLexer(), Header, Err)) {
1178     return TokError(Err.str());
1179   }
1180   Lex();
1181   return false;
1182 }
1183 
1184 bool AMDGPUAsmParser::ParseDirectiveAMDKernelCodeT() {
1185 
1186   amd_kernel_code_t Header;
1187   AMDGPU::initDefaultAMDKernelCodeT(Header, getSTI().getFeatureBits());
1188 
1189   while (true) {
1190 
1191     if (getLexer().isNot(AsmToken::EndOfStatement))
1192       return TokError("amd_kernel_code_t values must begin on a new line");
1193 
1194     // Lex EndOfStatement.  This is in a while loop, because lexing a comment
1195     // will set the current token to EndOfStatement.
1196     while(getLexer().is(AsmToken::EndOfStatement))
1197       Lex();
1198 
1199     if (getLexer().isNot(AsmToken::Identifier))
1200       return TokError("expected value identifier or .end_amd_kernel_code_t");
1201 
1202     StringRef ID = getLexer().getTok().getIdentifier();
1203     Lex();
1204 
1205     if (ID == ".end_amd_kernel_code_t")
1206       break;
1207 
1208     if (ParseAMDKernelCodeTValue(ID, Header))
1209       return true;
1210   }
1211 
1212   getTargetStreamer().EmitAMDKernelCodeT(Header);
1213 
1214   return false;
1215 }
1216 
1217 bool AMDGPUAsmParser::ParseSectionDirectiveHSAText() {
1218   getParser().getStreamer().SwitchSection(
1219       AMDGPU::getHSATextSection(getContext()));
1220   return false;
1221 }
1222 
1223 bool AMDGPUAsmParser::ParseDirectiveAMDGPUHsaKernel() {
1224   if (getLexer().isNot(AsmToken::Identifier))
1225     return TokError("expected symbol name");
1226 
1227   StringRef KernelName = Parser.getTok().getString();
1228 
1229   getTargetStreamer().EmitAMDGPUSymbolType(KernelName,
1230                                            ELF::STT_AMDGPU_HSA_KERNEL);
1231   Lex();
1232   return false;
1233 }
1234 
1235 bool AMDGPUAsmParser::ParseDirectiveAMDGPUHsaModuleGlobal() {
1236   if (getLexer().isNot(AsmToken::Identifier))
1237     return TokError("expected symbol name");
1238 
1239   StringRef GlobalName = Parser.getTok().getIdentifier();
1240 
1241   getTargetStreamer().EmitAMDGPUHsaModuleScopeGlobal(GlobalName);
1242   Lex();
1243   return false;
1244 }
1245 
1246 bool AMDGPUAsmParser::ParseDirectiveAMDGPUHsaProgramGlobal() {
1247   if (getLexer().isNot(AsmToken::Identifier))
1248     return TokError("expected symbol name");
1249 
1250   StringRef GlobalName = Parser.getTok().getIdentifier();
1251 
1252   getTargetStreamer().EmitAMDGPUHsaProgramScopeGlobal(GlobalName);
1253   Lex();
1254   return false;
1255 }
1256 
1257 bool AMDGPUAsmParser::ParseSectionDirectiveHSADataGlobalAgent() {
1258   getParser().getStreamer().SwitchSection(
1259       AMDGPU::getHSADataGlobalAgentSection(getContext()));
1260   return false;
1261 }
1262 
1263 bool AMDGPUAsmParser::ParseSectionDirectiveHSADataGlobalProgram() {
1264   getParser().getStreamer().SwitchSection(
1265       AMDGPU::getHSADataGlobalProgramSection(getContext()));
1266   return false;
1267 }
1268 
1269 bool AMDGPUAsmParser::ParseSectionDirectiveHSARodataReadonlyAgent() {
1270   getParser().getStreamer().SwitchSection(
1271       AMDGPU::getHSARodataReadonlyAgentSection(getContext()));
1272   return false;
1273 }
1274 
1275 bool AMDGPUAsmParser::ParseDirective(AsmToken DirectiveID) {
1276   StringRef IDVal = DirectiveID.getString();
1277 
1278   if (IDVal == ".hsa_code_object_version")
1279     return ParseDirectiveHSACodeObjectVersion();
1280 
1281   if (IDVal == ".hsa_code_object_isa")
1282     return ParseDirectiveHSACodeObjectISA();
1283 
1284   if (IDVal == ".amd_kernel_code_t")
1285     return ParseDirectiveAMDKernelCodeT();
1286 
1287   if (IDVal == ".hsatext")
1288     return ParseSectionDirectiveHSAText();
1289 
1290   if (IDVal == ".amdgpu_hsa_kernel")
1291     return ParseDirectiveAMDGPUHsaKernel();
1292 
1293   if (IDVal == ".amdgpu_hsa_module_global")
1294     return ParseDirectiveAMDGPUHsaModuleGlobal();
1295 
1296   if (IDVal == ".amdgpu_hsa_program_global")
1297     return ParseDirectiveAMDGPUHsaProgramGlobal();
1298 
1299   if (IDVal == ".hsadata_global_agent")
1300     return ParseSectionDirectiveHSADataGlobalAgent();
1301 
1302   if (IDVal == ".hsadata_global_program")
1303     return ParseSectionDirectiveHSADataGlobalProgram();
1304 
1305   if (IDVal == ".hsarodata_readonly_agent")
1306     return ParseSectionDirectiveHSARodataReadonlyAgent();
1307 
1308   return true;
1309 }
1310 
1311 bool AMDGPUAsmParser::subtargetHasRegister(const MCRegisterInfo &MRI,
1312                                            unsigned RegNo) const {
1313   if (isCI())
1314     return true;
1315 
1316   if (isSI()) {
1317     // No flat_scr
1318     switch (RegNo) {
1319     case AMDGPU::FLAT_SCR:
1320     case AMDGPU::FLAT_SCR_LO:
1321     case AMDGPU::FLAT_SCR_HI:
1322       return false;
1323     default:
1324       return true;
1325     }
1326   }
1327 
1328   // VI only has 102 SGPRs, so make sure we aren't trying to use the 2 more that
1329   // SI/CI have.
1330   for (MCRegAliasIterator R(AMDGPU::SGPR102_SGPR103, &MRI, true);
1331        R.isValid(); ++R) {
1332     if (*R == RegNo)
1333       return false;
1334   }
1335 
1336   return true;
1337 }
1338 
1339 AMDGPUAsmParser::OperandMatchResultTy
1340 AMDGPUAsmParser::parseOperand(OperandVector &Operands, StringRef Mnemonic) {
1341 
1342   // Try to parse with a custom parser
1343   OperandMatchResultTy ResTy = MatchOperandParserImpl(Operands, Mnemonic);
1344 
1345   // If we successfully parsed the operand or if there as an error parsing,
1346   // we are done.
1347   //
1348   // If we are parsing after we reach EndOfStatement then this means we
1349   // are appending default values to the Operands list.  This is only done
1350   // by custom parser, so we shouldn't continue on to the generic parsing.
1351   if (ResTy == MatchOperand_Success || ResTy == MatchOperand_ParseFail ||
1352       getLexer().is(AsmToken::EndOfStatement))
1353     return ResTy;
1354 
1355   ResTy = parseRegOrImm(Operands);
1356 
1357   if (ResTy == MatchOperand_Success)
1358     return ResTy;
1359 
1360   if (getLexer().getKind() == AsmToken::Identifier) {
1361     const auto &Tok = Parser.getTok();
1362     Operands.push_back(AMDGPUOperand::CreateToken(Tok.getString(), Tok.getLoc()));
1363     Parser.Lex();
1364     return MatchOperand_Success;
1365   }
1366   return MatchOperand_NoMatch;
1367 }
1368 
1369 bool AMDGPUAsmParser::ParseInstruction(ParseInstructionInfo &Info,
1370                                        StringRef Name,
1371                                        SMLoc NameLoc, OperandVector &Operands) {
1372 
1373   // Clear any forced encodings from the previous instruction.
1374   setForcedEncodingSize(0);
1375 
1376   if (Name.endswith("_e64"))
1377     setForcedEncodingSize(64);
1378   else if (Name.endswith("_e32"))
1379     setForcedEncodingSize(32);
1380 
1381   // Add the instruction mnemonic
1382   Operands.push_back(AMDGPUOperand::CreateToken(Name, NameLoc));
1383 
1384 
1385   if (Name.endswith("_e64")) { Name = Name.substr(0, Name.size() - 4); }
1386   if (Name.endswith("_e32")) { Name = Name.substr(0, Name.size() - 4); }
1387 
1388   while (!getLexer().is(AsmToken::EndOfStatement)) {
1389     AMDGPUAsmParser::OperandMatchResultTy Res = parseOperand(Operands, Name);
1390 
1391     // Eat the comma or space if there is one.
1392     if (getLexer().is(AsmToken::Comma))
1393       Parser.Lex();
1394 
1395     switch (Res) {
1396       case MatchOperand_Success: break;
1397       case MatchOperand_ParseFail:
1398         Error(getLexer().getLoc(), "failed parsing operand.");
1399         while (!getLexer().is(AsmToken::EndOfStatement)) {
1400           Parser.Lex();
1401         }
1402         return true;
1403       case MatchOperand_NoMatch:
1404         Error(getLexer().getLoc(), "not a valid operand.");
1405         while (!getLexer().is(AsmToken::EndOfStatement)) {
1406           Parser.Lex();
1407         }
1408         return true;
1409     }
1410   }
1411 
1412   return false;
1413 }
1414 
1415 //===----------------------------------------------------------------------===//
1416 // Utility functions
1417 //===----------------------------------------------------------------------===//
1418 
1419 AMDGPUAsmParser::OperandMatchResultTy
1420 AMDGPUAsmParser::parseIntWithPrefix(const char *Prefix, int64_t &Int) {
1421   switch(getLexer().getKind()) {
1422     default: return MatchOperand_NoMatch;
1423     case AsmToken::Identifier: {
1424       StringRef Name = Parser.getTok().getString();
1425       if (!Name.equals(Prefix)) {
1426         return MatchOperand_NoMatch;
1427       }
1428 
1429       Parser.Lex();
1430       if (getLexer().isNot(AsmToken::Colon))
1431         return MatchOperand_ParseFail;
1432 
1433       Parser.Lex();
1434       if (getLexer().isNot(AsmToken::Integer))
1435         return MatchOperand_ParseFail;
1436 
1437       if (getParser().parseAbsoluteExpression(Int))
1438         return MatchOperand_ParseFail;
1439       break;
1440     }
1441   }
1442   return MatchOperand_Success;
1443 }
1444 
1445 AMDGPUAsmParser::OperandMatchResultTy
1446 AMDGPUAsmParser::parseIntWithPrefix(const char *Prefix, OperandVector &Operands,
1447                                     enum AMDGPUOperand::ImmTy ImmTy,
1448                                     bool (*ConvertResult)(int64_t&)) {
1449 
1450   SMLoc S = Parser.getTok().getLoc();
1451   int64_t Value = 0;
1452 
1453   AMDGPUAsmParser::OperandMatchResultTy Res = parseIntWithPrefix(Prefix, Value);
1454   if (Res != MatchOperand_Success)
1455     return Res;
1456 
1457   if (ConvertResult && !ConvertResult(Value)) {
1458     return MatchOperand_ParseFail;
1459   }
1460 
1461   Operands.push_back(AMDGPUOperand::CreateImm(Value, S, ImmTy));
1462   return MatchOperand_Success;
1463 }
1464 
1465 AMDGPUAsmParser::OperandMatchResultTy
1466 AMDGPUAsmParser::parseNamedBit(const char *Name, OperandVector &Operands,
1467                                enum AMDGPUOperand::ImmTy ImmTy) {
1468   int64_t Bit = 0;
1469   SMLoc S = Parser.getTok().getLoc();
1470 
1471   // We are at the end of the statement, and this is a default argument, so
1472   // use a default value.
1473   if (getLexer().isNot(AsmToken::EndOfStatement)) {
1474     switch(getLexer().getKind()) {
1475       case AsmToken::Identifier: {
1476         StringRef Tok = Parser.getTok().getString();
1477         if (Tok == Name) {
1478           Bit = 1;
1479           Parser.Lex();
1480         } else if (Tok.startswith("no") && Tok.endswith(Name)) {
1481           Bit = 0;
1482           Parser.Lex();
1483         } else {
1484           return MatchOperand_NoMatch;
1485         }
1486         break;
1487       }
1488       default:
1489         return MatchOperand_NoMatch;
1490     }
1491   }
1492 
1493   Operands.push_back(AMDGPUOperand::CreateImm(Bit, S, ImmTy));
1494   return MatchOperand_Success;
1495 }
1496 
1497 typedef std::map<enum AMDGPUOperand::ImmTy, unsigned> OptionalImmIndexMap;
1498 
1499 void addOptionalImmOperand(MCInst& Inst, const OperandVector& Operands,
1500                            OptionalImmIndexMap& OptionalIdx,
1501                            enum AMDGPUOperand::ImmTy ImmT, int64_t Default = 0) {
1502   auto i = OptionalIdx.find(ImmT);
1503   if (i != OptionalIdx.end()) {
1504     unsigned Idx = i->second;
1505     ((AMDGPUOperand &)*Operands[Idx]).addImmOperands(Inst, 1);
1506   } else {
1507     Inst.addOperand(MCOperand::createImm(Default));
1508   }
1509 }
1510 
1511 AMDGPUAsmParser::OperandMatchResultTy
1512 AMDGPUAsmParser::parseStringWithPrefix(const char *Prefix, StringRef &Value) {
1513   if (getLexer().isNot(AsmToken::Identifier)) {
1514     return MatchOperand_NoMatch;
1515   }
1516   StringRef Tok = Parser.getTok().getString();
1517   if (Tok != Prefix) {
1518     return MatchOperand_NoMatch;
1519   }
1520 
1521   Parser.Lex();
1522   if (getLexer().isNot(AsmToken::Colon)) {
1523     return MatchOperand_ParseFail;
1524   }
1525 
1526   Parser.Lex();
1527   if (getLexer().isNot(AsmToken::Identifier)) {
1528     return MatchOperand_ParseFail;
1529   }
1530 
1531   Value = Parser.getTok().getString();
1532   return MatchOperand_Success;
1533 }
1534 
1535 //===----------------------------------------------------------------------===//
1536 // ds
1537 //===----------------------------------------------------------------------===//
1538 
1539 void AMDGPUAsmParser::cvtDSOffset01(MCInst &Inst,
1540                                     const OperandVector &Operands) {
1541 
1542   OptionalImmIndexMap OptionalIdx;
1543 
1544   for (unsigned i = 1, e = Operands.size(); i != e; ++i) {
1545     AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[i]);
1546 
1547     // Add the register arguments
1548     if (Op.isReg()) {
1549       Op.addRegOperands(Inst, 1);
1550       continue;
1551     }
1552 
1553     // Handle optional arguments
1554     OptionalIdx[Op.getImmTy()] = i;
1555   }
1556 
1557   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyOffset0);
1558   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyOffset1);
1559   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyGDS);
1560 
1561   Inst.addOperand(MCOperand::createReg(AMDGPU::M0)); // m0
1562 }
1563 
1564 void AMDGPUAsmParser::cvtDS(MCInst &Inst, const OperandVector &Operands) {
1565 
1566   std::map<enum AMDGPUOperand::ImmTy, unsigned> OptionalIdx;
1567   bool GDSOnly = false;
1568 
1569   for (unsigned i = 1, e = Operands.size(); i != e; ++i) {
1570     AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[i]);
1571 
1572     // Add the register arguments
1573     if (Op.isReg()) {
1574       Op.addRegOperands(Inst, 1);
1575       continue;
1576     }
1577 
1578     if (Op.isToken() && Op.getToken() == "gds") {
1579       GDSOnly = true;
1580       continue;
1581     }
1582 
1583     // Handle optional arguments
1584     OptionalIdx[Op.getImmTy()] = i;
1585   }
1586 
1587   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyOffset);
1588   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyGDS);
1589 
1590   if (!GDSOnly) {
1591     addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyGDS);
1592   }
1593   Inst.addOperand(MCOperand::createReg(AMDGPU::M0)); // m0
1594 }
1595 
1596 
1597 //===----------------------------------------------------------------------===//
1598 // s_waitcnt
1599 //===----------------------------------------------------------------------===//
1600 
1601 bool AMDGPUAsmParser::parseCnt(int64_t &IntVal) {
1602   StringRef CntName = Parser.getTok().getString();
1603   int64_t CntVal;
1604 
1605   Parser.Lex();
1606   if (getLexer().isNot(AsmToken::LParen))
1607     return true;
1608 
1609   Parser.Lex();
1610   if (getLexer().isNot(AsmToken::Integer))
1611     return true;
1612 
1613   if (getParser().parseAbsoluteExpression(CntVal))
1614     return true;
1615 
1616   if (getLexer().isNot(AsmToken::RParen))
1617     return true;
1618 
1619   Parser.Lex();
1620   if (getLexer().is(AsmToken::Amp) || getLexer().is(AsmToken::Comma))
1621     Parser.Lex();
1622 
1623   int CntShift;
1624   int CntMask;
1625 
1626   if (CntName == "vmcnt") {
1627     CntMask = 0xf;
1628     CntShift = 0;
1629   } else if (CntName == "expcnt") {
1630     CntMask = 0x7;
1631     CntShift = 4;
1632   } else if (CntName == "lgkmcnt") {
1633     CntMask = 0xf;
1634     CntShift = 8;
1635   } else {
1636     return true;
1637   }
1638 
1639   IntVal &= ~(CntMask << CntShift);
1640   IntVal |= (CntVal << CntShift);
1641   return false;
1642 }
1643 
1644 AMDGPUAsmParser::OperandMatchResultTy
1645 AMDGPUAsmParser::parseSWaitCntOps(OperandVector &Operands) {
1646   // Disable all counters by default.
1647   // vmcnt   [3:0]
1648   // expcnt  [6:4]
1649   // lgkmcnt [11:8]
1650   int64_t CntVal = 0xf7f;
1651   SMLoc S = Parser.getTok().getLoc();
1652 
1653   switch(getLexer().getKind()) {
1654     default: return MatchOperand_ParseFail;
1655     case AsmToken::Integer:
1656       // The operand can be an integer value.
1657       if (getParser().parseAbsoluteExpression(CntVal))
1658         return MatchOperand_ParseFail;
1659       break;
1660 
1661     case AsmToken::Identifier:
1662       do {
1663         if (parseCnt(CntVal))
1664           return MatchOperand_ParseFail;
1665       } while(getLexer().isNot(AsmToken::EndOfStatement));
1666       break;
1667   }
1668   Operands.push_back(AMDGPUOperand::CreateImm(CntVal, S));
1669   return MatchOperand_Success;
1670 }
1671 
1672 bool AMDGPUAsmParser::parseHwregOperand(int64_t &HwRegCode, int64_t &Offset, int64_t &Width, bool &IsIdentifier) {
1673   if (Parser.getTok().getString() != "hwreg")
1674     return true;
1675   Parser.Lex();
1676 
1677   if (getLexer().isNot(AsmToken::LParen))
1678     return true;
1679   Parser.Lex();
1680 
1681   if (getLexer().is(AsmToken::Identifier)) {
1682     IsIdentifier = true;
1683     HwRegCode = StringSwitch<unsigned>(Parser.getTok().getString())
1684       .Case("HW_REG_MODE"     , 1)
1685       .Case("HW_REG_STATUS"   , 2)
1686       .Case("HW_REG_TRAPSTS"  , 3)
1687       .Case("HW_REG_HW_ID"    , 4)
1688       .Case("HW_REG_GPR_ALLOC", 5)
1689       .Case("HW_REG_LDS_ALLOC", 6)
1690       .Case("HW_REG_IB_STS"   , 7)
1691       .Default(-1);
1692     Parser.Lex();
1693   } else {
1694     IsIdentifier = false;
1695     if (getLexer().isNot(AsmToken::Integer))
1696       return true;
1697     if (getParser().parseAbsoluteExpression(HwRegCode))
1698       return true;
1699   }
1700 
1701   if (getLexer().is(AsmToken::RParen)) {
1702     Parser.Lex();
1703     return false;
1704   }
1705 
1706   // optional params
1707   if (getLexer().isNot(AsmToken::Comma))
1708     return true;
1709   Parser.Lex();
1710 
1711   if (getLexer().isNot(AsmToken::Integer))
1712     return true;
1713   if (getParser().parseAbsoluteExpression(Offset))
1714     return true;
1715 
1716   if (getLexer().isNot(AsmToken::Comma))
1717     return true;
1718   Parser.Lex();
1719 
1720   if (getLexer().isNot(AsmToken::Integer))
1721     return true;
1722   if (getParser().parseAbsoluteExpression(Width))
1723     return true;
1724 
1725   if (getLexer().isNot(AsmToken::RParen))
1726     return true;
1727   Parser.Lex();
1728 
1729   return false;
1730 }
1731 
1732 AMDGPUAsmParser::OperandMatchResultTy
1733 AMDGPUAsmParser::parseHwreg(OperandVector &Operands) {
1734   int64_t Imm16Val = 0;
1735   SMLoc S = Parser.getTok().getLoc();
1736 
1737   switch(getLexer().getKind()) {
1738     default: return MatchOperand_NoMatch;
1739     case AsmToken::Integer:
1740       // The operand can be an integer value.
1741       if (getParser().parseAbsoluteExpression(Imm16Val))
1742         return MatchOperand_ParseFail;
1743       if (!isInt<16>(Imm16Val) && !isUInt<16>(Imm16Val)) {
1744         Error(S, "invalid immediate: only 16-bit values are legal");
1745         // Do not return error code, but create an imm operand anyway and proceed
1746         // to the next operand, if any. That avoids unneccessary error messages.
1747       }
1748       break;
1749 
1750     case AsmToken::Identifier: {
1751         bool IsIdentifier = false;
1752         int64_t HwRegCode = -1;
1753         int64_t Offset = 0; // default
1754         int64_t Width = 32; // default
1755         if (parseHwregOperand(HwRegCode, Offset, Width, IsIdentifier))
1756           return MatchOperand_ParseFail;
1757         // HwRegCode (6) [5:0]
1758         // Offset (5) [10:6]
1759         // WidthMinusOne (5) [15:11]
1760         if (HwRegCode < 0 || HwRegCode > 63) {
1761           if (IsIdentifier)
1762             Error(S, "invalid symbolic name of hardware register");
1763           else
1764             Error(S, "invalid code of hardware register: only 6-bit values are legal");
1765         }
1766         if (Offset < 0 || Offset > 31)
1767           Error(S, "invalid bit offset: only 5-bit values are legal");
1768         if (Width < 1 || Width > 32)
1769           Error(S, "invalid bitfield width: only values from 1 to 32 are legal");
1770         Imm16Val = HwRegCode | (Offset << 6) | ((Width-1) << 11);
1771       }
1772       break;
1773   }
1774   Operands.push_back(AMDGPUOperand::CreateImm(Imm16Val, S, AMDGPUOperand::ImmTyHwreg));
1775   return MatchOperand_Success;
1776 }
1777 
1778 bool AMDGPUOperand::isSWaitCnt() const {
1779   return isImm();
1780 }
1781 
1782 bool AMDGPUOperand::isHwreg() const {
1783   return isImmTy(ImmTyHwreg);
1784 }
1785 
1786 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultHwreg() const {
1787   return AMDGPUOperand::CreateImm(0, SMLoc(), AMDGPUOperand::ImmTyHwreg);
1788 }
1789 
1790 bool AMDGPUAsmParser::parseSendMsg(OperandInfoTy &Msg, OperandInfoTy &Operation, int64_t &StreamId) {
1791   using namespace llvm::AMDGPU::SendMsg;
1792 
1793   if (Parser.getTok().getString() != "sendmsg")
1794     return true;
1795   Parser.Lex();
1796 
1797   if (getLexer().isNot(AsmToken::LParen))
1798     return true;
1799   Parser.Lex();
1800 
1801   if (getLexer().is(AsmToken::Identifier)) {
1802     Msg.IsSymbolic = true;
1803     Msg.Id = ID_UNKNOWN_;
1804     const std::string tok = Parser.getTok().getString();
1805     for (int i = ID_GAPS_FIRST_; i < ID_GAPS_LAST_; ++i) {
1806       switch(i) {
1807         default: continue; // Omit gaps.
1808         case ID_INTERRUPT: case ID_GS: case ID_GS_DONE:  case ID_SYSMSG: break;
1809       }
1810       if (tok == IdSymbolic[i]) {
1811         Msg.Id = i;
1812         break;
1813       }
1814     }
1815     Parser.Lex();
1816   } else {
1817     Msg.IsSymbolic = false;
1818     if (getLexer().isNot(AsmToken::Integer))
1819       return true;
1820     if (getParser().parseAbsoluteExpression(Msg.Id))
1821       return true;
1822     if (getLexer().is(AsmToken::Integer))
1823       if (getParser().parseAbsoluteExpression(Msg.Id))
1824         Msg.Id = ID_UNKNOWN_;
1825   }
1826   if (Msg.Id == ID_UNKNOWN_) // Don't know how to parse the rest.
1827     return false;
1828 
1829   if (!(Msg.Id == ID_GS || Msg.Id == ID_GS_DONE || Msg.Id == ID_SYSMSG)) {
1830     if (getLexer().isNot(AsmToken::RParen))
1831       return true;
1832     Parser.Lex();
1833     return false;
1834   }
1835 
1836   if (getLexer().isNot(AsmToken::Comma))
1837     return true;
1838   Parser.Lex();
1839 
1840   assert(Msg.Id == ID_GS || Msg.Id == ID_GS_DONE || Msg.Id == ID_SYSMSG);
1841   Operation.Id = ID_UNKNOWN_;
1842   if (getLexer().is(AsmToken::Identifier)) {
1843     Operation.IsSymbolic = true;
1844     const char* const *S = (Msg.Id == ID_SYSMSG) ? OpSysSymbolic : OpGsSymbolic;
1845     const int F = (Msg.Id == ID_SYSMSG) ? OP_SYS_FIRST_ : OP_GS_FIRST_;
1846     const int L = (Msg.Id == ID_SYSMSG) ? OP_SYS_LAST_ : OP_GS_LAST_;
1847     const std::string Tok = Parser.getTok().getString();
1848     for (int i = F; i < L; ++i) {
1849       if (Tok == S[i]) {
1850         Operation.Id = i;
1851         break;
1852       }
1853     }
1854     Parser.Lex();
1855   } else {
1856     Operation.IsSymbolic = false;
1857     if (getLexer().isNot(AsmToken::Integer))
1858       return true;
1859     if (getParser().parseAbsoluteExpression(Operation.Id))
1860       return true;
1861   }
1862 
1863   if ((Msg.Id == ID_GS || Msg.Id == ID_GS_DONE) && Operation.Id != OP_GS_NOP) {
1864     // Stream id is optional.
1865     if (getLexer().is(AsmToken::RParen)) {
1866       Parser.Lex();
1867       return false;
1868     }
1869 
1870     if (getLexer().isNot(AsmToken::Comma))
1871       return true;
1872     Parser.Lex();
1873 
1874     if (getLexer().isNot(AsmToken::Integer))
1875       return true;
1876     if (getParser().parseAbsoluteExpression(StreamId))
1877       return true;
1878   }
1879 
1880   if (getLexer().isNot(AsmToken::RParen))
1881     return true;
1882   Parser.Lex();
1883   return false;
1884 }
1885 
1886 AMDGPUAsmParser::OperandMatchResultTy
1887 AMDGPUAsmParser::parseSendMsgOp(OperandVector &Operands) {
1888   using namespace llvm::AMDGPU::SendMsg;
1889 
1890   int64_t Imm16Val = 0;
1891   SMLoc S = Parser.getTok().getLoc();
1892 
1893   switch(getLexer().getKind()) {
1894   default:
1895     return MatchOperand_NoMatch;
1896   case AsmToken::Integer:
1897     // The operand can be an integer value.
1898     if (getParser().parseAbsoluteExpression(Imm16Val))
1899       return MatchOperand_NoMatch;
1900     if (!isInt<16>(Imm16Val) && !isUInt<16>(Imm16Val)) {
1901       Error(S, "invalid immediate: only 16-bit values are legal");
1902       // Do not return error code, but create an imm operand anyway and proceed
1903       // to the next operand, if any. That avoids unneccessary error messages.
1904     }
1905     break;
1906   case AsmToken::Identifier: {
1907       OperandInfoTy Msg(ID_UNKNOWN_);
1908       OperandInfoTy Operation(OP_UNKNOWN_);
1909       int64_t StreamId = STREAM_ID_DEFAULT;
1910       if (parseSendMsg(Msg, Operation, StreamId))
1911         return MatchOperand_NoMatch;
1912       do {
1913         // Validate and encode message ID.
1914         if (! ((ID_INTERRUPT <= Msg.Id && Msg.Id <= ID_GS_DONE)
1915                 || Msg.Id == ID_SYSMSG)) {
1916           if (Msg.IsSymbolic)
1917             Error(S, "invalid/unsupported symbolic name of message");
1918           else
1919             Error(S, "invalid/unsupported code of message");
1920           break;
1921         }
1922         Imm16Val = Msg.Id;
1923         // Validate and encode operation ID.
1924         if (Msg.Id == ID_GS || Msg.Id == ID_GS_DONE) {
1925           if (! (OP_GS_FIRST_ <= Operation.Id && Operation.Id < OP_GS_LAST_)) {
1926             if (Operation.IsSymbolic)
1927               Error(S, "invalid symbolic name of GS_OP");
1928             else
1929               Error(S, "invalid code of GS_OP: only 2-bit values are legal");
1930             break;
1931           }
1932           if (Operation.Id == OP_GS_NOP
1933               && Msg.Id != ID_GS_DONE) {
1934             Error(S, "invalid GS_OP: NOP is for GS_DONE only");
1935             break;
1936           }
1937           Imm16Val |= (Operation.Id << OP_SHIFT_);
1938         }
1939         if (Msg.Id == ID_SYSMSG) {
1940           if (! (OP_SYS_FIRST_ <= Operation.Id && Operation.Id < OP_SYS_LAST_)) {
1941             if (Operation.IsSymbolic)
1942               Error(S, "invalid/unsupported symbolic name of SYSMSG_OP");
1943             else
1944               Error(S, "invalid/unsupported code of SYSMSG_OP");
1945             break;
1946           }
1947           Imm16Val |= (Operation.Id << OP_SHIFT_);
1948         }
1949         // Validate and encode stream ID.
1950         if ((Msg.Id == ID_GS || Msg.Id == ID_GS_DONE) && Operation.Id != OP_GS_NOP) {
1951           if (! (STREAM_ID_FIRST_ <= StreamId && StreamId < STREAM_ID_LAST_)) {
1952             Error(S, "invalid stream id: only 2-bit values are legal");
1953             break;
1954           }
1955           Imm16Val |= (StreamId << STREAM_ID_SHIFT_);
1956         }
1957       } while (0);
1958     }
1959     break;
1960   }
1961   Operands.push_back(AMDGPUOperand::CreateImm(Imm16Val, S, AMDGPUOperand::ImmTySendMsg));
1962   return MatchOperand_Success;
1963 }
1964 
1965 bool AMDGPUOperand::isSendMsg() const {
1966   return isImmTy(ImmTySendMsg);
1967 }
1968 
1969 //===----------------------------------------------------------------------===//
1970 // sopp branch targets
1971 //===----------------------------------------------------------------------===//
1972 
1973 AMDGPUAsmParser::OperandMatchResultTy
1974 AMDGPUAsmParser::parseSOppBrTarget(OperandVector &Operands) {
1975   SMLoc S = Parser.getTok().getLoc();
1976 
1977   switch (getLexer().getKind()) {
1978     default: return MatchOperand_ParseFail;
1979     case AsmToken::Integer: {
1980       int64_t Imm;
1981       if (getParser().parseAbsoluteExpression(Imm))
1982         return MatchOperand_ParseFail;
1983       Operands.push_back(AMDGPUOperand::CreateImm(Imm, S));
1984       return MatchOperand_Success;
1985     }
1986 
1987     case AsmToken::Identifier:
1988       Operands.push_back(AMDGPUOperand::CreateExpr(
1989           MCSymbolRefExpr::create(getContext().getOrCreateSymbol(
1990                                   Parser.getTok().getString()), getContext()), S));
1991       Parser.Lex();
1992       return MatchOperand_Success;
1993   }
1994 }
1995 
1996 //===----------------------------------------------------------------------===//
1997 // mubuf
1998 //===----------------------------------------------------------------------===//
1999 
2000 bool AMDGPUOperand::isMubufOffset() const {
2001   return isImmTy(ImmTyOffset) && isUInt<12>(getImm());
2002 }
2003 
2004 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultMubufOffset() const {
2005   return AMDGPUOperand::CreateImm(0, SMLoc(), AMDGPUOperand::ImmTyOffset);
2006 }
2007 
2008 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultGLC() const {
2009   return AMDGPUOperand::CreateImm(0, SMLoc(), AMDGPUOperand::ImmTyGLC);
2010 }
2011 
2012 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultSLC() const {
2013   return AMDGPUOperand::CreateImm(0, SMLoc(), AMDGPUOperand::ImmTySLC);
2014 }
2015 
2016 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultTFE() const {
2017   return AMDGPUOperand::CreateImm(0, SMLoc(), AMDGPUOperand::ImmTyTFE);
2018 }
2019 
2020 void AMDGPUAsmParser::cvtMubufImpl(MCInst &Inst,
2021                                const OperandVector &Operands,
2022                                bool IsAtomic, bool IsAtomicReturn) {
2023   OptionalImmIndexMap OptionalIdx;
2024   assert(IsAtomicReturn ? IsAtomic : true);
2025 
2026   for (unsigned i = 1, e = Operands.size(); i != e; ++i) {
2027     AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[i]);
2028 
2029     // Add the register arguments
2030     if (Op.isReg()) {
2031       Op.addRegOperands(Inst, 1);
2032       continue;
2033     }
2034 
2035     // Handle the case where soffset is an immediate
2036     if (Op.isImm() && Op.getImmTy() == AMDGPUOperand::ImmTyNone) {
2037       Op.addImmOperands(Inst, 1);
2038       continue;
2039     }
2040 
2041     // Handle tokens like 'offen' which are sometimes hard-coded into the
2042     // asm string.  There are no MCInst operands for these.
2043     if (Op.isToken()) {
2044       continue;
2045     }
2046     assert(Op.isImm());
2047 
2048     // Handle optional arguments
2049     OptionalIdx[Op.getImmTy()] = i;
2050   }
2051 
2052   // Copy $vdata_in operand and insert as $vdata for MUBUF_Atomic RTN insns.
2053   if (IsAtomicReturn) {
2054     MCInst::iterator I = Inst.begin(); // $vdata_in is always at the beginning.
2055     Inst.insert(I, *I);
2056   }
2057 
2058   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyOffset);
2059   if (!IsAtomic) { // glc is hard-coded.
2060     addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyGLC);
2061   }
2062   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySLC);
2063   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyTFE);
2064 }
2065 
2066 //===----------------------------------------------------------------------===//
2067 // mimg
2068 //===----------------------------------------------------------------------===//
2069 
2070 void AMDGPUAsmParser::cvtMIMG(MCInst &Inst, const OperandVector &Operands) {
2071   unsigned I = 1;
2072   const MCInstrDesc &Desc = MII.get(Inst.getOpcode());
2073   for (unsigned J = 0; J < Desc.getNumDefs(); ++J) {
2074     ((AMDGPUOperand &)*Operands[I++]).addRegOperands(Inst, 1);
2075   }
2076 
2077   OptionalImmIndexMap OptionalIdx;
2078 
2079   for (unsigned E = Operands.size(); I != E; ++I) {
2080     AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[I]);
2081 
2082     // Add the register arguments
2083     if (Op.isRegOrImm()) {
2084       Op.addRegOrImmOperands(Inst, 1);
2085       continue;
2086     } else if (Op.isImmModifier()) {
2087       OptionalIdx[Op.getImmTy()] = I;
2088     } else {
2089       assert(false);
2090     }
2091   }
2092 
2093   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyDMask);
2094   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyUNorm);
2095   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyGLC);
2096   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyDA);
2097   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyR128);
2098   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyTFE);
2099   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyLWE);
2100   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySLC);
2101 }
2102 
2103 void AMDGPUAsmParser::cvtMIMGAtomic(MCInst &Inst, const OperandVector &Operands) {
2104   unsigned I = 1;
2105   const MCInstrDesc &Desc = MII.get(Inst.getOpcode());
2106   for (unsigned J = 0; J < Desc.getNumDefs(); ++J) {
2107     ((AMDGPUOperand &)*Operands[I++]).addRegOperands(Inst, 1);
2108   }
2109 
2110   // Add src, same as dst
2111   ((AMDGPUOperand &)*Operands[I]).addRegOperands(Inst, 1);
2112 
2113   OptionalImmIndexMap OptionalIdx;
2114 
2115   for (unsigned E = Operands.size(); I != E; ++I) {
2116     AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[I]);
2117 
2118     // Add the register arguments
2119     if (Op.isRegOrImm()) {
2120       Op.addRegOrImmOperands(Inst, 1);
2121       continue;
2122     } else if (Op.isImmModifier()) {
2123       OptionalIdx[Op.getImmTy()] = I;
2124     } else {
2125       assert(false);
2126     }
2127   }
2128 
2129   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyDMask);
2130   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyUNorm);
2131   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyGLC);
2132   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyDA);
2133   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyR128);
2134   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyTFE);
2135   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyLWE);
2136   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySLC);
2137 }
2138 
2139 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultDMask() const {
2140   return AMDGPUOperand::CreateImm(0, SMLoc(), AMDGPUOperand::ImmTyDMask);
2141 }
2142 
2143 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultUNorm() const {
2144   return AMDGPUOperand::CreateImm(0, SMLoc(), AMDGPUOperand::ImmTyUNorm);
2145 }
2146 
2147 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultDA() const {
2148   return AMDGPUOperand::CreateImm(0, SMLoc(), AMDGPUOperand::ImmTyDA);
2149 }
2150 
2151 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultR128() const {
2152   return AMDGPUOperand::CreateImm(0, SMLoc(), AMDGPUOperand::ImmTyR128);
2153 }
2154 
2155 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultLWE() const {
2156   return AMDGPUOperand::CreateImm(0, SMLoc(), AMDGPUOperand::ImmTyLWE);
2157 }
2158 
2159 //===----------------------------------------------------------------------===//
2160 // smrd
2161 //===----------------------------------------------------------------------===//
2162 
2163 bool AMDGPUOperand::isSMRDOffset() const {
2164 
2165   // FIXME: Support 20-bit offsets on VI.  We need to to pass subtarget
2166   // information here.
2167   return isImm() && isUInt<8>(getImm());
2168 }
2169 
2170 bool AMDGPUOperand::isSMRDLiteralOffset() const {
2171   // 32-bit literals are only supported on CI and we only want to use them
2172   // when the offset is > 8-bits.
2173   return isImm() && !isUInt<8>(getImm()) && isUInt<32>(getImm());
2174 }
2175 
2176 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultSMRDOffset() const {
2177   return AMDGPUOperand::CreateImm(0, SMLoc(), AMDGPUOperand::ImmTyOffset);
2178 }
2179 
2180 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultSMRDLiteralOffset() const {
2181   return AMDGPUOperand::CreateImm(0, SMLoc(), AMDGPUOperand::ImmTyOffset);
2182 }
2183 
2184 //===----------------------------------------------------------------------===//
2185 // vop3
2186 //===----------------------------------------------------------------------===//
2187 
2188 static bool ConvertOmodMul(int64_t &Mul) {
2189   if (Mul != 1 && Mul != 2 && Mul != 4)
2190     return false;
2191 
2192   Mul >>= 1;
2193   return true;
2194 }
2195 
2196 static bool ConvertOmodDiv(int64_t &Div) {
2197   if (Div == 1) {
2198     Div = 0;
2199     return true;
2200   }
2201 
2202   if (Div == 2) {
2203     Div = 3;
2204     return true;
2205   }
2206 
2207   return false;
2208 }
2209 
2210 static bool ConvertBoundCtrl(int64_t &BoundCtrl) {
2211   if (BoundCtrl == 0) {
2212     BoundCtrl = 1;
2213     return true;
2214   } else if (BoundCtrl == -1) {
2215     BoundCtrl = 0;
2216     return true;
2217   }
2218   return false;
2219 }
2220 
2221 // Note: the order in this table matches the order of operands in AsmString.
2222 static const OptionalOperand AMDGPUOptionalOperandTable[] = {
2223   {"offen",   AMDGPUOperand::ImmTyOffen, true, nullptr},
2224   {"idxen",   AMDGPUOperand::ImmTyIdxen, true, nullptr},
2225   {"addr64",  AMDGPUOperand::ImmTyAddr64, true, nullptr},
2226   {"offset0", AMDGPUOperand::ImmTyOffset0, false, nullptr},
2227   {"offset1", AMDGPUOperand::ImmTyOffset1, false, nullptr},
2228   {"gds",     AMDGPUOperand::ImmTyGDS, true, nullptr},
2229   {"offset",  AMDGPUOperand::ImmTyOffset, false, nullptr},
2230   {"glc",     AMDGPUOperand::ImmTyGLC, true, nullptr},
2231   {"slc",     AMDGPUOperand::ImmTySLC, true, nullptr},
2232   {"tfe",     AMDGPUOperand::ImmTyTFE, true, nullptr},
2233   {"clamp",   AMDGPUOperand::ImmTyClampSI, true, nullptr},
2234   {"omod",    AMDGPUOperand::ImmTyOModSI, false, ConvertOmodMul},
2235   {"unorm",   AMDGPUOperand::ImmTyUNorm, true, nullptr},
2236   {"da",      AMDGPUOperand::ImmTyDA,    true, nullptr},
2237   {"r128",    AMDGPUOperand::ImmTyR128,  true, nullptr},
2238   {"lwe",     AMDGPUOperand::ImmTyLWE,   true, nullptr},
2239   {"dmask",   AMDGPUOperand::ImmTyDMask, false, nullptr},
2240   {"row_mask",   AMDGPUOperand::ImmTyDppRowMask, false, nullptr},
2241   {"bank_mask",  AMDGPUOperand::ImmTyDppBankMask, false, nullptr},
2242   {"bound_ctrl", AMDGPUOperand::ImmTyDppBoundCtrl, false, ConvertBoundCtrl},
2243   {"sdwa_sel",   AMDGPUOperand::ImmTySdwaSel, false, nullptr},
2244   {"dst_unused", AMDGPUOperand::ImmTySdwaDstUnused, false, nullptr},
2245 };
2246 
2247 AMDGPUAsmParser::OperandMatchResultTy AMDGPUAsmParser::parseOptionalOperand(OperandVector &Operands) {
2248   OperandMatchResultTy res;
2249   for (const OptionalOperand &Op : AMDGPUOptionalOperandTable) {
2250     // try to parse any optional operand here
2251     if (Op.IsBit) {
2252       res = parseNamedBit(Op.Name, Operands, Op.Type);
2253     } else if (Op.Type == AMDGPUOperand::ImmTyOModSI) {
2254       res = parseOModOperand(Operands);
2255     } else if (Op.Type == AMDGPUOperand::ImmTySdwaSel) {
2256       res = parseSDWASel(Operands);
2257     } else if (Op.Type == AMDGPUOperand::ImmTySdwaDstUnused) {
2258       res = parseSDWADstUnused(Operands);
2259     } else {
2260       res = parseIntWithPrefix(Op.Name, Operands, Op.Type, Op.ConvertResult);
2261     }
2262     if (res != MatchOperand_NoMatch) {
2263       return res;
2264     }
2265   }
2266   return MatchOperand_NoMatch;
2267 }
2268 
2269 AMDGPUAsmParser::OperandMatchResultTy AMDGPUAsmParser::parseOModOperand(OperandVector &Operands)
2270 {
2271   StringRef Name = Parser.getTok().getString();
2272   if (Name == "mul") {
2273     return parseIntWithPrefix("mul", Operands, AMDGPUOperand::ImmTyOModSI, ConvertOmodMul);
2274   } else if (Name == "div") {
2275     return parseIntWithPrefix("div", Operands, AMDGPUOperand::ImmTyOModSI, ConvertOmodDiv);
2276   } else {
2277     return MatchOperand_NoMatch;
2278   }
2279 }
2280 
2281 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultClampSI() const {
2282   return AMDGPUOperand::CreateImm(0, SMLoc(), AMDGPUOperand::ImmTyClampSI);
2283 }
2284 
2285 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultOModSI() const {
2286   return AMDGPUOperand::CreateImm(1, SMLoc(), AMDGPUOperand::ImmTyOModSI);
2287 }
2288 
2289 void AMDGPUAsmParser::cvtId(MCInst &Inst, const OperandVector &Operands) {
2290   unsigned I = 1;
2291   const MCInstrDesc &Desc = MII.get(Inst.getOpcode());
2292   for (unsigned J = 0; J < Desc.getNumDefs(); ++J) {
2293     ((AMDGPUOperand &)*Operands[I++]).addRegOperands(Inst, 1);
2294   }
2295   for (unsigned E = Operands.size(); I != E; ++I)
2296     ((AMDGPUOperand &)*Operands[I]).addRegOrImmOperands(Inst, 1);
2297 }
2298 
2299 void AMDGPUAsmParser::cvtVOP3_2_mod(MCInst &Inst, const OperandVector &Operands) {
2300   uint64_t TSFlags = MII.get(Inst.getOpcode()).TSFlags;
2301   if (TSFlags & SIInstrFlags::VOP3) {
2302     cvtVOP3(Inst, Operands);
2303   } else {
2304     cvtId(Inst, Operands);
2305   }
2306 }
2307 
2308 void AMDGPUAsmParser::cvtVOP3(MCInst &Inst, const OperandVector &Operands) {
2309   OptionalImmIndexMap OptionalIdx;
2310   unsigned I = 1;
2311   const MCInstrDesc &Desc = MII.get(Inst.getOpcode());
2312   for (unsigned J = 0; J < Desc.getNumDefs(); ++J) {
2313     ((AMDGPUOperand &)*Operands[I++]).addRegOperands(Inst, 1);
2314   }
2315 
2316   for (unsigned E = Operands.size(); I != E; ++I) {
2317     AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[I]);
2318     if (Op.isRegOrImmWithInputMods()) {
2319       Op.addRegOrImmWithInputModsOperands(Inst, 2);
2320     } else if (Op.isImm()) {
2321       OptionalIdx[Op.getImmTy()] = I;
2322     } else {
2323       assert(false);
2324     }
2325   }
2326 
2327   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyClampSI);
2328   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyOModSI);
2329 }
2330 
2331 //===----------------------------------------------------------------------===//
2332 // dpp
2333 //===----------------------------------------------------------------------===//
2334 
2335 bool AMDGPUOperand::isDPPCtrl() const {
2336   bool result = isImm() && getImmTy() == ImmTyDppCtrl && isUInt<9>(getImm());
2337   if (result) {
2338     int64_t Imm = getImm();
2339     return ((Imm >= 0x000) && (Imm <= 0x0ff)) ||
2340            ((Imm >= 0x101) && (Imm <= 0x10f)) ||
2341            ((Imm >= 0x111) && (Imm <= 0x11f)) ||
2342            ((Imm >= 0x121) && (Imm <= 0x12f)) ||
2343            (Imm == 0x130) ||
2344            (Imm == 0x134) ||
2345            (Imm == 0x138) ||
2346            (Imm == 0x13c) ||
2347            (Imm == 0x140) ||
2348            (Imm == 0x141) ||
2349            (Imm == 0x142) ||
2350            (Imm == 0x143);
2351   }
2352   return false;
2353 }
2354 
2355 AMDGPUAsmParser::OperandMatchResultTy
2356 AMDGPUAsmParser::parseDPPCtrl(OperandVector &Operands) {
2357   SMLoc S = Parser.getTok().getLoc();
2358   StringRef Prefix;
2359   int64_t Int;
2360 
2361   if (getLexer().getKind() == AsmToken::Identifier) {
2362     Prefix = Parser.getTok().getString();
2363   } else {
2364     return MatchOperand_NoMatch;
2365   }
2366 
2367   if (Prefix == "row_mirror") {
2368     Int = 0x140;
2369   } else if (Prefix == "row_half_mirror") {
2370     Int = 0x141;
2371   } else {
2372     // Check to prevent parseDPPCtrlOps from eating invalid tokens
2373     if (Prefix != "quad_perm"
2374         && Prefix != "row_shl"
2375         && Prefix != "row_shr"
2376         && Prefix != "row_ror"
2377         && Prefix != "wave_shl"
2378         && Prefix != "wave_rol"
2379         && Prefix != "wave_shr"
2380         && Prefix != "wave_ror"
2381         && Prefix != "row_bcast") {
2382       return MatchOperand_NoMatch;
2383     }
2384 
2385     Parser.Lex();
2386     if (getLexer().isNot(AsmToken::Colon))
2387       return MatchOperand_ParseFail;
2388 
2389     if (Prefix == "quad_perm") {
2390       // quad_perm:[%d,%d,%d,%d]
2391       Parser.Lex();
2392       if (getLexer().isNot(AsmToken::LBrac))
2393         return MatchOperand_ParseFail;
2394 
2395       Parser.Lex();
2396       if (getLexer().isNot(AsmToken::Integer))
2397         return MatchOperand_ParseFail;
2398       Int = getLexer().getTok().getIntVal();
2399 
2400       Parser.Lex();
2401       if (getLexer().isNot(AsmToken::Comma))
2402         return MatchOperand_ParseFail;
2403       Parser.Lex();
2404       if (getLexer().isNot(AsmToken::Integer))
2405         return MatchOperand_ParseFail;
2406       Int += (getLexer().getTok().getIntVal() << 2);
2407 
2408       Parser.Lex();
2409       if (getLexer().isNot(AsmToken::Comma))
2410         return MatchOperand_ParseFail;
2411       Parser.Lex();
2412       if (getLexer().isNot(AsmToken::Integer))
2413         return MatchOperand_ParseFail;
2414       Int += (getLexer().getTok().getIntVal() << 4);
2415 
2416       Parser.Lex();
2417       if (getLexer().isNot(AsmToken::Comma))
2418         return MatchOperand_ParseFail;
2419       Parser.Lex();
2420       if (getLexer().isNot(AsmToken::Integer))
2421         return MatchOperand_ParseFail;
2422       Int += (getLexer().getTok().getIntVal() << 6);
2423 
2424       Parser.Lex();
2425       if (getLexer().isNot(AsmToken::RBrac))
2426         return MatchOperand_ParseFail;
2427 
2428     } else {
2429       // sel:%d
2430       Parser.Lex();
2431       if (getLexer().isNot(AsmToken::Integer))
2432         return MatchOperand_ParseFail;
2433       Int = getLexer().getTok().getIntVal();
2434 
2435       if (Prefix == "row_shl") {
2436         Int |= 0x100;
2437       } else if (Prefix == "row_shr") {
2438         Int |= 0x110;
2439       } else if (Prefix == "row_ror") {
2440         Int |= 0x120;
2441       } else if (Prefix == "wave_shl") {
2442         Int = 0x130;
2443       } else if (Prefix == "wave_rol") {
2444         Int = 0x134;
2445       } else if (Prefix == "wave_shr") {
2446         Int = 0x138;
2447       } else if (Prefix == "wave_ror") {
2448         Int = 0x13C;
2449       } else if (Prefix == "row_bcast") {
2450         if (Int == 15) {
2451           Int = 0x142;
2452         } else if (Int == 31) {
2453           Int = 0x143;
2454         }
2455       } else {
2456         return MatchOperand_ParseFail;
2457       }
2458     }
2459   }
2460   Parser.Lex(); // eat last token
2461 
2462   Operands.push_back(AMDGPUOperand::CreateImm(Int, S,
2463                                               AMDGPUOperand::ImmTyDppCtrl));
2464   return MatchOperand_Success;
2465 }
2466 
2467 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultRowMask() const {
2468   return AMDGPUOperand::CreateImm(0xf, SMLoc(), AMDGPUOperand::ImmTyDppRowMask);
2469 }
2470 
2471 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultBankMask() const {
2472   return AMDGPUOperand::CreateImm(0xf, SMLoc(), AMDGPUOperand::ImmTyDppBankMask);
2473 }
2474 
2475 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultBoundCtrl() const {
2476   return AMDGPUOperand::CreateImm(0, SMLoc(), AMDGPUOperand::ImmTyDppBoundCtrl);
2477 }
2478 
2479 void AMDGPUAsmParser::cvtDPP(MCInst &Inst, const OperandVector &Operands) {
2480   OptionalImmIndexMap OptionalIdx;
2481 
2482   unsigned I = 1;
2483   const MCInstrDesc &Desc = MII.get(Inst.getOpcode());
2484   for (unsigned J = 0; J < Desc.getNumDefs(); ++J) {
2485     ((AMDGPUOperand &)*Operands[I++]).addRegOperands(Inst, 1);
2486   }
2487 
2488   for (unsigned E = Operands.size(); I != E; ++I) {
2489     AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[I]);
2490     // Add the register arguments
2491     if (Op.isRegOrImmWithInputMods()) {
2492       // We convert only instructions with modifiers
2493       Op.addRegOrImmWithInputModsOperands(Inst, 2);
2494     } else if (Op.isDPPCtrl()) {
2495       Op.addImmOperands(Inst, 1);
2496     } else if (Op.isImm()) {
2497       // Handle optional arguments
2498       OptionalIdx[Op.getImmTy()] = I;
2499     } else {
2500       llvm_unreachable("Invalid operand type");
2501     }
2502   }
2503 
2504   // ToDo: fix default values for row_mask and bank_mask
2505   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyDppRowMask, 0xf);
2506   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyDppBankMask, 0xf);
2507   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyDppBoundCtrl);
2508 }
2509 
2510 //===----------------------------------------------------------------------===//
2511 // sdwa
2512 //===----------------------------------------------------------------------===//
2513 
2514 AMDGPUAsmParser::OperandMatchResultTy
2515 AMDGPUAsmParser::parseSDWASel(OperandVector &Operands) {
2516   SMLoc S = Parser.getTok().getLoc();
2517   StringRef Value;
2518   AMDGPUAsmParser::OperandMatchResultTy res;
2519 
2520   res = parseStringWithPrefix("dst_sel", Value);
2521   if (res == MatchOperand_ParseFail) {
2522     return MatchOperand_ParseFail;
2523   } else if (res == MatchOperand_NoMatch) {
2524     res = parseStringWithPrefix("src0_sel", Value);
2525     if (res == MatchOperand_ParseFail) {
2526       return MatchOperand_ParseFail;
2527     } else if (res == MatchOperand_NoMatch) {
2528       res = parseStringWithPrefix("src1_sel", Value);
2529       if (res != MatchOperand_Success) {
2530         return res;
2531       }
2532     }
2533   }
2534 
2535   int64_t Int;
2536   Int = StringSwitch<int64_t>(Value)
2537         .Case("BYTE_0", 0)
2538         .Case("BYTE_1", 1)
2539         .Case("BYTE_2", 2)
2540         .Case("BYTE_3", 3)
2541         .Case("WORD_0", 4)
2542         .Case("WORD_1", 5)
2543         .Case("DWORD", 6)
2544         .Default(0xffffffff);
2545   Parser.Lex(); // eat last token
2546 
2547   if (Int == 0xffffffff) {
2548     return MatchOperand_ParseFail;
2549   }
2550 
2551   Operands.push_back(AMDGPUOperand::CreateImm(Int, S,
2552                                               AMDGPUOperand::ImmTySdwaSel));
2553   return MatchOperand_Success;
2554 }
2555 
2556 AMDGPUAsmParser::OperandMatchResultTy
2557 AMDGPUAsmParser::parseSDWADstUnused(OperandVector &Operands) {
2558   SMLoc S = Parser.getTok().getLoc();
2559   StringRef Value;
2560   AMDGPUAsmParser::OperandMatchResultTy res;
2561 
2562   res = parseStringWithPrefix("dst_unused", Value);
2563   if (res != MatchOperand_Success) {
2564     return res;
2565   }
2566 
2567   int64_t Int;
2568   Int = StringSwitch<int64_t>(Value)
2569         .Case("UNUSED_PAD", 0)
2570         .Case("UNUSED_SEXT", 1)
2571         .Case("UNUSED_PRESERVE", 2)
2572         .Default(0xffffffff);
2573   Parser.Lex(); // eat last token
2574 
2575   if (Int == 0xffffffff) {
2576     return MatchOperand_ParseFail;
2577   }
2578 
2579   Operands.push_back(AMDGPUOperand::CreateImm(Int, S,
2580                                               AMDGPUOperand::ImmTySdwaDstUnused));
2581   return MatchOperand_Success;
2582 }
2583 
2584 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultSDWASel() const {
2585   return AMDGPUOperand::CreateImm(6, SMLoc(), AMDGPUOperand::ImmTySdwaSel);
2586 }
2587 
2588 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultSDWADstUnused() const {
2589   return AMDGPUOperand::CreateImm(0, SMLoc(), AMDGPUOperand::ImmTySdwaDstUnused);
2590 }
2591 
2592 
2593 /// Force static initialization.
2594 extern "C" void LLVMInitializeAMDGPUAsmParser() {
2595   RegisterMCAsmParser<AMDGPUAsmParser> A(TheAMDGPUTarget);
2596   RegisterMCAsmParser<AMDGPUAsmParser> B(TheGCNTarget);
2597 }
2598 
2599 #define GET_REGISTER_MATCHER
2600 #define GET_MATCHER_IMPLEMENTATION
2601 #include "AMDGPUGenAsmMatcher.inc"
2602 
2603 
2604 // This fuction should be defined after auto-generated include so that we have
2605 // MatchClassKind enum defined
2606 unsigned AMDGPUAsmParser::validateTargetOperandClass(MCParsedAsmOperand &Op,
2607                                                      unsigned Kind) {
2608   // Tokens like "glc" would be parsed as immediate operands in ParseOperand().
2609   // But MatchInstructionImpl() expects to meet token and fails to validate
2610   // operand. This method checks if we are given immediate operand but expect to
2611   // get corresponding token.
2612   AMDGPUOperand &Operand = (AMDGPUOperand&)Op;
2613   switch (Kind) {
2614   case MCK_addr64:
2615     return Operand.isAddr64() ? Match_Success : Match_InvalidOperand;
2616   case MCK_gds:
2617     return Operand.isGDS() ? Match_Success : Match_InvalidOperand;
2618   case MCK_glc:
2619     return Operand.isGLC() ? Match_Success : Match_InvalidOperand;
2620   case MCK_idxen:
2621     return Operand.isIdxen() ? Match_Success : Match_InvalidOperand;
2622   case MCK_offen:
2623     return Operand.isOffen() ? Match_Success : Match_InvalidOperand;
2624   default: return Match_InvalidOperand;
2625   }
2626 }
2627