1 //===-- AMDGPUAsmParser.cpp - Parse SI asm to MCInst instructions ---------===//
2 //
3 //                     The LLVM Compiler Infrastructure
4 //
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
7 //
8 //===----------------------------------------------------------------------===//
9 
10 #include "AMDKernelCodeT.h"
11 #include "MCTargetDesc/AMDGPUMCTargetDesc.h"
12 #include "MCTargetDesc/AMDGPUTargetStreamer.h"
13 #include "SIDefines.h"
14 #include "Utils/AMDGPUBaseInfo.h"
15 #include "Utils/AMDKernelCodeTUtils.h"
16 #include "Utils/AMDGPUAsmUtils.h"
17 #include "llvm/ADT/APFloat.h"
18 #include "llvm/ADT/STLExtras.h"
19 #include "llvm/ADT/SmallBitVector.h"
20 #include "llvm/ADT/SmallString.h"
21 #include "llvm/ADT/StringSwitch.h"
22 #include "llvm/ADT/Twine.h"
23 #include "llvm/CodeGen/MachineValueType.h"
24 #include "llvm/MC/MCContext.h"
25 #include "llvm/MC/MCExpr.h"
26 #include "llvm/MC/MCInst.h"
27 #include "llvm/MC/MCInstrInfo.h"
28 #include "llvm/MC/MCParser/MCAsmLexer.h"
29 #include "llvm/MC/MCParser/MCAsmParser.h"
30 #include "llvm/MC/MCParser/MCParsedAsmOperand.h"
31 #include "llvm/MC/MCParser/MCTargetAsmParser.h"
32 #include "llvm/MC/MCRegisterInfo.h"
33 #include "llvm/MC/MCStreamer.h"
34 #include "llvm/MC/MCSubtargetInfo.h"
35 #include "llvm/MC/MCSymbolELF.h"
36 #include "llvm/Support/Debug.h"
37 #include "llvm/Support/ELF.h"
38 #include "llvm/Support/SourceMgr.h"
39 #include "llvm/Support/TargetRegistry.h"
40 #include "llvm/Support/raw_ostream.h"
41 #include "llvm/Support/MathExtras.h"
42 
43 using namespace llvm;
44 using namespace llvm::AMDGPU;
45 
46 namespace {
47 
48 class AMDGPUAsmParser;
49 struct OptionalOperand;
50 
51 enum RegisterKind { IS_UNKNOWN, IS_VGPR, IS_SGPR, IS_TTMP, IS_SPECIAL };
52 
53 //===----------------------------------------------------------------------===//
54 // Operand
55 //===----------------------------------------------------------------------===//
56 
57 class AMDGPUOperand : public MCParsedAsmOperand {
58   enum KindTy {
59     Token,
60     Immediate,
61     Register,
62     Expression
63   } Kind;
64 
65   SMLoc StartLoc, EndLoc;
66   const AMDGPUAsmParser *AsmParser;
67 
68 public:
69   AMDGPUOperand(enum KindTy Kind_, const AMDGPUAsmParser *AsmParser_)
70     : MCParsedAsmOperand(), Kind(Kind_), AsmParser(AsmParser_) {}
71 
72   typedef std::unique_ptr<AMDGPUOperand> Ptr;
73 
74   struct Modifiers {
75     bool Abs;
76     bool Neg;
77     bool Sext;
78 
79     bool hasFPModifiers() const { return Abs || Neg; }
80     bool hasIntModifiers() const { return Sext; }
81     bool hasModifiers() const { return hasFPModifiers() || hasIntModifiers(); }
82 
83     int64_t getFPModifiersOperand() const {
84       int64_t Operand = 0;
85       Operand |= Abs ? SISrcMods::ABS : 0;
86       Operand |= Neg ? SISrcMods::NEG : 0;
87       return Operand;
88     }
89 
90     int64_t getIntModifiersOperand() const {
91       int64_t Operand = 0;
92       Operand |= Sext ? SISrcMods::SEXT : 0;
93       return Operand;
94     }
95 
96     int64_t getModifiersOperand() const {
97       assert(!(hasFPModifiers() && hasIntModifiers())
98            && "fp and int modifiers should not be used simultaneously");
99       if (hasFPModifiers()) {
100         return getFPModifiersOperand();
101       } else if (hasIntModifiers()) {
102         return getIntModifiersOperand();
103       } else {
104         return 0;
105       }
106     }
107 
108     friend raw_ostream &operator <<(raw_ostream &OS, AMDGPUOperand::Modifiers Mods);
109   };
110 
111   enum ImmTy {
112     ImmTyNone,
113     ImmTyGDS,
114     ImmTyOffen,
115     ImmTyIdxen,
116     ImmTyAddr64,
117     ImmTyOffset,
118     ImmTyOffset0,
119     ImmTyOffset1,
120     ImmTyGLC,
121     ImmTySLC,
122     ImmTyTFE,
123     ImmTyClampSI,
124     ImmTyOModSI,
125     ImmTyDppCtrl,
126     ImmTyDppRowMask,
127     ImmTyDppBankMask,
128     ImmTyDppBoundCtrl,
129     ImmTySdwaDstSel,
130     ImmTySdwaSrc0Sel,
131     ImmTySdwaSrc1Sel,
132     ImmTySdwaDstUnused,
133     ImmTyDMask,
134     ImmTyUNorm,
135     ImmTyDA,
136     ImmTyR128,
137     ImmTyLWE,
138     ImmTyHwreg,
139     ImmTySendMsg,
140   };
141 
142   struct TokOp {
143     const char *Data;
144     unsigned Length;
145   };
146 
147   struct ImmOp {
148     int64_t Val;
149     ImmTy Type;
150     bool IsFPImm;
151     Modifiers Mods;
152   };
153 
154   struct RegOp {
155     unsigned RegNo;
156     bool IsForcedVOP3;
157     Modifiers Mods;
158   };
159 
160   union {
161     TokOp Tok;
162     ImmOp Imm;
163     RegOp Reg;
164     const MCExpr *Expr;
165   };
166 
167   bool isToken() const override {
168     if (Kind == Token)
169       return true;
170 
171     if (Kind != Expression || !Expr)
172       return false;
173 
174     // When parsing operands, we can't always tell if something was meant to be
175     // a token, like 'gds', or an expression that references a global variable.
176     // In this case, we assume the string is an expression, and if we need to
177     // interpret is a token, then we treat the symbol name as the token.
178     return isa<MCSymbolRefExpr>(Expr);
179   }
180 
181   bool isImm() const override {
182     return Kind == Immediate;
183   }
184 
185   bool isInlinableImm(MVT type) const;
186   bool isLiteralImm(MVT type) const;
187 
188   bool isRegKind() const {
189     return Kind == Register;
190   }
191 
192   bool isReg() const override {
193     return isRegKind() && !Reg.Mods.hasModifiers();
194   }
195 
196   bool isRegOrImmWithInputMods(MVT type) const {
197     return isRegKind() || isInlinableImm(type);
198   }
199 
200   bool isRegOrImmWithInt32InputMods() const {
201     return isRegOrImmWithInputMods(MVT::i32);
202   }
203 
204   bool isRegOrImmWithInt64InputMods() const {
205     return isRegOrImmWithInputMods(MVT::i64);
206   }
207 
208   bool isRegOrImmWithFP32InputMods() const {
209     return isRegOrImmWithInputMods(MVT::f32);
210   }
211 
212   bool isRegOrImmWithFP64InputMods() const {
213     return isRegOrImmWithInputMods(MVT::f64);
214   }
215 
216   bool isImmTy(ImmTy ImmT) const {
217     return isImm() && Imm.Type == ImmT;
218   }
219 
220   bool isImmModifier() const {
221     return isImm() && Imm.Type != ImmTyNone;
222   }
223 
224   bool isClampSI() const { return isImmTy(ImmTyClampSI); }
225   bool isOModSI() const { return isImmTy(ImmTyOModSI); }
226   bool isDMask() const { return isImmTy(ImmTyDMask); }
227   bool isUNorm() const { return isImmTy(ImmTyUNorm); }
228   bool isDA() const { return isImmTy(ImmTyDA); }
229   bool isR128() const { return isImmTy(ImmTyUNorm); }
230   bool isLWE() const { return isImmTy(ImmTyLWE); }
231   bool isOffen() const { return isImmTy(ImmTyOffen); }
232   bool isIdxen() const { return isImmTy(ImmTyIdxen); }
233   bool isAddr64() const { return isImmTy(ImmTyAddr64); }
234   bool isOffset() const { return isImmTy(ImmTyOffset) && isUInt<16>(getImm()); }
235   bool isOffset0() const { return isImmTy(ImmTyOffset0) && isUInt<16>(getImm()); }
236   bool isOffset1() const { return isImmTy(ImmTyOffset1) && isUInt<8>(getImm()); }
237   bool isGDS() const { return isImmTy(ImmTyGDS); }
238   bool isGLC() const { return isImmTy(ImmTyGLC); }
239   bool isSLC() const { return isImmTy(ImmTySLC); }
240   bool isTFE() const { return isImmTy(ImmTyTFE); }
241   bool isBankMask() const { return isImmTy(ImmTyDppBankMask); }
242   bool isRowMask() const { return isImmTy(ImmTyDppRowMask); }
243   bool isBoundCtrl() const { return isImmTy(ImmTyDppBoundCtrl); }
244   bool isSDWADstSel() const { return isImmTy(ImmTySdwaDstSel); }
245   bool isSDWASrc0Sel() const { return isImmTy(ImmTySdwaSrc0Sel); }
246   bool isSDWASrc1Sel() const { return isImmTy(ImmTySdwaSrc1Sel); }
247   bool isSDWADstUnused() const { return isImmTy(ImmTySdwaDstUnused); }
248 
249   bool isMod() const {
250     return isClampSI() || isOModSI();
251   }
252 
253   bool isRegOrImm() const {
254     return isReg() || isImm();
255   }
256 
257   bool isRegClass(unsigned RCID) const;
258 
259   bool isSCSrcB32() const {
260     return isRegClass(AMDGPU::SReg_32RegClassID) || isInlinableImm(MVT::i32);
261   }
262 
263   bool isSCSrcB64() const {
264     return isRegClass(AMDGPU::SReg_64RegClassID) || isInlinableImm(MVT::i64);
265   }
266 
267   bool isSCSrcF32() const {
268     return isRegClass(AMDGPU::SReg_32RegClassID) || isInlinableImm(MVT::f32);
269   }
270 
271   bool isSCSrcF64() const {
272     return isRegClass(AMDGPU::SReg_64RegClassID) || isInlinableImm(MVT::f64);
273   }
274 
275   bool isSSrcB32() const {
276     return isSCSrcB32() || isLiteralImm(MVT::i32) || isExpr();
277   }
278 
279   bool isSSrcB64() const {
280     // TODO: Find out how SALU supports extension of 32-bit literals to 64 bits.
281     // See isVSrc64().
282     return isSCSrcB64() || isLiteralImm(MVT::i64);
283   }
284 
285   bool isSSrcF32() const {
286     return isSCSrcB32() || isLiteralImm(MVT::f32) || isExpr();
287   }
288 
289   bool isSSrcF64() const {
290     return isSCSrcB64() || isLiteralImm(MVT::f64);
291   }
292 
293   bool isVCSrcB32() const {
294     return isRegClass(AMDGPU::VS_32RegClassID) || isInlinableImm(MVT::i32);
295   }
296 
297   bool isVCSrcB64() const {
298     return isRegClass(AMDGPU::VS_64RegClassID) || isInlinableImm(MVT::i64);
299   }
300 
301   bool isVCSrcF32() const {
302     return isRegClass(AMDGPU::VS_32RegClassID) || isInlinableImm(MVT::f32);
303   }
304 
305   bool isVCSrcF64() const {
306     return isRegClass(AMDGPU::VS_64RegClassID) || isInlinableImm(MVT::f64);
307   }
308 
309   bool isVSrcB32() const {
310     return isVCSrcF32() || isLiteralImm(MVT::i32);
311   }
312 
313   bool isVSrcB64() const {
314     return isVCSrcF64() || isLiteralImm(MVT::i64);
315   }
316 
317   bool isVSrcF32() const {
318     return isVCSrcF32() || isLiteralImm(MVT::f32);
319   }
320 
321   bool isVSrcF64() const {
322     return isVCSrcF64() || isLiteralImm(MVT::f64);
323   }
324 
325   bool isKImmFP32() const {
326     return isLiteralImm(MVT::f32);
327   }
328 
329   bool isMem() const override {
330     return false;
331   }
332 
333   bool isExpr() const {
334     return Kind == Expression;
335   }
336 
337   bool isSoppBrTarget() const {
338     return isExpr() || isImm();
339   }
340 
341   bool isSWaitCnt() const;
342   bool isHwreg() const;
343   bool isSendMsg() const;
344   bool isSMRDOffset() const;
345   bool isSMRDLiteralOffset() const;
346   bool isDPPCtrl() const;
347   bool isGPRIdxMode() const;
348 
349   StringRef getExpressionAsToken() const {
350     assert(isExpr());
351     const MCSymbolRefExpr *S = cast<MCSymbolRefExpr>(Expr);
352     return S->getSymbol().getName();
353   }
354 
355 
356   StringRef getToken() const {
357     assert(isToken());
358 
359     if (Kind == Expression)
360       return getExpressionAsToken();
361 
362     return StringRef(Tok.Data, Tok.Length);
363   }
364 
365   int64_t getImm() const {
366     assert(isImm());
367     return Imm.Val;
368   }
369 
370   enum ImmTy getImmTy() const {
371     assert(isImm());
372     return Imm.Type;
373   }
374 
375   unsigned getReg() const override {
376     return Reg.RegNo;
377   }
378 
379   SMLoc getStartLoc() const override {
380     return StartLoc;
381   }
382 
383   SMLoc getEndLoc() const override {
384     return EndLoc;
385   }
386 
387   Modifiers getModifiers() const {
388     assert(isRegKind() || isImmTy(ImmTyNone));
389     return isRegKind() ? Reg.Mods : Imm.Mods;
390   }
391 
392   void setModifiers(Modifiers Mods) {
393     assert(isRegKind() || isImmTy(ImmTyNone));
394     if (isRegKind())
395       Reg.Mods = Mods;
396     else
397       Imm.Mods = Mods;
398   }
399 
400   bool hasModifiers() const {
401     return getModifiers().hasModifiers();
402   }
403 
404   bool hasFPModifiers() const {
405     return getModifiers().hasFPModifiers();
406   }
407 
408   bool hasIntModifiers() const {
409     return getModifiers().hasIntModifiers();
410   }
411 
412   void addImmOperands(MCInst &Inst, unsigned N, bool ApplyModifiers = true) const;
413 
414   void addLiteralImmOperand(MCInst &Inst, int64_t Val) const;
415 
416   void addKImmFP32Operands(MCInst &Inst, unsigned N) const;
417 
418   void addRegOperands(MCInst &Inst, unsigned N) const;
419 
420   void addRegOrImmOperands(MCInst &Inst, unsigned N) const {
421     if (isRegKind())
422       addRegOperands(Inst, N);
423     else if (isExpr())
424       Inst.addOperand(MCOperand::createExpr(Expr));
425     else
426       addImmOperands(Inst, N);
427   }
428 
429   void addRegOrImmWithInputModsOperands(MCInst &Inst, unsigned N) const {
430     Modifiers Mods = getModifiers();
431     Inst.addOperand(MCOperand::createImm(Mods.getModifiersOperand()));
432     if (isRegKind()) {
433       addRegOperands(Inst, N);
434     } else {
435       addImmOperands(Inst, N, false);
436     }
437   }
438 
439   void addRegOrImmWithFPInputModsOperands(MCInst &Inst, unsigned N) const {
440     assert(!hasIntModifiers());
441     addRegOrImmWithInputModsOperands(Inst, N);
442   }
443 
444   void addRegOrImmWithIntInputModsOperands(MCInst &Inst, unsigned N) const {
445     assert(!hasFPModifiers());
446     addRegOrImmWithInputModsOperands(Inst, N);
447   }
448 
449   void addSoppBrTargetOperands(MCInst &Inst, unsigned N) const {
450     if (isImm())
451       addImmOperands(Inst, N);
452     else {
453       assert(isExpr());
454       Inst.addOperand(MCOperand::createExpr(Expr));
455     }
456   }
457 
458   void printImmTy(raw_ostream& OS, ImmTy Type) const {
459     switch (Type) {
460     case ImmTyNone: OS << "None"; break;
461     case ImmTyGDS: OS << "GDS"; break;
462     case ImmTyOffen: OS << "Offen"; break;
463     case ImmTyIdxen: OS << "Idxen"; break;
464     case ImmTyAddr64: OS << "Addr64"; break;
465     case ImmTyOffset: OS << "Offset"; break;
466     case ImmTyOffset0: OS << "Offset0"; break;
467     case ImmTyOffset1: OS << "Offset1"; break;
468     case ImmTyGLC: OS << "GLC"; break;
469     case ImmTySLC: OS << "SLC"; break;
470     case ImmTyTFE: OS << "TFE"; break;
471     case ImmTyClampSI: OS << "ClampSI"; break;
472     case ImmTyOModSI: OS << "OModSI"; break;
473     case ImmTyDppCtrl: OS << "DppCtrl"; break;
474     case ImmTyDppRowMask: OS << "DppRowMask"; break;
475     case ImmTyDppBankMask: OS << "DppBankMask"; break;
476     case ImmTyDppBoundCtrl: OS << "DppBoundCtrl"; break;
477     case ImmTySdwaDstSel: OS << "SdwaDstSel"; break;
478     case ImmTySdwaSrc0Sel: OS << "SdwaSrc0Sel"; break;
479     case ImmTySdwaSrc1Sel: OS << "SdwaSrc1Sel"; break;
480     case ImmTySdwaDstUnused: OS << "SdwaDstUnused"; break;
481     case ImmTyDMask: OS << "DMask"; break;
482     case ImmTyUNorm: OS << "UNorm"; break;
483     case ImmTyDA: OS << "DA"; break;
484     case ImmTyR128: OS << "R128"; break;
485     case ImmTyLWE: OS << "LWE"; break;
486     case ImmTyHwreg: OS << "Hwreg"; break;
487     case ImmTySendMsg: OS << "SendMsg"; break;
488     }
489   }
490 
491   void print(raw_ostream &OS) const override {
492     switch (Kind) {
493     case Register:
494       OS << "<register " << getReg() << " mods: " << Reg.Mods << '>';
495       break;
496     case Immediate:
497       OS << '<' << getImm();
498       if (getImmTy() != ImmTyNone) {
499         OS << " type: "; printImmTy(OS, getImmTy());
500       }
501       OS << " mods: " << Imm.Mods << '>';
502       break;
503     case Token:
504       OS << '\'' << getToken() << '\'';
505       break;
506     case Expression:
507       OS << "<expr " << *Expr << '>';
508       break;
509     }
510   }
511 
512   static AMDGPUOperand::Ptr CreateImm(const AMDGPUAsmParser *AsmParser,
513                                       int64_t Val, SMLoc Loc,
514                                       enum ImmTy Type = ImmTyNone,
515                                       bool IsFPImm = false) {
516     auto Op = llvm::make_unique<AMDGPUOperand>(Immediate, AsmParser);
517     Op->Imm.Val = Val;
518     Op->Imm.IsFPImm = IsFPImm;
519     Op->Imm.Type = Type;
520     Op->Imm.Mods = {false, false, false};
521     Op->StartLoc = Loc;
522     Op->EndLoc = Loc;
523     return Op;
524   }
525 
526   static AMDGPUOperand::Ptr CreateToken(const AMDGPUAsmParser *AsmParser,
527                                         StringRef Str, SMLoc Loc,
528                                         bool HasExplicitEncodingSize = true) {
529     auto Res = llvm::make_unique<AMDGPUOperand>(Token, AsmParser);
530     Res->Tok.Data = Str.data();
531     Res->Tok.Length = Str.size();
532     Res->StartLoc = Loc;
533     Res->EndLoc = Loc;
534     return Res;
535   }
536 
537   static AMDGPUOperand::Ptr CreateReg(const AMDGPUAsmParser *AsmParser,
538                                       unsigned RegNo, SMLoc S,
539                                       SMLoc E,
540                                       bool ForceVOP3) {
541     auto Op = llvm::make_unique<AMDGPUOperand>(Register, AsmParser);
542     Op->Reg.RegNo = RegNo;
543     Op->Reg.Mods = {false, false, false};
544     Op->Reg.IsForcedVOP3 = ForceVOP3;
545     Op->StartLoc = S;
546     Op->EndLoc = E;
547     return Op;
548   }
549 
550   static AMDGPUOperand::Ptr CreateExpr(const AMDGPUAsmParser *AsmParser,
551                                        const class MCExpr *Expr, SMLoc S) {
552     auto Op = llvm::make_unique<AMDGPUOperand>(Expression, AsmParser);
553     Op->Expr = Expr;
554     Op->StartLoc = S;
555     Op->EndLoc = S;
556     return Op;
557   }
558 };
559 
560 raw_ostream &operator <<(raw_ostream &OS, AMDGPUOperand::Modifiers Mods) {
561   OS << "abs:" << Mods.Abs << " neg: " << Mods.Neg << " sext:" << Mods.Sext;
562   return OS;
563 }
564 
565 //===----------------------------------------------------------------------===//
566 // AsmParser
567 //===----------------------------------------------------------------------===//
568 
569 class AMDGPUAsmParser : public MCTargetAsmParser {
570   const MCInstrInfo &MII;
571   MCAsmParser &Parser;
572 
573   unsigned ForcedEncodingSize;
574   bool ForcedDPP;
575   bool ForcedSDWA;
576 
577   /// @name Auto-generated Match Functions
578   /// {
579 
580 #define GET_ASSEMBLER_HEADER
581 #include "AMDGPUGenAsmMatcher.inc"
582 
583   /// }
584 
585 private:
586   bool ParseDirectiveMajorMinor(uint32_t &Major, uint32_t &Minor);
587   bool ParseDirectiveHSACodeObjectVersion();
588   bool ParseDirectiveHSACodeObjectISA();
589   bool ParseAMDKernelCodeTValue(StringRef ID, amd_kernel_code_t &Header);
590   bool ParseDirectiveAMDKernelCodeT();
591   bool ParseSectionDirectiveHSAText();
592   bool subtargetHasRegister(const MCRegisterInfo &MRI, unsigned RegNo) const;
593   bool ParseDirectiveAMDGPUHsaKernel();
594   bool ParseDirectiveAMDGPUHsaModuleGlobal();
595   bool ParseDirectiveAMDGPUHsaProgramGlobal();
596   bool ParseSectionDirectiveHSADataGlobalAgent();
597   bool ParseSectionDirectiveHSADataGlobalProgram();
598   bool ParseSectionDirectiveHSARodataReadonlyAgent();
599   bool AddNextRegisterToList(unsigned& Reg, unsigned& RegWidth, RegisterKind RegKind, unsigned Reg1, unsigned RegNum);
600   bool ParseAMDGPURegister(RegisterKind& RegKind, unsigned& Reg, unsigned& RegNum, unsigned& RegWidth);
601   void cvtMubufImpl(MCInst &Inst, const OperandVector &Operands, bool IsAtomic, bool IsAtomicReturn);
602 
603 public:
604   enum AMDGPUMatchResultTy {
605     Match_PreferE32 = FIRST_TARGET_MATCH_RESULT_TY
606   };
607 
608   AMDGPUAsmParser(const MCSubtargetInfo &STI, MCAsmParser &_Parser,
609                const MCInstrInfo &MII,
610                const MCTargetOptions &Options)
611       : MCTargetAsmParser(Options, STI), MII(MII), Parser(_Parser),
612         ForcedEncodingSize(0),
613         ForcedDPP(false),
614         ForcedSDWA(false) {
615     MCAsmParserExtension::Initialize(Parser);
616 
617     if (getSTI().getFeatureBits().none()) {
618       // Set default features.
619       copySTI().ToggleFeature("SOUTHERN_ISLANDS");
620     }
621 
622     setAvailableFeatures(ComputeAvailableFeatures(getSTI().getFeatureBits()));
623 
624     {
625       // TODO: make those pre-defined variables read-only.
626       // Currently there is none suitable machinery in the core llvm-mc for this.
627       // MCSymbol::isRedefinable is intended for another purpose, and
628       // AsmParser::parseDirectiveSet() cannot be specialized for specific target.
629       AMDGPU::IsaVersion Isa = AMDGPU::getIsaVersion(getSTI().getFeatureBits());
630       MCContext &Ctx = getContext();
631       MCSymbol *Sym = Ctx.getOrCreateSymbol(Twine(".option.machine_version_major"));
632       Sym->setVariableValue(MCConstantExpr::create(Isa.Major, Ctx));
633       Sym = Ctx.getOrCreateSymbol(Twine(".option.machine_version_minor"));
634       Sym->setVariableValue(MCConstantExpr::create(Isa.Minor, Ctx));
635       Sym = Ctx.getOrCreateSymbol(Twine(".option.machine_version_stepping"));
636       Sym->setVariableValue(MCConstantExpr::create(Isa.Stepping, Ctx));
637     }
638   }
639 
640   bool isSI() const {
641     return AMDGPU::isSI(getSTI());
642   }
643 
644   bool isCI() const {
645     return AMDGPU::isCI(getSTI());
646   }
647 
648   bool isVI() const {
649     return AMDGPU::isVI(getSTI());
650   }
651 
652   bool hasSGPR102_SGPR103() const {
653     return !isVI();
654   }
655 
656   AMDGPUTargetStreamer &getTargetStreamer() {
657     MCTargetStreamer &TS = *getParser().getStreamer().getTargetStreamer();
658     return static_cast<AMDGPUTargetStreamer &>(TS);
659   }
660 
661   const MCRegisterInfo *getMRI() const {
662     // We need this const_cast because for some reason getContext() is not const
663     // in MCAsmParser.
664     return const_cast<AMDGPUAsmParser*>(this)->getContext().getRegisterInfo();
665   }
666 
667   const MCInstrInfo *getMII() const {
668     return &MII;
669   }
670 
671   void setForcedEncodingSize(unsigned Size) { ForcedEncodingSize = Size; }
672   void setForcedDPP(bool ForceDPP_) { ForcedDPP = ForceDPP_; }
673   void setForcedSDWA(bool ForceSDWA_) { ForcedSDWA = ForceSDWA_; }
674 
675   unsigned getForcedEncodingSize() const { return ForcedEncodingSize; }
676   bool isForcedVOP3() const { return ForcedEncodingSize == 64; }
677   bool isForcedDPP() const { return ForcedDPP; }
678   bool isForcedSDWA() const { return ForcedSDWA; }
679 
680   std::unique_ptr<AMDGPUOperand> parseRegister();
681   bool ParseRegister(unsigned &RegNo, SMLoc &StartLoc, SMLoc &EndLoc) override;
682   unsigned checkTargetMatchPredicate(MCInst &Inst) override;
683   unsigned validateTargetOperandClass(MCParsedAsmOperand &Op,
684                                       unsigned Kind) override;
685   bool MatchAndEmitInstruction(SMLoc IDLoc, unsigned &Opcode,
686                                OperandVector &Operands, MCStreamer &Out,
687                                uint64_t &ErrorInfo,
688                                bool MatchingInlineAsm) override;
689   bool ParseDirective(AsmToken DirectiveID) override;
690   OperandMatchResultTy parseOperand(OperandVector &Operands, StringRef Mnemonic);
691   StringRef parseMnemonicSuffix(StringRef Name);
692   bool ParseInstruction(ParseInstructionInfo &Info, StringRef Name,
693                         SMLoc NameLoc, OperandVector &Operands) override;
694   //bool ProcessInstruction(MCInst &Inst);
695 
696   OperandMatchResultTy parseIntWithPrefix(const char *Prefix, int64_t &Int);
697   OperandMatchResultTy parseIntWithPrefix(const char *Prefix,
698                                           OperandVector &Operands,
699                                           enum AMDGPUOperand::ImmTy ImmTy = AMDGPUOperand::ImmTyNone,
700                                           bool (*ConvertResult)(int64_t&) = 0);
701   OperandMatchResultTy parseNamedBit(const char *Name, OperandVector &Operands,
702                                      enum AMDGPUOperand::ImmTy ImmTy = AMDGPUOperand::ImmTyNone);
703   OperandMatchResultTy parseStringWithPrefix(StringRef Prefix, StringRef &Value);
704 
705   OperandMatchResultTy parseImm(OperandVector &Operands);
706   OperandMatchResultTy parseRegOrImm(OperandVector &Operands);
707   OperandMatchResultTy parseRegOrImmWithFPInputMods(OperandVector &Operands);
708   OperandMatchResultTy parseRegOrImmWithIntInputMods(OperandVector &Operands);
709 
710   void cvtDSOffset01(MCInst &Inst, const OperandVector &Operands);
711   void cvtDS(MCInst &Inst, const OperandVector &Operands);
712 
713   bool parseCnt(int64_t &IntVal);
714   OperandMatchResultTy parseSWaitCntOps(OperandVector &Operands);
715   OperandMatchResultTy parseHwreg(OperandVector &Operands);
716 
717 private:
718   struct OperandInfoTy {
719     int64_t Id;
720     bool IsSymbolic;
721     OperandInfoTy(int64_t Id_) : Id(Id_), IsSymbolic(false) { }
722   };
723 
724   bool parseSendMsgConstruct(OperandInfoTy &Msg, OperandInfoTy &Operation, int64_t &StreamId);
725   bool parseHwregConstruct(OperandInfoTy &HwReg, int64_t &Offset, int64_t &Width);
726 public:
727   OperandMatchResultTy parseOptionalOperand(OperandVector &Operands);
728 
729   OperandMatchResultTy parseSendMsgOp(OperandVector &Operands);
730   OperandMatchResultTy parseSOppBrTarget(OperandVector &Operands);
731 
732   void cvtMubuf(MCInst &Inst, const OperandVector &Operands) { cvtMubufImpl(Inst, Operands, false, false); }
733   void cvtMubufAtomic(MCInst &Inst, const OperandVector &Operands) { cvtMubufImpl(Inst, Operands, true, false); }
734   void cvtMubufAtomicReturn(MCInst &Inst, const OperandVector &Operands) { cvtMubufImpl(Inst, Operands, true, true); }
735   AMDGPUOperand::Ptr defaultGLC() const;
736   AMDGPUOperand::Ptr defaultSLC() const;
737   AMDGPUOperand::Ptr defaultTFE() const;
738 
739   AMDGPUOperand::Ptr defaultDMask() const;
740   AMDGPUOperand::Ptr defaultUNorm() const;
741   AMDGPUOperand::Ptr defaultDA() const;
742   AMDGPUOperand::Ptr defaultR128() const;
743   AMDGPUOperand::Ptr defaultLWE() const;
744   AMDGPUOperand::Ptr defaultSMRDOffset() const;
745   AMDGPUOperand::Ptr defaultSMRDLiteralOffset() const;
746 
747   OperandMatchResultTy parseOModOperand(OperandVector &Operands);
748 
749   void cvtId(MCInst &Inst, const OperandVector &Operands);
750   void cvtVOP3_2_mod(MCInst &Inst, const OperandVector &Operands);
751   void cvtVOP3(MCInst &Inst, const OperandVector &Operands);
752 
753   void cvtMIMG(MCInst &Inst, const OperandVector &Operands);
754   void cvtMIMGAtomic(MCInst &Inst, const OperandVector &Operands);
755 
756   OperandMatchResultTy parseDPPCtrl(OperandVector &Operands);
757   AMDGPUOperand::Ptr defaultRowMask() const;
758   AMDGPUOperand::Ptr defaultBankMask() const;
759   AMDGPUOperand::Ptr defaultBoundCtrl() const;
760   void cvtDPP(MCInst &Inst, const OperandVector &Operands);
761 
762   OperandMatchResultTy parseSDWASel(OperandVector &Operands, StringRef Prefix,
763                                     AMDGPUOperand::ImmTy Type);
764   OperandMatchResultTy parseSDWADstUnused(OperandVector &Operands);
765   void cvtSdwaVOP1(MCInst &Inst, const OperandVector &Operands);
766   void cvtSdwaVOP2(MCInst &Inst, const OperandVector &Operands);
767   void cvtSdwaVOPC(MCInst &Inst, const OperandVector &Operands);
768   void cvtSDWA(MCInst &Inst, const OperandVector &Operands,
769                uint64_t BasicInstType);
770 };
771 
772 struct OptionalOperand {
773   const char *Name;
774   AMDGPUOperand::ImmTy Type;
775   bool IsBit;
776   bool (*ConvertResult)(int64_t&);
777 };
778 
779 }
780 
781 //===----------------------------------------------------------------------===//
782 // Operand
783 //===----------------------------------------------------------------------===//
784 
785 bool AMDGPUOperand::isInlinableImm(MVT type) const {
786   if (!isImmTy(ImmTyNone)) {
787     // Only plain immediates are inlinable (e.g. "clamp" attribute is not)
788     return false;
789   }
790   // TODO: We should avoid using host float here. It would be better to
791   // check the float bit values which is what a few other places do.
792   // We've had bot failures before due to weird NaN support on mips hosts.
793 
794   APInt Literal(64, Imm.Val);
795 
796   if (Imm.IsFPImm) { // We got fp literal token
797     if (type == MVT::f64 || type == MVT::i64) { // Expected 64-bit operand
798       return AMDGPU::isInlinableLiteral64(Imm.Val, AsmParser->isVI());
799     } else { // Expected 32-bit operand
800       bool lost;
801       APFloat FPLiteral(APFloat::IEEEdouble, Literal);
802       // Convert literal to single precision
803       APFloat::opStatus status = FPLiteral.convert(APFloat::IEEEsingle,
804                                                     APFloat::rmNearestTiesToEven,
805                                                     &lost);
806       // We allow precision lost but not overflow or underflow
807       if (status != APFloat::opOK &&
808           lost &&
809           ((status & APFloat::opOverflow)  != 0 ||
810             (status & APFloat::opUnderflow) != 0)) {
811         return false;
812       }
813       // Check if single precision literal is inlinable
814       return AMDGPU::isInlinableLiteral32(
815               static_cast<int32_t>(FPLiteral.bitcastToAPInt().getZExtValue()),
816               AsmParser->isVI());
817     }
818   } else { // We got int literal token
819     if (type == MVT::f64 || type == MVT::i64) { // Expected 64-bit operand
820       return AMDGPU::isInlinableLiteral64(Imm.Val, AsmParser->isVI());
821     } else { // Expected 32-bit operand
822       return AMDGPU::isInlinableLiteral32(
823             static_cast<int32_t>(Literal.getLoBits(32).getZExtValue()),
824             AsmParser->isVI());
825     }
826   }
827   return false;
828 }
829 
830 bool AMDGPUOperand::isLiteralImm(MVT type) const {
831   // Check that this imediate can be added as literal
832   if (!isImmTy(ImmTyNone)) {
833     return false;
834   }
835 
836   APInt Literal(64, Imm.Val);
837 
838   if (Imm.IsFPImm) { // We got fp literal token
839     if (type == MVT::f64) { // Expected 64-bit fp operand
840       // We would set low 64-bits of literal to zeroes but we accept this literals
841       return true;
842     } else if (type == MVT::i64) { // Expected 64-bit int operand
843         // We don't allow fp literals in 64-bit integer instructions. It is
844         // unclear how we should encode them.
845       return false;
846     } else { // Expected 32-bit operand
847       bool lost;
848       APFloat FPLiteral(APFloat::IEEEdouble, Literal);
849       // Convert literal to single precision
850       APFloat::opStatus status = FPLiteral.convert(APFloat::IEEEsingle,
851                                                     APFloat::rmNearestTiesToEven,
852                                                     &lost);
853       // We allow precision lost but not overflow or underflow
854       if (status != APFloat::opOK &&
855           lost &&
856           ((status & APFloat::opOverflow)  != 0 ||
857             (status & APFloat::opUnderflow) != 0)) {
858         return false;
859       }
860       return true;
861     }
862   } else { // We got int literal token
863     APInt HiBits = Literal.getHiBits(32);
864     if (HiBits == 0xffffffff &&
865         (*Literal.getLoBits(32).getRawData() & 0x80000000) != 0) {
866       // If high 32 bits aren't zeroes then they all should be ones and 32nd
867       // bit should be set. So that this 64-bit literal is sign-extension of
868       // 32-bit value.
869       return true;
870     } else if (HiBits == 0) {
871       return true;
872     }
873   }
874   return false;
875 }
876 
877 bool AMDGPUOperand::isRegClass(unsigned RCID) const {
878   return isReg() && AsmParser->getMRI()->getRegClass(RCID).contains(getReg());
879 }
880 
881 void AMDGPUOperand::addImmOperands(MCInst &Inst, unsigned N, bool ApplyModifiers) const {
882   int64_t Val = Imm.Val;
883   if (isImmTy(ImmTyNone) && ApplyModifiers && Imm.Mods.hasFPModifiers() && Imm.Mods.Neg) {
884     // Apply modifiers to immediate value. Only negate can get here
885     if (Imm.IsFPImm) {
886       APFloat F(BitsToDouble(Val));
887       F.changeSign();
888       Val = F.bitcastToAPInt().getZExtValue();
889     } else {
890       Val = -Val;
891     }
892   }
893 
894   if (AMDGPU::isSISrcOperand(AsmParser->getMII()->get(Inst.getOpcode()), Inst.getNumOperands())) {
895     addLiteralImmOperand(Inst, Val);
896   } else {
897     Inst.addOperand(MCOperand::createImm(Val));
898   }
899 }
900 
901 void AMDGPUOperand::addLiteralImmOperand(MCInst &Inst, int64_t Val) const {
902   const auto& InstDesc = AsmParser->getMII()->get(Inst.getOpcode());
903   auto OpNum = Inst.getNumOperands();
904   // Check that this operand accepts literals
905   assert(AMDGPU::isSISrcOperand(InstDesc, OpNum));
906 
907   APInt Literal(64, Val);
908   auto OpSize = AMDGPU::getRegOperandSize(AsmParser->getMRI(), InstDesc, OpNum); // expected operand size
909 
910   if (Imm.IsFPImm) { // We got fp literal token
911     if (OpSize == 8) { // Expected 64-bit operand
912       // Check if literal is inlinable
913       if (AMDGPU::isInlinableLiteral64(Literal.getZExtValue(), AsmParser->isVI())) {
914         Inst.addOperand(MCOperand::createImm(Literal.getZExtValue()));
915       } else if (AMDGPU::isSISrcFPOperand(InstDesc, OpNum)) { // Expected 64-bit fp operand
916         // For fp operands we check if low 32 bits are zeros
917         if (Literal.getLoBits(32) != 0) {
918           const_cast<AMDGPUAsmParser *>(AsmParser)->Warning(Inst.getLoc(),
919                             "Can't encode literal as exact 64-bit"
920                             " floating-point operand. Low 32-bits will be"
921                             " set to zero");
922         }
923         Inst.addOperand(MCOperand::createImm(Literal.lshr(32).getZExtValue()));
924       } else {
925         // We don't allow fp literals in 64-bit integer instructions. It is
926         // unclear how we should encode them. This case should be checked earlier
927         // in predicate methods (isLiteralImm())
928         llvm_unreachable("fp literal in 64-bit integer instruction.");
929       }
930     } else { // Expected 32-bit operand
931       bool lost;
932       APFloat FPLiteral(APFloat::IEEEdouble, Literal);
933       // Convert literal to single precision
934       FPLiteral.convert(APFloat::IEEEsingle, APFloat::rmNearestTiesToEven, &lost);
935       // We allow precision lost but not overflow or underflow. This should be
936       // checked earlier in isLiteralImm()
937       Inst.addOperand(MCOperand::createImm(FPLiteral.bitcastToAPInt().getZExtValue()));
938     }
939   } else { // We got int literal token
940     if (OpSize == 8) { // Expected 64-bit operand
941       auto LiteralVal = Literal.getZExtValue();
942       if (AMDGPU::isInlinableLiteral64(LiteralVal, AsmParser->isVI())) {
943         Inst.addOperand(MCOperand::createImm(LiteralVal));
944         return;
945       }
946     } else { // Expected 32-bit operand
947       auto LiteralVal = static_cast<int32_t>(Literal.getLoBits(32).getZExtValue());
948       if (AMDGPU::isInlinableLiteral32(LiteralVal, AsmParser->isVI())) {
949         Inst.addOperand(MCOperand::createImm(LiteralVal));
950         return;
951       }
952     }
953     Inst.addOperand(MCOperand::createImm(Literal.getLoBits(32).getZExtValue()));
954   }
955 }
956 
957 void AMDGPUOperand::addKImmFP32Operands(MCInst &Inst, unsigned N) const {
958   APInt Literal(64, Imm.Val);
959   if (Imm.IsFPImm) { // We got fp literal
960     bool lost;
961     APFloat FPLiteral(APFloat::IEEEdouble, Literal);
962     FPLiteral.convert(APFloat::IEEEsingle, APFloat::rmNearestTiesToEven, &lost);
963     Inst.addOperand(MCOperand::createImm(FPLiteral.bitcastToAPInt().getZExtValue()));
964   } else { // We got int literal token
965     Inst.addOperand(MCOperand::createImm(Literal.getLoBits(32).getZExtValue()));
966   }
967 }
968 
969 void AMDGPUOperand::addRegOperands(MCInst &Inst, unsigned N) const {
970   Inst.addOperand(MCOperand::createReg(AMDGPU::getMCReg(getReg(), AsmParser->getSTI())));
971 }
972 
973 //===----------------------------------------------------------------------===//
974 // AsmParser
975 //===----------------------------------------------------------------------===//
976 
977 static int getRegClass(RegisterKind Is, unsigned RegWidth) {
978   if (Is == IS_VGPR) {
979     switch (RegWidth) {
980       default: return -1;
981       case 1: return AMDGPU::VGPR_32RegClassID;
982       case 2: return AMDGPU::VReg_64RegClassID;
983       case 3: return AMDGPU::VReg_96RegClassID;
984       case 4: return AMDGPU::VReg_128RegClassID;
985       case 8: return AMDGPU::VReg_256RegClassID;
986       case 16: return AMDGPU::VReg_512RegClassID;
987     }
988   } else if (Is == IS_TTMP) {
989     switch (RegWidth) {
990       default: return -1;
991       case 1: return AMDGPU::TTMP_32RegClassID;
992       case 2: return AMDGPU::TTMP_64RegClassID;
993       case 4: return AMDGPU::TTMP_128RegClassID;
994     }
995   } else if (Is == IS_SGPR) {
996     switch (RegWidth) {
997       default: return -1;
998       case 1: return AMDGPU::SGPR_32RegClassID;
999       case 2: return AMDGPU::SGPR_64RegClassID;
1000       case 4: return AMDGPU::SGPR_128RegClassID;
1001       case 8: return AMDGPU::SReg_256RegClassID;
1002       case 16: return AMDGPU::SReg_512RegClassID;
1003     }
1004   }
1005   return -1;
1006 }
1007 
1008 static unsigned getSpecialRegForName(StringRef RegName) {
1009   return StringSwitch<unsigned>(RegName)
1010     .Case("exec", AMDGPU::EXEC)
1011     .Case("vcc", AMDGPU::VCC)
1012     .Case("flat_scratch", AMDGPU::FLAT_SCR)
1013     .Case("m0", AMDGPU::M0)
1014     .Case("scc", AMDGPU::SCC)
1015     .Case("tba", AMDGPU::TBA)
1016     .Case("tma", AMDGPU::TMA)
1017     .Case("flat_scratch_lo", AMDGPU::FLAT_SCR_LO)
1018     .Case("flat_scratch_hi", AMDGPU::FLAT_SCR_HI)
1019     .Case("vcc_lo", AMDGPU::VCC_LO)
1020     .Case("vcc_hi", AMDGPU::VCC_HI)
1021     .Case("exec_lo", AMDGPU::EXEC_LO)
1022     .Case("exec_hi", AMDGPU::EXEC_HI)
1023     .Case("tma_lo", AMDGPU::TMA_LO)
1024     .Case("tma_hi", AMDGPU::TMA_HI)
1025     .Case("tba_lo", AMDGPU::TBA_LO)
1026     .Case("tba_hi", AMDGPU::TBA_HI)
1027     .Default(0);
1028 }
1029 
1030 bool AMDGPUAsmParser::ParseRegister(unsigned &RegNo, SMLoc &StartLoc, SMLoc &EndLoc) {
1031   auto R = parseRegister();
1032   if (!R) return true;
1033   assert(R->isReg());
1034   RegNo = R->getReg();
1035   StartLoc = R->getStartLoc();
1036   EndLoc = R->getEndLoc();
1037   return false;
1038 }
1039 
1040 bool AMDGPUAsmParser::AddNextRegisterToList(unsigned& Reg, unsigned& RegWidth, RegisterKind RegKind, unsigned Reg1, unsigned RegNum)
1041 {
1042   switch (RegKind) {
1043   case IS_SPECIAL:
1044     if (Reg == AMDGPU::EXEC_LO && Reg1 == AMDGPU::EXEC_HI) { Reg = AMDGPU::EXEC; RegWidth = 2; return true; }
1045     if (Reg == AMDGPU::FLAT_SCR_LO && Reg1 == AMDGPU::FLAT_SCR_HI) { Reg = AMDGPU::FLAT_SCR; RegWidth = 2; return true; }
1046     if (Reg == AMDGPU::VCC_LO && Reg1 == AMDGPU::VCC_HI) { Reg = AMDGPU::VCC; RegWidth = 2; return true; }
1047     if (Reg == AMDGPU::TBA_LO && Reg1 == AMDGPU::TBA_HI) { Reg = AMDGPU::TBA; RegWidth = 2; return true; }
1048     if (Reg == AMDGPU::TMA_LO && Reg1 == AMDGPU::TMA_HI) { Reg = AMDGPU::TMA; RegWidth = 2; return true; }
1049     return false;
1050   case IS_VGPR:
1051   case IS_SGPR:
1052   case IS_TTMP:
1053     if (Reg1 != Reg + RegWidth) { return false; }
1054     RegWidth++;
1055     return true;
1056   default:
1057     assert(false); return false;
1058   }
1059 }
1060 
1061 bool AMDGPUAsmParser::ParseAMDGPURegister(RegisterKind& RegKind, unsigned& Reg, unsigned& RegNum, unsigned& RegWidth)
1062 {
1063   const MCRegisterInfo *TRI = getContext().getRegisterInfo();
1064   if (getLexer().is(AsmToken::Identifier)) {
1065     StringRef RegName = Parser.getTok().getString();
1066     if ((Reg = getSpecialRegForName(RegName))) {
1067       Parser.Lex();
1068       RegKind = IS_SPECIAL;
1069     } else {
1070       unsigned RegNumIndex = 0;
1071       if (RegName[0] == 'v') {
1072         RegNumIndex = 1;
1073         RegKind = IS_VGPR;
1074       } else if (RegName[0] == 's') {
1075         RegNumIndex = 1;
1076         RegKind = IS_SGPR;
1077       } else if (RegName.startswith("ttmp")) {
1078         RegNumIndex = strlen("ttmp");
1079         RegKind = IS_TTMP;
1080       } else {
1081         return false;
1082       }
1083       if (RegName.size() > RegNumIndex) {
1084         // Single 32-bit register: vXX.
1085         if (RegName.substr(RegNumIndex).getAsInteger(10, RegNum))
1086           return false;
1087         Parser.Lex();
1088         RegWidth = 1;
1089       } else {
1090         // Range of registers: v[XX:YY]. ":YY" is optional.
1091         Parser.Lex();
1092         int64_t RegLo, RegHi;
1093         if (getLexer().isNot(AsmToken::LBrac))
1094           return false;
1095         Parser.Lex();
1096 
1097         if (getParser().parseAbsoluteExpression(RegLo))
1098           return false;
1099 
1100         const bool isRBrace = getLexer().is(AsmToken::RBrac);
1101         if (!isRBrace && getLexer().isNot(AsmToken::Colon))
1102           return false;
1103         Parser.Lex();
1104 
1105         if (isRBrace) {
1106           RegHi = RegLo;
1107         } else {
1108           if (getParser().parseAbsoluteExpression(RegHi))
1109             return false;
1110 
1111           if (getLexer().isNot(AsmToken::RBrac))
1112             return false;
1113           Parser.Lex();
1114         }
1115         RegNum = (unsigned) RegLo;
1116         RegWidth = (RegHi - RegLo) + 1;
1117       }
1118     }
1119   } else if (getLexer().is(AsmToken::LBrac)) {
1120     // List of consecutive registers: [s0,s1,s2,s3]
1121     Parser.Lex();
1122     if (!ParseAMDGPURegister(RegKind, Reg, RegNum, RegWidth))
1123       return false;
1124     if (RegWidth != 1)
1125       return false;
1126     RegisterKind RegKind1;
1127     unsigned Reg1, RegNum1, RegWidth1;
1128     do {
1129       if (getLexer().is(AsmToken::Comma)) {
1130         Parser.Lex();
1131       } else if (getLexer().is(AsmToken::RBrac)) {
1132         Parser.Lex();
1133         break;
1134       } else if (ParseAMDGPURegister(RegKind1, Reg1, RegNum1, RegWidth1)) {
1135         if (RegWidth1 != 1) {
1136           return false;
1137         }
1138         if (RegKind1 != RegKind) {
1139           return false;
1140         }
1141         if (!AddNextRegisterToList(Reg, RegWidth, RegKind1, Reg1, RegNum1)) {
1142           return false;
1143         }
1144       } else {
1145         return false;
1146       }
1147     } while (true);
1148   } else {
1149     return false;
1150   }
1151   switch (RegKind) {
1152   case IS_SPECIAL:
1153     RegNum = 0;
1154     RegWidth = 1;
1155     break;
1156   case IS_VGPR:
1157   case IS_SGPR:
1158   case IS_TTMP:
1159   {
1160     unsigned Size = 1;
1161     if (RegKind == IS_SGPR || RegKind == IS_TTMP) {
1162       // SGPR and TTMP registers must be are aligned. Max required alignment is 4 dwords.
1163       Size = std::min(RegWidth, 4u);
1164     }
1165     if (RegNum % Size != 0)
1166       return false;
1167     RegNum = RegNum / Size;
1168     int RCID = getRegClass(RegKind, RegWidth);
1169     if (RCID == -1)
1170       return false;
1171     const MCRegisterClass RC = TRI->getRegClass(RCID);
1172     if (RegNum >= RC.getNumRegs())
1173       return false;
1174     Reg = RC.getRegister(RegNum);
1175     break;
1176   }
1177 
1178   default:
1179     assert(false); return false;
1180   }
1181 
1182   if (!subtargetHasRegister(*TRI, Reg))
1183     return false;
1184   return true;
1185 }
1186 
1187 std::unique_ptr<AMDGPUOperand> AMDGPUAsmParser::parseRegister() {
1188   const auto &Tok = Parser.getTok();
1189   SMLoc StartLoc = Tok.getLoc();
1190   SMLoc EndLoc = Tok.getEndLoc();
1191   RegisterKind RegKind;
1192   unsigned Reg, RegNum, RegWidth;
1193 
1194   if (!ParseAMDGPURegister(RegKind, Reg, RegNum, RegWidth)) {
1195     return nullptr;
1196   }
1197   return AMDGPUOperand::CreateReg(this, Reg, StartLoc, EndLoc, false);
1198 }
1199 
1200 AMDGPUAsmParser::OperandMatchResultTy
1201 AMDGPUAsmParser::parseImm(OperandVector &Operands) {
1202   // TODO: add syntactic sugar for 1/(2*PI)
1203   bool Minus = false;
1204   if (getLexer().getKind() == AsmToken::Minus) {
1205     Minus = true;
1206     Parser.Lex();
1207   }
1208 
1209   SMLoc S = Parser.getTok().getLoc();
1210   switch(getLexer().getKind()) {
1211   case AsmToken::Integer: {
1212     int64_t IntVal;
1213     if (getParser().parseAbsoluteExpression(IntVal))
1214       return MatchOperand_ParseFail;
1215     if (Minus)
1216       IntVal *= -1;
1217     Operands.push_back(AMDGPUOperand::CreateImm(this, IntVal, S));
1218     return MatchOperand_Success;
1219   }
1220   case AsmToken::Real: {
1221     int64_t IntVal;
1222     if (getParser().parseAbsoluteExpression(IntVal))
1223       return MatchOperand_ParseFail;
1224 
1225     APFloat F(BitsToDouble(IntVal));
1226     if (Minus)
1227       F.changeSign();
1228     Operands.push_back(
1229         AMDGPUOperand::CreateImm(this, F.bitcastToAPInt().getZExtValue(), S,
1230                                  AMDGPUOperand::ImmTyNone, true));
1231     return MatchOperand_Success;
1232   }
1233   default:
1234     return Minus ? MatchOperand_ParseFail : MatchOperand_NoMatch;
1235   }
1236 }
1237 
1238 AMDGPUAsmParser::OperandMatchResultTy
1239 AMDGPUAsmParser::parseRegOrImm(OperandVector &Operands) {
1240   auto res = parseImm(Operands);
1241   if (res != MatchOperand_NoMatch) {
1242     return res;
1243   }
1244 
1245   if (auto R = parseRegister()) {
1246     assert(R->isReg());
1247     R->Reg.IsForcedVOP3 = isForcedVOP3();
1248     Operands.push_back(std::move(R));
1249     return MatchOperand_Success;
1250   }
1251   return MatchOperand_ParseFail;
1252 }
1253 
1254 AMDGPUAsmParser::OperandMatchResultTy
1255 AMDGPUAsmParser::parseRegOrImmWithFPInputMods(OperandVector &Operands) {
1256   // XXX: During parsing we can't determine if minus sign means
1257   // negate-modifier or negative immediate value.
1258   // By default we suppose it is modifier.
1259   bool Negate = false, Abs = false, Abs2 = false;
1260 
1261   if (getLexer().getKind()== AsmToken::Minus) {
1262     Parser.Lex();
1263     Negate = true;
1264   }
1265 
1266   if (getLexer().getKind() == AsmToken::Identifier && Parser.getTok().getString() == "abs") {
1267     Parser.Lex();
1268     Abs2 = true;
1269     if (getLexer().isNot(AsmToken::LParen)) {
1270       Error(Parser.getTok().getLoc(), "expected left paren after abs");
1271       return MatchOperand_ParseFail;
1272     }
1273     Parser.Lex();
1274   }
1275 
1276   if (getLexer().getKind() == AsmToken::Pipe) {
1277     if (Abs2) {
1278       Error(Parser.getTok().getLoc(), "expected register or immediate");
1279       return MatchOperand_ParseFail;
1280     }
1281     Parser.Lex();
1282     Abs = true;
1283   }
1284 
1285   auto Res = parseRegOrImm(Operands);
1286   if (Res != MatchOperand_Success) {
1287     return Res;
1288   }
1289 
1290   AMDGPUOperand::Modifiers Mods = {false, false, false};
1291   if (Negate) {
1292     Mods.Neg = true;
1293   }
1294   if (Abs) {
1295     if (getLexer().getKind() != AsmToken::Pipe) {
1296       Error(Parser.getTok().getLoc(), "expected vertical bar");
1297       return MatchOperand_ParseFail;
1298     }
1299     Parser.Lex();
1300     Mods.Abs = true;
1301   }
1302   if (Abs2) {
1303     if (getLexer().isNot(AsmToken::RParen)) {
1304       Error(Parser.getTok().getLoc(), "expected closing parentheses");
1305       return MatchOperand_ParseFail;
1306     }
1307     Parser.Lex();
1308     Mods.Abs = true;
1309   }
1310 
1311   if (Mods.hasFPModifiers()) {
1312     AMDGPUOperand &Op = static_cast<AMDGPUOperand &>(*Operands.back());
1313     Op.setModifiers(Mods);
1314   }
1315   return MatchOperand_Success;
1316 }
1317 
1318 AMDGPUAsmParser::OperandMatchResultTy
1319 AMDGPUAsmParser::parseRegOrImmWithIntInputMods(OperandVector &Operands) {
1320   bool Sext = false;
1321 
1322   if (getLexer().getKind() == AsmToken::Identifier && Parser.getTok().getString() == "sext") {
1323     Parser.Lex();
1324     Sext = true;
1325     if (getLexer().isNot(AsmToken::LParen)) {
1326       Error(Parser.getTok().getLoc(), "expected left paren after sext");
1327       return MatchOperand_ParseFail;
1328     }
1329     Parser.Lex();
1330   }
1331 
1332   auto Res = parseRegOrImm(Operands);
1333   if (Res != MatchOperand_Success) {
1334     return Res;
1335   }
1336 
1337   AMDGPUOperand::Modifiers Mods = {false, false, false};
1338   if (Sext) {
1339     if (getLexer().isNot(AsmToken::RParen)) {
1340       Error(Parser.getTok().getLoc(), "expected closing parentheses");
1341       return MatchOperand_ParseFail;
1342     }
1343     Parser.Lex();
1344     Mods.Sext = true;
1345   }
1346 
1347   if (Mods.hasIntModifiers()) {
1348     AMDGPUOperand &Op = static_cast<AMDGPUOperand &>(*Operands.back());
1349     Op.setModifiers(Mods);
1350   }
1351   return MatchOperand_Success;
1352 }
1353 
1354 unsigned AMDGPUAsmParser::checkTargetMatchPredicate(MCInst &Inst) {
1355 
1356   uint64_t TSFlags = MII.get(Inst.getOpcode()).TSFlags;
1357 
1358   if ((getForcedEncodingSize() == 32 && (TSFlags & SIInstrFlags::VOP3)) ||
1359       (getForcedEncodingSize() == 64 && !(TSFlags & SIInstrFlags::VOP3)) ||
1360       (isForcedDPP() && !(TSFlags & SIInstrFlags::DPP)) ||
1361       (isForcedSDWA() && !(TSFlags & SIInstrFlags::SDWA)) )
1362     return Match_InvalidOperand;
1363 
1364   if ((TSFlags & SIInstrFlags::VOP3) &&
1365       (TSFlags & SIInstrFlags::VOPAsmPrefer32Bit) &&
1366       getForcedEncodingSize() != 64)
1367     return Match_PreferE32;
1368 
1369   if (Inst.getOpcode() == AMDGPU::V_MAC_F16_sdwa ||
1370       Inst.getOpcode() == AMDGPU::V_MAC_F32_sdwa) {
1371     // v_mac_f32/16 allow only dst_sel == DWORD;
1372     auto OpNum = AMDGPU::getNamedOperandIdx(Inst.getOpcode(), AMDGPU::OpName::dst_sel);
1373     const auto &Op = Inst.getOperand(OpNum);
1374     if (!Op.isImm() || Op.getImm() != AMDGPU::SDWA::SdwaSel::DWORD) {
1375       return Match_InvalidOperand;
1376     }
1377   }
1378 
1379   return Match_Success;
1380 }
1381 
1382 bool AMDGPUAsmParser::MatchAndEmitInstruction(SMLoc IDLoc, unsigned &Opcode,
1383                                               OperandVector &Operands,
1384                                               MCStreamer &Out,
1385                                               uint64_t &ErrorInfo,
1386                                               bool MatchingInlineAsm) {
1387   // What asm variants we should check
1388   std::vector<unsigned> MatchedVariants;
1389   if (getForcedEncodingSize() == 32) {
1390     MatchedVariants = {AMDGPUAsmVariants::DEFAULT};
1391   } else if (isForcedVOP3()) {
1392     MatchedVariants = {AMDGPUAsmVariants::VOP3};
1393   } else if (isForcedSDWA()) {
1394     MatchedVariants = {AMDGPUAsmVariants::SDWA};
1395   } else if (isForcedDPP()) {
1396     MatchedVariants = {AMDGPUAsmVariants::DPP};
1397   } else {
1398     MatchedVariants = {AMDGPUAsmVariants::DEFAULT,
1399                        AMDGPUAsmVariants::VOP3,
1400                        AMDGPUAsmVariants::SDWA,
1401                        AMDGPUAsmVariants::DPP};
1402   }
1403 
1404   MCInst Inst;
1405   unsigned Result = Match_Success;
1406   for (auto Variant : MatchedVariants) {
1407     uint64_t EI;
1408     auto R = MatchInstructionImpl(Operands, Inst, EI, MatchingInlineAsm,
1409                                   Variant);
1410     // We order match statuses from least to most specific. We use most specific
1411     // status as resulting
1412     // Match_MnemonicFail < Match_InvalidOperand < Match_MissingFeature < Match_PreferE32
1413     if ((R == Match_Success) ||
1414         (R == Match_PreferE32) ||
1415         (R == Match_MissingFeature && Result != Match_PreferE32) ||
1416         (R == Match_InvalidOperand && Result != Match_MissingFeature
1417                                    && Result != Match_PreferE32) ||
1418         (R == Match_MnemonicFail   && Result != Match_InvalidOperand
1419                                    && Result != Match_MissingFeature
1420                                    && Result != Match_PreferE32)) {
1421       Result = R;
1422       ErrorInfo = EI;
1423     }
1424     if (R == Match_Success)
1425       break;
1426   }
1427 
1428   switch (Result) {
1429   default: break;
1430   case Match_Success:
1431     Inst.setLoc(IDLoc);
1432     Out.EmitInstruction(Inst, getSTI());
1433     return false;
1434 
1435   case Match_MissingFeature:
1436     return Error(IDLoc, "instruction not supported on this GPU");
1437 
1438   case Match_MnemonicFail:
1439     return Error(IDLoc, "unrecognized instruction mnemonic");
1440 
1441   case Match_InvalidOperand: {
1442     SMLoc ErrorLoc = IDLoc;
1443     if (ErrorInfo != ~0ULL) {
1444       if (ErrorInfo >= Operands.size()) {
1445         return Error(IDLoc, "too few operands for instruction");
1446       }
1447       ErrorLoc = ((AMDGPUOperand &)*Operands[ErrorInfo]).getStartLoc();
1448       if (ErrorLoc == SMLoc())
1449         ErrorLoc = IDLoc;
1450     }
1451     return Error(ErrorLoc, "invalid operand for instruction");
1452   }
1453 
1454   case Match_PreferE32:
1455     return Error(IDLoc, "internal error: instruction without _e64 suffix "
1456                         "should be encoded as e32");
1457   }
1458   llvm_unreachable("Implement any new match types added!");
1459 }
1460 
1461 bool AMDGPUAsmParser::ParseDirectiveMajorMinor(uint32_t &Major,
1462                                                uint32_t &Minor) {
1463   if (getLexer().isNot(AsmToken::Integer))
1464     return TokError("invalid major version");
1465 
1466   Major = getLexer().getTok().getIntVal();
1467   Lex();
1468 
1469   if (getLexer().isNot(AsmToken::Comma))
1470     return TokError("minor version number required, comma expected");
1471   Lex();
1472 
1473   if (getLexer().isNot(AsmToken::Integer))
1474     return TokError("invalid minor version");
1475 
1476   Minor = getLexer().getTok().getIntVal();
1477   Lex();
1478 
1479   return false;
1480 }
1481 
1482 bool AMDGPUAsmParser::ParseDirectiveHSACodeObjectVersion() {
1483 
1484   uint32_t Major;
1485   uint32_t Minor;
1486 
1487   if (ParseDirectiveMajorMinor(Major, Minor))
1488     return true;
1489 
1490   getTargetStreamer().EmitDirectiveHSACodeObjectVersion(Major, Minor);
1491   return false;
1492 }
1493 
1494 bool AMDGPUAsmParser::ParseDirectiveHSACodeObjectISA() {
1495 
1496   uint32_t Major;
1497   uint32_t Minor;
1498   uint32_t Stepping;
1499   StringRef VendorName;
1500   StringRef ArchName;
1501 
1502   // If this directive has no arguments, then use the ISA version for the
1503   // targeted GPU.
1504   if (getLexer().is(AsmToken::EndOfStatement)) {
1505     AMDGPU::IsaVersion Isa = AMDGPU::getIsaVersion(getSTI().getFeatureBits());
1506     getTargetStreamer().EmitDirectiveHSACodeObjectISA(Isa.Major, Isa.Minor,
1507                                                       Isa.Stepping,
1508                                                       "AMD", "AMDGPU");
1509     return false;
1510   }
1511 
1512 
1513   if (ParseDirectiveMajorMinor(Major, Minor))
1514     return true;
1515 
1516   if (getLexer().isNot(AsmToken::Comma))
1517     return TokError("stepping version number required, comma expected");
1518   Lex();
1519 
1520   if (getLexer().isNot(AsmToken::Integer))
1521     return TokError("invalid stepping version");
1522 
1523   Stepping = getLexer().getTok().getIntVal();
1524   Lex();
1525 
1526   if (getLexer().isNot(AsmToken::Comma))
1527     return TokError("vendor name required, comma expected");
1528   Lex();
1529 
1530   if (getLexer().isNot(AsmToken::String))
1531     return TokError("invalid vendor name");
1532 
1533   VendorName = getLexer().getTok().getStringContents();
1534   Lex();
1535 
1536   if (getLexer().isNot(AsmToken::Comma))
1537     return TokError("arch name required, comma expected");
1538   Lex();
1539 
1540   if (getLexer().isNot(AsmToken::String))
1541     return TokError("invalid arch name");
1542 
1543   ArchName = getLexer().getTok().getStringContents();
1544   Lex();
1545 
1546   getTargetStreamer().EmitDirectiveHSACodeObjectISA(Major, Minor, Stepping,
1547                                                     VendorName, ArchName);
1548   return false;
1549 }
1550 
1551 bool AMDGPUAsmParser::ParseAMDKernelCodeTValue(StringRef ID,
1552                                                amd_kernel_code_t &Header) {
1553   SmallString<40> ErrStr;
1554   raw_svector_ostream Err(ErrStr);
1555   if (!parseAmdKernelCodeField(ID, getParser(), Header, Err)) {
1556     return TokError(Err.str());
1557   }
1558   Lex();
1559   return false;
1560 }
1561 
1562 bool AMDGPUAsmParser::ParseDirectiveAMDKernelCodeT() {
1563 
1564   amd_kernel_code_t Header;
1565   AMDGPU::initDefaultAMDKernelCodeT(Header, getSTI().getFeatureBits());
1566 
1567   while (true) {
1568 
1569     // Lex EndOfStatement.  This is in a while loop, because lexing a comment
1570     // will set the current token to EndOfStatement.
1571     while(getLexer().is(AsmToken::EndOfStatement))
1572       Lex();
1573 
1574     if (getLexer().isNot(AsmToken::Identifier))
1575       return TokError("expected value identifier or .end_amd_kernel_code_t");
1576 
1577     StringRef ID = getLexer().getTok().getIdentifier();
1578     Lex();
1579 
1580     if (ID == ".end_amd_kernel_code_t")
1581       break;
1582 
1583     if (ParseAMDKernelCodeTValue(ID, Header))
1584       return true;
1585   }
1586 
1587   getTargetStreamer().EmitAMDKernelCodeT(Header);
1588 
1589   return false;
1590 }
1591 
1592 bool AMDGPUAsmParser::ParseSectionDirectiveHSAText() {
1593   getParser().getStreamer().SwitchSection(
1594       AMDGPU::getHSATextSection(getContext()));
1595   return false;
1596 }
1597 
1598 bool AMDGPUAsmParser::ParseDirectiveAMDGPUHsaKernel() {
1599   if (getLexer().isNot(AsmToken::Identifier))
1600     return TokError("expected symbol name");
1601 
1602   StringRef KernelName = Parser.getTok().getString();
1603 
1604   getTargetStreamer().EmitAMDGPUSymbolType(KernelName,
1605                                            ELF::STT_AMDGPU_HSA_KERNEL);
1606   Lex();
1607   return false;
1608 }
1609 
1610 bool AMDGPUAsmParser::ParseDirectiveAMDGPUHsaModuleGlobal() {
1611   if (getLexer().isNot(AsmToken::Identifier))
1612     return TokError("expected symbol name");
1613 
1614   StringRef GlobalName = Parser.getTok().getIdentifier();
1615 
1616   getTargetStreamer().EmitAMDGPUHsaModuleScopeGlobal(GlobalName);
1617   Lex();
1618   return false;
1619 }
1620 
1621 bool AMDGPUAsmParser::ParseDirectiveAMDGPUHsaProgramGlobal() {
1622   if (getLexer().isNot(AsmToken::Identifier))
1623     return TokError("expected symbol name");
1624 
1625   StringRef GlobalName = Parser.getTok().getIdentifier();
1626 
1627   getTargetStreamer().EmitAMDGPUHsaProgramScopeGlobal(GlobalName);
1628   Lex();
1629   return false;
1630 }
1631 
1632 bool AMDGPUAsmParser::ParseSectionDirectiveHSADataGlobalAgent() {
1633   getParser().getStreamer().SwitchSection(
1634       AMDGPU::getHSADataGlobalAgentSection(getContext()));
1635   return false;
1636 }
1637 
1638 bool AMDGPUAsmParser::ParseSectionDirectiveHSADataGlobalProgram() {
1639   getParser().getStreamer().SwitchSection(
1640       AMDGPU::getHSADataGlobalProgramSection(getContext()));
1641   return false;
1642 }
1643 
1644 bool AMDGPUAsmParser::ParseSectionDirectiveHSARodataReadonlyAgent() {
1645   getParser().getStreamer().SwitchSection(
1646       AMDGPU::getHSARodataReadonlyAgentSection(getContext()));
1647   return false;
1648 }
1649 
1650 bool AMDGPUAsmParser::ParseDirective(AsmToken DirectiveID) {
1651   StringRef IDVal = DirectiveID.getString();
1652 
1653   if (IDVal == ".hsa_code_object_version")
1654     return ParseDirectiveHSACodeObjectVersion();
1655 
1656   if (IDVal == ".hsa_code_object_isa")
1657     return ParseDirectiveHSACodeObjectISA();
1658 
1659   if (IDVal == ".amd_kernel_code_t")
1660     return ParseDirectiveAMDKernelCodeT();
1661 
1662   if (IDVal == ".hsatext")
1663     return ParseSectionDirectiveHSAText();
1664 
1665   if (IDVal == ".amdgpu_hsa_kernel")
1666     return ParseDirectiveAMDGPUHsaKernel();
1667 
1668   if (IDVal == ".amdgpu_hsa_module_global")
1669     return ParseDirectiveAMDGPUHsaModuleGlobal();
1670 
1671   if (IDVal == ".amdgpu_hsa_program_global")
1672     return ParseDirectiveAMDGPUHsaProgramGlobal();
1673 
1674   if (IDVal == ".hsadata_global_agent")
1675     return ParseSectionDirectiveHSADataGlobalAgent();
1676 
1677   if (IDVal == ".hsadata_global_program")
1678     return ParseSectionDirectiveHSADataGlobalProgram();
1679 
1680   if (IDVal == ".hsarodata_readonly_agent")
1681     return ParseSectionDirectiveHSARodataReadonlyAgent();
1682 
1683   return true;
1684 }
1685 
1686 bool AMDGPUAsmParser::subtargetHasRegister(const MCRegisterInfo &MRI,
1687                                            unsigned RegNo) const {
1688   if (isCI())
1689     return true;
1690 
1691   if (isSI()) {
1692     // No flat_scr
1693     switch (RegNo) {
1694     case AMDGPU::FLAT_SCR:
1695     case AMDGPU::FLAT_SCR_LO:
1696     case AMDGPU::FLAT_SCR_HI:
1697       return false;
1698     default:
1699       return true;
1700     }
1701   }
1702 
1703   // VI only has 102 SGPRs, so make sure we aren't trying to use the 2 more that
1704   // SI/CI have.
1705   for (MCRegAliasIterator R(AMDGPU::SGPR102_SGPR103, &MRI, true);
1706        R.isValid(); ++R) {
1707     if (*R == RegNo)
1708       return false;
1709   }
1710 
1711   return true;
1712 }
1713 
1714 AMDGPUAsmParser::OperandMatchResultTy
1715 AMDGPUAsmParser::parseOperand(OperandVector &Operands, StringRef Mnemonic) {
1716 
1717   // Try to parse with a custom parser
1718   OperandMatchResultTy ResTy = MatchOperandParserImpl(Operands, Mnemonic);
1719 
1720   // If we successfully parsed the operand or if there as an error parsing,
1721   // we are done.
1722   //
1723   // If we are parsing after we reach EndOfStatement then this means we
1724   // are appending default values to the Operands list.  This is only done
1725   // by custom parser, so we shouldn't continue on to the generic parsing.
1726   if (ResTy == MatchOperand_Success || ResTy == MatchOperand_ParseFail ||
1727       getLexer().is(AsmToken::EndOfStatement))
1728     return ResTy;
1729 
1730   ResTy = parseRegOrImm(Operands);
1731 
1732   if (ResTy == MatchOperand_Success)
1733     return ResTy;
1734 
1735   if (getLexer().getKind() == AsmToken::Identifier) {
1736     // If this identifier is a symbol, we want to create an expression for it.
1737     // It is a little difficult to distinguish between a symbol name, and
1738     // an instruction flag like 'gds'.  In order to do this, we parse
1739     // all tokens as expressions and then treate the symbol name as the token
1740     // string when we want to interpret the operand as a token.
1741     const auto &Tok = Parser.getTok();
1742     SMLoc S = Tok.getLoc();
1743     const MCExpr *Expr = nullptr;
1744     if (!Parser.parseExpression(Expr)) {
1745       Operands.push_back(AMDGPUOperand::CreateExpr(this, Expr, S));
1746       return MatchOperand_Success;
1747     }
1748 
1749     Operands.push_back(AMDGPUOperand::CreateToken(this, Tok.getString(), Tok.getLoc()));
1750     Parser.Lex();
1751     return MatchOperand_Success;
1752   }
1753   return MatchOperand_NoMatch;
1754 }
1755 
1756 StringRef AMDGPUAsmParser::parseMnemonicSuffix(StringRef Name) {
1757   // Clear any forced encodings from the previous instruction.
1758   setForcedEncodingSize(0);
1759   setForcedDPP(false);
1760   setForcedSDWA(false);
1761 
1762   if (Name.endswith("_e64")) {
1763     setForcedEncodingSize(64);
1764     return Name.substr(0, Name.size() - 4);
1765   } else if (Name.endswith("_e32")) {
1766     setForcedEncodingSize(32);
1767     return Name.substr(0, Name.size() - 4);
1768   } else if (Name.endswith("_dpp")) {
1769     setForcedDPP(true);
1770     return Name.substr(0, Name.size() - 4);
1771   } else if (Name.endswith("_sdwa")) {
1772     setForcedSDWA(true);
1773     return Name.substr(0, Name.size() - 5);
1774   }
1775   return Name;
1776 }
1777 
1778 bool AMDGPUAsmParser::ParseInstruction(ParseInstructionInfo &Info,
1779                                        StringRef Name,
1780                                        SMLoc NameLoc, OperandVector &Operands) {
1781   // Add the instruction mnemonic
1782   Name = parseMnemonicSuffix(Name);
1783   Operands.push_back(AMDGPUOperand::CreateToken(this, Name, NameLoc));
1784 
1785   while (!getLexer().is(AsmToken::EndOfStatement)) {
1786     AMDGPUAsmParser::OperandMatchResultTy Res = parseOperand(Operands, Name);
1787 
1788     // Eat the comma or space if there is one.
1789     if (getLexer().is(AsmToken::Comma))
1790       Parser.Lex();
1791 
1792     switch (Res) {
1793       case MatchOperand_Success: break;
1794       case MatchOperand_ParseFail:
1795         Error(getLexer().getLoc(), "failed parsing operand.");
1796         while (!getLexer().is(AsmToken::EndOfStatement)) {
1797           Parser.Lex();
1798         }
1799         return true;
1800       case MatchOperand_NoMatch:
1801         Error(getLexer().getLoc(), "not a valid operand.");
1802         while (!getLexer().is(AsmToken::EndOfStatement)) {
1803           Parser.Lex();
1804         }
1805         return true;
1806     }
1807   }
1808 
1809   return false;
1810 }
1811 
1812 //===----------------------------------------------------------------------===//
1813 // Utility functions
1814 //===----------------------------------------------------------------------===//
1815 
1816 AMDGPUAsmParser::OperandMatchResultTy
1817 AMDGPUAsmParser::parseIntWithPrefix(const char *Prefix, int64_t &Int) {
1818   switch(getLexer().getKind()) {
1819     default: return MatchOperand_NoMatch;
1820     case AsmToken::Identifier: {
1821       StringRef Name = Parser.getTok().getString();
1822       if (!Name.equals(Prefix)) {
1823         return MatchOperand_NoMatch;
1824       }
1825 
1826       Parser.Lex();
1827       if (getLexer().isNot(AsmToken::Colon))
1828         return MatchOperand_ParseFail;
1829 
1830       Parser.Lex();
1831       if (getLexer().isNot(AsmToken::Integer))
1832         return MatchOperand_ParseFail;
1833 
1834       if (getParser().parseAbsoluteExpression(Int))
1835         return MatchOperand_ParseFail;
1836       break;
1837     }
1838   }
1839   return MatchOperand_Success;
1840 }
1841 
1842 AMDGPUAsmParser::OperandMatchResultTy
1843 AMDGPUAsmParser::parseIntWithPrefix(const char *Prefix, OperandVector &Operands,
1844                                     enum AMDGPUOperand::ImmTy ImmTy,
1845                                     bool (*ConvertResult)(int64_t&)) {
1846 
1847   SMLoc S = Parser.getTok().getLoc();
1848   int64_t Value = 0;
1849 
1850   AMDGPUAsmParser::OperandMatchResultTy Res = parseIntWithPrefix(Prefix, Value);
1851   if (Res != MatchOperand_Success)
1852     return Res;
1853 
1854   if (ConvertResult && !ConvertResult(Value)) {
1855     return MatchOperand_ParseFail;
1856   }
1857 
1858   Operands.push_back(AMDGPUOperand::CreateImm(this, Value, S, ImmTy));
1859   return MatchOperand_Success;
1860 }
1861 
1862 AMDGPUAsmParser::OperandMatchResultTy
1863 AMDGPUAsmParser::parseNamedBit(const char *Name, OperandVector &Operands,
1864                                enum AMDGPUOperand::ImmTy ImmTy) {
1865   int64_t Bit = 0;
1866   SMLoc S = Parser.getTok().getLoc();
1867 
1868   // We are at the end of the statement, and this is a default argument, so
1869   // use a default value.
1870   if (getLexer().isNot(AsmToken::EndOfStatement)) {
1871     switch(getLexer().getKind()) {
1872       case AsmToken::Identifier: {
1873         StringRef Tok = Parser.getTok().getString();
1874         if (Tok == Name) {
1875           Bit = 1;
1876           Parser.Lex();
1877         } else if (Tok.startswith("no") && Tok.endswith(Name)) {
1878           Bit = 0;
1879           Parser.Lex();
1880         } else {
1881           return MatchOperand_NoMatch;
1882         }
1883         break;
1884       }
1885       default:
1886         return MatchOperand_NoMatch;
1887     }
1888   }
1889 
1890   Operands.push_back(AMDGPUOperand::CreateImm(this, Bit, S, ImmTy));
1891   return MatchOperand_Success;
1892 }
1893 
1894 typedef std::map<enum AMDGPUOperand::ImmTy, unsigned> OptionalImmIndexMap;
1895 
1896 void addOptionalImmOperand(MCInst& Inst, const OperandVector& Operands,
1897                            OptionalImmIndexMap& OptionalIdx,
1898                            enum AMDGPUOperand::ImmTy ImmT, int64_t Default = 0) {
1899   auto i = OptionalIdx.find(ImmT);
1900   if (i != OptionalIdx.end()) {
1901     unsigned Idx = i->second;
1902     ((AMDGPUOperand &)*Operands[Idx]).addImmOperands(Inst, 1);
1903   } else {
1904     Inst.addOperand(MCOperand::createImm(Default));
1905   }
1906 }
1907 
1908 AMDGPUAsmParser::OperandMatchResultTy
1909 AMDGPUAsmParser::parseStringWithPrefix(StringRef Prefix, StringRef &Value) {
1910   if (getLexer().isNot(AsmToken::Identifier)) {
1911     return MatchOperand_NoMatch;
1912   }
1913   StringRef Tok = Parser.getTok().getString();
1914   if (Tok != Prefix) {
1915     return MatchOperand_NoMatch;
1916   }
1917 
1918   Parser.Lex();
1919   if (getLexer().isNot(AsmToken::Colon)) {
1920     return MatchOperand_ParseFail;
1921   }
1922 
1923   Parser.Lex();
1924   if (getLexer().isNot(AsmToken::Identifier)) {
1925     return MatchOperand_ParseFail;
1926   }
1927 
1928   Value = Parser.getTok().getString();
1929   return MatchOperand_Success;
1930 }
1931 
1932 //===----------------------------------------------------------------------===//
1933 // ds
1934 //===----------------------------------------------------------------------===//
1935 
1936 void AMDGPUAsmParser::cvtDSOffset01(MCInst &Inst,
1937                                     const OperandVector &Operands) {
1938 
1939   OptionalImmIndexMap OptionalIdx;
1940 
1941   for (unsigned i = 1, e = Operands.size(); i != e; ++i) {
1942     AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[i]);
1943 
1944     // Add the register arguments
1945     if (Op.isReg()) {
1946       Op.addRegOperands(Inst, 1);
1947       continue;
1948     }
1949 
1950     // Handle optional arguments
1951     OptionalIdx[Op.getImmTy()] = i;
1952   }
1953 
1954   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyOffset0);
1955   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyOffset1);
1956   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyGDS);
1957 
1958   Inst.addOperand(MCOperand::createReg(AMDGPU::M0)); // m0
1959 }
1960 
1961 void AMDGPUAsmParser::cvtDS(MCInst &Inst, const OperandVector &Operands) {
1962 
1963   std::map<enum AMDGPUOperand::ImmTy, unsigned> OptionalIdx;
1964   bool GDSOnly = false;
1965 
1966   for (unsigned i = 1, e = Operands.size(); i != e; ++i) {
1967     AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[i]);
1968 
1969     // Add the register arguments
1970     if (Op.isReg()) {
1971       Op.addRegOperands(Inst, 1);
1972       continue;
1973     }
1974 
1975     if (Op.isToken() && Op.getToken() == "gds") {
1976       GDSOnly = true;
1977       continue;
1978     }
1979 
1980     // Handle optional arguments
1981     OptionalIdx[Op.getImmTy()] = i;
1982   }
1983 
1984   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyOffset);
1985   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyGDS);
1986 
1987   if (!GDSOnly) {
1988     addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyGDS);
1989   }
1990   Inst.addOperand(MCOperand::createReg(AMDGPU::M0)); // m0
1991 }
1992 
1993 
1994 //===----------------------------------------------------------------------===//
1995 // s_waitcnt
1996 //===----------------------------------------------------------------------===//
1997 
1998 bool AMDGPUAsmParser::parseCnt(int64_t &IntVal) {
1999   StringRef CntName = Parser.getTok().getString();
2000   int64_t CntVal;
2001 
2002   Parser.Lex();
2003   if (getLexer().isNot(AsmToken::LParen))
2004     return true;
2005 
2006   Parser.Lex();
2007   if (getLexer().isNot(AsmToken::Integer))
2008     return true;
2009 
2010   if (getParser().parseAbsoluteExpression(CntVal))
2011     return true;
2012 
2013   if (getLexer().isNot(AsmToken::RParen))
2014     return true;
2015 
2016   Parser.Lex();
2017   if (getLexer().is(AsmToken::Amp) || getLexer().is(AsmToken::Comma))
2018     Parser.Lex();
2019 
2020   IsaVersion IV = getIsaVersion(getSTI().getFeatureBits());
2021   if (CntName == "vmcnt")
2022     IntVal = encodeVmcnt(IV, IntVal, CntVal);
2023   else if (CntName == "expcnt")
2024     IntVal = encodeExpcnt(IV, IntVal, CntVal);
2025   else if (CntName == "lgkmcnt")
2026     IntVal = encodeLgkmcnt(IV, IntVal, CntVal);
2027   else
2028     return true;
2029 
2030   return false;
2031 }
2032 
2033 AMDGPUAsmParser::OperandMatchResultTy
2034 AMDGPUAsmParser::parseSWaitCntOps(OperandVector &Operands) {
2035   IsaVersion IV = getIsaVersion(getSTI().getFeatureBits());
2036   int64_t Waitcnt = getWaitcntBitMask(IV);
2037   SMLoc S = Parser.getTok().getLoc();
2038 
2039   switch(getLexer().getKind()) {
2040     default: return MatchOperand_ParseFail;
2041     case AsmToken::Integer:
2042       // The operand can be an integer value.
2043       if (getParser().parseAbsoluteExpression(Waitcnt))
2044         return MatchOperand_ParseFail;
2045       break;
2046 
2047     case AsmToken::Identifier:
2048       do {
2049         if (parseCnt(Waitcnt))
2050           return MatchOperand_ParseFail;
2051       } while(getLexer().isNot(AsmToken::EndOfStatement));
2052       break;
2053   }
2054   Operands.push_back(AMDGPUOperand::CreateImm(this, Waitcnt, S));
2055   return MatchOperand_Success;
2056 }
2057 
2058 bool AMDGPUAsmParser::parseHwregConstruct(OperandInfoTy &HwReg, int64_t &Offset, int64_t &Width) {
2059   using namespace llvm::AMDGPU::Hwreg;
2060 
2061   if (Parser.getTok().getString() != "hwreg")
2062     return true;
2063   Parser.Lex();
2064 
2065   if (getLexer().isNot(AsmToken::LParen))
2066     return true;
2067   Parser.Lex();
2068 
2069   if (getLexer().is(AsmToken::Identifier)) {
2070     HwReg.IsSymbolic = true;
2071     HwReg.Id = ID_UNKNOWN_;
2072     const StringRef tok = Parser.getTok().getString();
2073     for (int i = ID_SYMBOLIC_FIRST_; i < ID_SYMBOLIC_LAST_; ++i) {
2074       if (tok == IdSymbolic[i]) {
2075         HwReg.Id = i;
2076         break;
2077       }
2078     }
2079     Parser.Lex();
2080   } else {
2081     HwReg.IsSymbolic = false;
2082     if (getLexer().isNot(AsmToken::Integer))
2083       return true;
2084     if (getParser().parseAbsoluteExpression(HwReg.Id))
2085       return true;
2086   }
2087 
2088   if (getLexer().is(AsmToken::RParen)) {
2089     Parser.Lex();
2090     return false;
2091   }
2092 
2093   // optional params
2094   if (getLexer().isNot(AsmToken::Comma))
2095     return true;
2096   Parser.Lex();
2097 
2098   if (getLexer().isNot(AsmToken::Integer))
2099     return true;
2100   if (getParser().parseAbsoluteExpression(Offset))
2101     return true;
2102 
2103   if (getLexer().isNot(AsmToken::Comma))
2104     return true;
2105   Parser.Lex();
2106 
2107   if (getLexer().isNot(AsmToken::Integer))
2108     return true;
2109   if (getParser().parseAbsoluteExpression(Width))
2110     return true;
2111 
2112   if (getLexer().isNot(AsmToken::RParen))
2113     return true;
2114   Parser.Lex();
2115 
2116   return false;
2117 }
2118 
2119 AMDGPUAsmParser::OperandMatchResultTy
2120 AMDGPUAsmParser::parseHwreg(OperandVector &Operands) {
2121   using namespace llvm::AMDGPU::Hwreg;
2122 
2123   int64_t Imm16Val = 0;
2124   SMLoc S = Parser.getTok().getLoc();
2125 
2126   switch(getLexer().getKind()) {
2127     default: return MatchOperand_NoMatch;
2128     case AsmToken::Integer:
2129       // The operand can be an integer value.
2130       if (getParser().parseAbsoluteExpression(Imm16Val))
2131         return MatchOperand_NoMatch;
2132       if (Imm16Val < 0 || !isUInt<16>(Imm16Val)) {
2133         Error(S, "invalid immediate: only 16-bit values are legal");
2134         // Do not return error code, but create an imm operand anyway and proceed
2135         // to the next operand, if any. That avoids unneccessary error messages.
2136       }
2137       break;
2138 
2139     case AsmToken::Identifier: {
2140         OperandInfoTy HwReg(ID_UNKNOWN_);
2141         int64_t Offset = OFFSET_DEFAULT_;
2142         int64_t Width = WIDTH_M1_DEFAULT_ + 1;
2143         if (parseHwregConstruct(HwReg, Offset, Width))
2144           return MatchOperand_ParseFail;
2145         if (HwReg.Id < 0 || !isUInt<ID_WIDTH_>(HwReg.Id)) {
2146           if (HwReg.IsSymbolic)
2147             Error(S, "invalid symbolic name of hardware register");
2148           else
2149             Error(S, "invalid code of hardware register: only 6-bit values are legal");
2150         }
2151         if (Offset < 0 || !isUInt<OFFSET_WIDTH_>(Offset))
2152           Error(S, "invalid bit offset: only 5-bit values are legal");
2153         if ((Width-1) < 0 || !isUInt<WIDTH_M1_WIDTH_>(Width-1))
2154           Error(S, "invalid bitfield width: only values from 1 to 32 are legal");
2155         Imm16Val = (HwReg.Id << ID_SHIFT_) | (Offset << OFFSET_SHIFT_) | ((Width-1) << WIDTH_M1_SHIFT_);
2156       }
2157       break;
2158   }
2159   Operands.push_back(AMDGPUOperand::CreateImm(this, Imm16Val, S, AMDGPUOperand::ImmTyHwreg));
2160   return MatchOperand_Success;
2161 }
2162 
2163 bool AMDGPUOperand::isSWaitCnt() const {
2164   return isImm();
2165 }
2166 
2167 bool AMDGPUOperand::isHwreg() const {
2168   return isImmTy(ImmTyHwreg);
2169 }
2170 
2171 bool AMDGPUAsmParser::parseSendMsgConstruct(OperandInfoTy &Msg, OperandInfoTy &Operation, int64_t &StreamId) {
2172   using namespace llvm::AMDGPU::SendMsg;
2173 
2174   if (Parser.getTok().getString() != "sendmsg")
2175     return true;
2176   Parser.Lex();
2177 
2178   if (getLexer().isNot(AsmToken::LParen))
2179     return true;
2180   Parser.Lex();
2181 
2182   if (getLexer().is(AsmToken::Identifier)) {
2183     Msg.IsSymbolic = true;
2184     Msg.Id = ID_UNKNOWN_;
2185     const std::string tok = Parser.getTok().getString();
2186     for (int i = ID_GAPS_FIRST_; i < ID_GAPS_LAST_; ++i) {
2187       switch(i) {
2188         default: continue; // Omit gaps.
2189         case ID_INTERRUPT: case ID_GS: case ID_GS_DONE:  case ID_SYSMSG: break;
2190       }
2191       if (tok == IdSymbolic[i]) {
2192         Msg.Id = i;
2193         break;
2194       }
2195     }
2196     Parser.Lex();
2197   } else {
2198     Msg.IsSymbolic = false;
2199     if (getLexer().isNot(AsmToken::Integer))
2200       return true;
2201     if (getParser().parseAbsoluteExpression(Msg.Id))
2202       return true;
2203     if (getLexer().is(AsmToken::Integer))
2204       if (getParser().parseAbsoluteExpression(Msg.Id))
2205         Msg.Id = ID_UNKNOWN_;
2206   }
2207   if (Msg.Id == ID_UNKNOWN_) // Don't know how to parse the rest.
2208     return false;
2209 
2210   if (!(Msg.Id == ID_GS || Msg.Id == ID_GS_DONE || Msg.Id == ID_SYSMSG)) {
2211     if (getLexer().isNot(AsmToken::RParen))
2212       return true;
2213     Parser.Lex();
2214     return false;
2215   }
2216 
2217   if (getLexer().isNot(AsmToken::Comma))
2218     return true;
2219   Parser.Lex();
2220 
2221   assert(Msg.Id == ID_GS || Msg.Id == ID_GS_DONE || Msg.Id == ID_SYSMSG);
2222   Operation.Id = ID_UNKNOWN_;
2223   if (getLexer().is(AsmToken::Identifier)) {
2224     Operation.IsSymbolic = true;
2225     const char* const *S = (Msg.Id == ID_SYSMSG) ? OpSysSymbolic : OpGsSymbolic;
2226     const int F = (Msg.Id == ID_SYSMSG) ? OP_SYS_FIRST_ : OP_GS_FIRST_;
2227     const int L = (Msg.Id == ID_SYSMSG) ? OP_SYS_LAST_ : OP_GS_LAST_;
2228     const StringRef Tok = Parser.getTok().getString();
2229     for (int i = F; i < L; ++i) {
2230       if (Tok == S[i]) {
2231         Operation.Id = i;
2232         break;
2233       }
2234     }
2235     Parser.Lex();
2236   } else {
2237     Operation.IsSymbolic = false;
2238     if (getLexer().isNot(AsmToken::Integer))
2239       return true;
2240     if (getParser().parseAbsoluteExpression(Operation.Id))
2241       return true;
2242   }
2243 
2244   if ((Msg.Id == ID_GS || Msg.Id == ID_GS_DONE) && Operation.Id != OP_GS_NOP) {
2245     // Stream id is optional.
2246     if (getLexer().is(AsmToken::RParen)) {
2247       Parser.Lex();
2248       return false;
2249     }
2250 
2251     if (getLexer().isNot(AsmToken::Comma))
2252       return true;
2253     Parser.Lex();
2254 
2255     if (getLexer().isNot(AsmToken::Integer))
2256       return true;
2257     if (getParser().parseAbsoluteExpression(StreamId))
2258       return true;
2259   }
2260 
2261   if (getLexer().isNot(AsmToken::RParen))
2262     return true;
2263   Parser.Lex();
2264   return false;
2265 }
2266 
2267 AMDGPUAsmParser::OperandMatchResultTy
2268 AMDGPUAsmParser::parseSendMsgOp(OperandVector &Operands) {
2269   using namespace llvm::AMDGPU::SendMsg;
2270 
2271   int64_t Imm16Val = 0;
2272   SMLoc S = Parser.getTok().getLoc();
2273 
2274   switch(getLexer().getKind()) {
2275   default:
2276     return MatchOperand_NoMatch;
2277   case AsmToken::Integer:
2278     // The operand can be an integer value.
2279     if (getParser().parseAbsoluteExpression(Imm16Val))
2280       return MatchOperand_NoMatch;
2281     if (Imm16Val < 0 || !isUInt<16>(Imm16Val)) {
2282       Error(S, "invalid immediate: only 16-bit values are legal");
2283       // Do not return error code, but create an imm operand anyway and proceed
2284       // to the next operand, if any. That avoids unneccessary error messages.
2285     }
2286     break;
2287   case AsmToken::Identifier: {
2288       OperandInfoTy Msg(ID_UNKNOWN_);
2289       OperandInfoTy Operation(OP_UNKNOWN_);
2290       int64_t StreamId = STREAM_ID_DEFAULT_;
2291       if (parseSendMsgConstruct(Msg, Operation, StreamId))
2292         return MatchOperand_ParseFail;
2293       do {
2294         // Validate and encode message ID.
2295         if (! ((ID_INTERRUPT <= Msg.Id && Msg.Id <= ID_GS_DONE)
2296                 || Msg.Id == ID_SYSMSG)) {
2297           if (Msg.IsSymbolic)
2298             Error(S, "invalid/unsupported symbolic name of message");
2299           else
2300             Error(S, "invalid/unsupported code of message");
2301           break;
2302         }
2303         Imm16Val = (Msg.Id << ID_SHIFT_);
2304         // Validate and encode operation ID.
2305         if (Msg.Id == ID_GS || Msg.Id == ID_GS_DONE) {
2306           if (! (OP_GS_FIRST_ <= Operation.Id && Operation.Id < OP_GS_LAST_)) {
2307             if (Operation.IsSymbolic)
2308               Error(S, "invalid symbolic name of GS_OP");
2309             else
2310               Error(S, "invalid code of GS_OP: only 2-bit values are legal");
2311             break;
2312           }
2313           if (Operation.Id == OP_GS_NOP
2314               && Msg.Id != ID_GS_DONE) {
2315             Error(S, "invalid GS_OP: NOP is for GS_DONE only");
2316             break;
2317           }
2318           Imm16Val |= (Operation.Id << OP_SHIFT_);
2319         }
2320         if (Msg.Id == ID_SYSMSG) {
2321           if (! (OP_SYS_FIRST_ <= Operation.Id && Operation.Id < OP_SYS_LAST_)) {
2322             if (Operation.IsSymbolic)
2323               Error(S, "invalid/unsupported symbolic name of SYSMSG_OP");
2324             else
2325               Error(S, "invalid/unsupported code of SYSMSG_OP");
2326             break;
2327           }
2328           Imm16Val |= (Operation.Id << OP_SHIFT_);
2329         }
2330         // Validate and encode stream ID.
2331         if ((Msg.Id == ID_GS || Msg.Id == ID_GS_DONE) && Operation.Id != OP_GS_NOP) {
2332           if (! (STREAM_ID_FIRST_ <= StreamId && StreamId < STREAM_ID_LAST_)) {
2333             Error(S, "invalid stream id: only 2-bit values are legal");
2334             break;
2335           }
2336           Imm16Val |= (StreamId << STREAM_ID_SHIFT_);
2337         }
2338       } while (0);
2339     }
2340     break;
2341   }
2342   Operands.push_back(AMDGPUOperand::CreateImm(this, Imm16Val, S, AMDGPUOperand::ImmTySendMsg));
2343   return MatchOperand_Success;
2344 }
2345 
2346 bool AMDGPUOperand::isSendMsg() const {
2347   return isImmTy(ImmTySendMsg);
2348 }
2349 
2350 //===----------------------------------------------------------------------===//
2351 // sopp branch targets
2352 //===----------------------------------------------------------------------===//
2353 
2354 AMDGPUAsmParser::OperandMatchResultTy
2355 AMDGPUAsmParser::parseSOppBrTarget(OperandVector &Operands) {
2356   SMLoc S = Parser.getTok().getLoc();
2357 
2358   switch (getLexer().getKind()) {
2359     default: return MatchOperand_ParseFail;
2360     case AsmToken::Integer: {
2361       int64_t Imm;
2362       if (getParser().parseAbsoluteExpression(Imm))
2363         return MatchOperand_ParseFail;
2364       Operands.push_back(AMDGPUOperand::CreateImm(this, Imm, S));
2365       return MatchOperand_Success;
2366     }
2367 
2368     case AsmToken::Identifier:
2369       Operands.push_back(AMDGPUOperand::CreateExpr(this,
2370           MCSymbolRefExpr::create(getContext().getOrCreateSymbol(
2371                                   Parser.getTok().getString()), getContext()), S));
2372       Parser.Lex();
2373       return MatchOperand_Success;
2374   }
2375 }
2376 
2377 //===----------------------------------------------------------------------===//
2378 // mubuf
2379 //===----------------------------------------------------------------------===//
2380 
2381 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultGLC() const {
2382   return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTyGLC);
2383 }
2384 
2385 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultSLC() const {
2386   return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTySLC);
2387 }
2388 
2389 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultTFE() const {
2390   return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTyTFE);
2391 }
2392 
2393 void AMDGPUAsmParser::cvtMubufImpl(MCInst &Inst,
2394                                const OperandVector &Operands,
2395                                bool IsAtomic, bool IsAtomicReturn) {
2396   OptionalImmIndexMap OptionalIdx;
2397   assert(IsAtomicReturn ? IsAtomic : true);
2398 
2399   for (unsigned i = 1, e = Operands.size(); i != e; ++i) {
2400     AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[i]);
2401 
2402     // Add the register arguments
2403     if (Op.isReg()) {
2404       Op.addRegOperands(Inst, 1);
2405       continue;
2406     }
2407 
2408     // Handle the case where soffset is an immediate
2409     if (Op.isImm() && Op.getImmTy() == AMDGPUOperand::ImmTyNone) {
2410       Op.addImmOperands(Inst, 1);
2411       continue;
2412     }
2413 
2414     // Handle tokens like 'offen' which are sometimes hard-coded into the
2415     // asm string.  There are no MCInst operands for these.
2416     if (Op.isToken()) {
2417       continue;
2418     }
2419     assert(Op.isImm());
2420 
2421     // Handle optional arguments
2422     OptionalIdx[Op.getImmTy()] = i;
2423   }
2424 
2425   // Copy $vdata_in operand and insert as $vdata for MUBUF_Atomic RTN insns.
2426   if (IsAtomicReturn) {
2427     MCInst::iterator I = Inst.begin(); // $vdata_in is always at the beginning.
2428     Inst.insert(I, *I);
2429   }
2430 
2431   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyOffset);
2432   if (!IsAtomic) { // glc is hard-coded.
2433     addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyGLC);
2434   }
2435   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySLC);
2436   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyTFE);
2437 }
2438 
2439 //===----------------------------------------------------------------------===//
2440 // mimg
2441 //===----------------------------------------------------------------------===//
2442 
2443 void AMDGPUAsmParser::cvtMIMG(MCInst &Inst, const OperandVector &Operands) {
2444   unsigned I = 1;
2445   const MCInstrDesc &Desc = MII.get(Inst.getOpcode());
2446   for (unsigned J = 0; J < Desc.getNumDefs(); ++J) {
2447     ((AMDGPUOperand &)*Operands[I++]).addRegOperands(Inst, 1);
2448   }
2449 
2450   OptionalImmIndexMap OptionalIdx;
2451 
2452   for (unsigned E = Operands.size(); I != E; ++I) {
2453     AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[I]);
2454 
2455     // Add the register arguments
2456     if (Op.isRegOrImm()) {
2457       Op.addRegOrImmOperands(Inst, 1);
2458       continue;
2459     } else if (Op.isImmModifier()) {
2460       OptionalIdx[Op.getImmTy()] = I;
2461     } else {
2462       assert(false);
2463     }
2464   }
2465 
2466   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyDMask);
2467   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyUNorm);
2468   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyGLC);
2469   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyDA);
2470   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyR128);
2471   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyTFE);
2472   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyLWE);
2473   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySLC);
2474 }
2475 
2476 void AMDGPUAsmParser::cvtMIMGAtomic(MCInst &Inst, const OperandVector &Operands) {
2477   unsigned I = 1;
2478   const MCInstrDesc &Desc = MII.get(Inst.getOpcode());
2479   for (unsigned J = 0; J < Desc.getNumDefs(); ++J) {
2480     ((AMDGPUOperand &)*Operands[I++]).addRegOperands(Inst, 1);
2481   }
2482 
2483   // Add src, same as dst
2484   ((AMDGPUOperand &)*Operands[I]).addRegOperands(Inst, 1);
2485 
2486   OptionalImmIndexMap OptionalIdx;
2487 
2488   for (unsigned E = Operands.size(); I != E; ++I) {
2489     AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[I]);
2490 
2491     // Add the register arguments
2492     if (Op.isRegOrImm()) {
2493       Op.addRegOrImmOperands(Inst, 1);
2494       continue;
2495     } else if (Op.isImmModifier()) {
2496       OptionalIdx[Op.getImmTy()] = I;
2497     } else {
2498       assert(false);
2499     }
2500   }
2501 
2502   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyDMask);
2503   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyUNorm);
2504   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyGLC);
2505   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyDA);
2506   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyR128);
2507   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyTFE);
2508   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyLWE);
2509   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySLC);
2510 }
2511 
2512 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultDMask() const {
2513   return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTyDMask);
2514 }
2515 
2516 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultUNorm() const {
2517   return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTyUNorm);
2518 }
2519 
2520 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultDA() const {
2521   return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTyDA);
2522 }
2523 
2524 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultR128() const {
2525   return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTyR128);
2526 }
2527 
2528 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultLWE() const {
2529   return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTyLWE);
2530 }
2531 
2532 //===----------------------------------------------------------------------===//
2533 // smrd
2534 //===----------------------------------------------------------------------===//
2535 
2536 bool AMDGPUOperand::isSMRDOffset() const {
2537 
2538   // FIXME: Support 20-bit offsets on VI.  We need to to pass subtarget
2539   // information here.
2540   return isImm() && isUInt<8>(getImm());
2541 }
2542 
2543 bool AMDGPUOperand::isSMRDLiteralOffset() const {
2544   // 32-bit literals are only supported on CI and we only want to use them
2545   // when the offset is > 8-bits.
2546   return isImm() && !isUInt<8>(getImm()) && isUInt<32>(getImm());
2547 }
2548 
2549 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultSMRDOffset() const {
2550   return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTyOffset);
2551 }
2552 
2553 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultSMRDLiteralOffset() const {
2554   return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTyOffset);
2555 }
2556 
2557 //===----------------------------------------------------------------------===//
2558 // vop3
2559 //===----------------------------------------------------------------------===//
2560 
2561 static bool ConvertOmodMul(int64_t &Mul) {
2562   if (Mul != 1 && Mul != 2 && Mul != 4)
2563     return false;
2564 
2565   Mul >>= 1;
2566   return true;
2567 }
2568 
2569 static bool ConvertOmodDiv(int64_t &Div) {
2570   if (Div == 1) {
2571     Div = 0;
2572     return true;
2573   }
2574 
2575   if (Div == 2) {
2576     Div = 3;
2577     return true;
2578   }
2579 
2580   return false;
2581 }
2582 
2583 static bool ConvertBoundCtrl(int64_t &BoundCtrl) {
2584   if (BoundCtrl == 0) {
2585     BoundCtrl = 1;
2586     return true;
2587   } else if (BoundCtrl == -1) {
2588     BoundCtrl = 0;
2589     return true;
2590   }
2591   return false;
2592 }
2593 
2594 // Note: the order in this table matches the order of operands in AsmString.
2595 static const OptionalOperand AMDGPUOptionalOperandTable[] = {
2596   {"offen",   AMDGPUOperand::ImmTyOffen, true, nullptr},
2597   {"idxen",   AMDGPUOperand::ImmTyIdxen, true, nullptr},
2598   {"addr64",  AMDGPUOperand::ImmTyAddr64, true, nullptr},
2599   {"offset0", AMDGPUOperand::ImmTyOffset0, false, nullptr},
2600   {"offset1", AMDGPUOperand::ImmTyOffset1, false, nullptr},
2601   {"gds",     AMDGPUOperand::ImmTyGDS, true, nullptr},
2602   {"offset",  AMDGPUOperand::ImmTyOffset, false, nullptr},
2603   {"glc",     AMDGPUOperand::ImmTyGLC, true, nullptr},
2604   {"slc",     AMDGPUOperand::ImmTySLC, true, nullptr},
2605   {"tfe",     AMDGPUOperand::ImmTyTFE, true, nullptr},
2606   {"clamp",   AMDGPUOperand::ImmTyClampSI, true, nullptr},
2607   {"omod",    AMDGPUOperand::ImmTyOModSI, false, ConvertOmodMul},
2608   {"unorm",   AMDGPUOperand::ImmTyUNorm, true, nullptr},
2609   {"da",      AMDGPUOperand::ImmTyDA,    true, nullptr},
2610   {"r128",    AMDGPUOperand::ImmTyR128,  true, nullptr},
2611   {"lwe",     AMDGPUOperand::ImmTyLWE,   true, nullptr},
2612   {"dmask",   AMDGPUOperand::ImmTyDMask, false, nullptr},
2613   {"row_mask",   AMDGPUOperand::ImmTyDppRowMask, false, nullptr},
2614   {"bank_mask",  AMDGPUOperand::ImmTyDppBankMask, false, nullptr},
2615   {"bound_ctrl", AMDGPUOperand::ImmTyDppBoundCtrl, false, ConvertBoundCtrl},
2616   {"dst_sel",    AMDGPUOperand::ImmTySdwaDstSel, false, nullptr},
2617   {"src0_sel",   AMDGPUOperand::ImmTySdwaSrc0Sel, false, nullptr},
2618   {"src1_sel",   AMDGPUOperand::ImmTySdwaSrc1Sel, false, nullptr},
2619   {"dst_unused", AMDGPUOperand::ImmTySdwaDstUnused, false, nullptr},
2620 };
2621 
2622 AMDGPUAsmParser::OperandMatchResultTy AMDGPUAsmParser::parseOptionalOperand(OperandVector &Operands) {
2623   OperandMatchResultTy res;
2624   for (const OptionalOperand &Op : AMDGPUOptionalOperandTable) {
2625     // try to parse any optional operand here
2626     if (Op.IsBit) {
2627       res = parseNamedBit(Op.Name, Operands, Op.Type);
2628     } else if (Op.Type == AMDGPUOperand::ImmTyOModSI) {
2629       res = parseOModOperand(Operands);
2630     } else if (Op.Type == AMDGPUOperand::ImmTySdwaDstSel ||
2631                Op.Type == AMDGPUOperand::ImmTySdwaSrc0Sel ||
2632                Op.Type == AMDGPUOperand::ImmTySdwaSrc1Sel) {
2633       res = parseSDWASel(Operands, Op.Name, Op.Type);
2634     } else if (Op.Type == AMDGPUOperand::ImmTySdwaDstUnused) {
2635       res = parseSDWADstUnused(Operands);
2636     } else {
2637       res = parseIntWithPrefix(Op.Name, Operands, Op.Type, Op.ConvertResult);
2638     }
2639     if (res != MatchOperand_NoMatch) {
2640       return res;
2641     }
2642   }
2643   return MatchOperand_NoMatch;
2644 }
2645 
2646 AMDGPUAsmParser::OperandMatchResultTy AMDGPUAsmParser::parseOModOperand(OperandVector &Operands)
2647 {
2648   StringRef Name = Parser.getTok().getString();
2649   if (Name == "mul") {
2650     return parseIntWithPrefix("mul", Operands, AMDGPUOperand::ImmTyOModSI, ConvertOmodMul);
2651   } else if (Name == "div") {
2652     return parseIntWithPrefix("div", Operands, AMDGPUOperand::ImmTyOModSI, ConvertOmodDiv);
2653   } else {
2654     return MatchOperand_NoMatch;
2655   }
2656 }
2657 
2658 void AMDGPUAsmParser::cvtId(MCInst &Inst, const OperandVector &Operands) {
2659   unsigned I = 1;
2660   const MCInstrDesc &Desc = MII.get(Inst.getOpcode());
2661   for (unsigned J = 0; J < Desc.getNumDefs(); ++J) {
2662     ((AMDGPUOperand &)*Operands[I++]).addRegOperands(Inst, 1);
2663   }
2664   for (unsigned E = Operands.size(); I != E; ++I)
2665     ((AMDGPUOperand &)*Operands[I]).addRegOrImmOperands(Inst, 1);
2666 }
2667 
2668 void AMDGPUAsmParser::cvtVOP3_2_mod(MCInst &Inst, const OperandVector &Operands) {
2669   uint64_t TSFlags = MII.get(Inst.getOpcode()).TSFlags;
2670   if (TSFlags & SIInstrFlags::VOP3) {
2671     cvtVOP3(Inst, Operands);
2672   } else {
2673     cvtId(Inst, Operands);
2674   }
2675 }
2676 
2677 static bool isRegOrImmWithInputMods(const MCInstrDesc &Desc, unsigned OpNum) {
2678       // 1. This operand is input modifiers
2679   return Desc.OpInfo[OpNum].OperandType == AMDGPU::OPERAND_INPUT_MODS
2680       // 2. This is not last operand
2681       && Desc.NumOperands > (OpNum + 1)
2682       // 3. Next operand is register class
2683       && Desc.OpInfo[OpNum + 1].RegClass != -1
2684       // 4. Next register is not tied to any other operand
2685       && Desc.getOperandConstraint(OpNum + 1, MCOI::OperandConstraint::TIED_TO) == -1;
2686 }
2687 
2688 void AMDGPUAsmParser::cvtVOP3(MCInst &Inst, const OperandVector &Operands) {
2689   OptionalImmIndexMap OptionalIdx;
2690   unsigned I = 1;
2691   const MCInstrDesc &Desc = MII.get(Inst.getOpcode());
2692   for (unsigned J = 0; J < Desc.getNumDefs(); ++J) {
2693     ((AMDGPUOperand &)*Operands[I++]).addRegOperands(Inst, 1);
2694   }
2695 
2696   for (unsigned E = Operands.size(); I != E; ++I) {
2697     AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[I]);
2698     if (isRegOrImmWithInputMods(Desc, Inst.getNumOperands())) {
2699       Op.addRegOrImmWithFPInputModsOperands(Inst, 2);
2700     } else if (Op.isImm()) {
2701       OptionalIdx[Op.getImmTy()] = I;
2702     } else {
2703       assert(false);
2704     }
2705   }
2706 
2707   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyClampSI);
2708   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyOModSI);
2709 
2710   // special case v_mac_f32:
2711   // it has src2 register operand that is tied to dst operand
2712   // we don't allow modifiers for this operand in assembler so src2_modifiers
2713   // should be 0
2714   if (Inst.getOpcode() == AMDGPU::V_MAC_F32_e64_si ||
2715       Inst.getOpcode() == AMDGPU::V_MAC_F32_e64_vi) {
2716     auto it = Inst.begin();
2717     std::advance(it, AMDGPU::getNamedOperandIdx(AMDGPU::V_MAC_F32_e64, AMDGPU::OpName::src2_modifiers));
2718     it = Inst.insert(it, MCOperand::createImm(0)); // no modifiers for src2
2719     ++it;
2720     Inst.insert(it, Inst.getOperand(0)); // src2 = dst
2721   }
2722 }
2723 
2724 //===----------------------------------------------------------------------===//
2725 // dpp
2726 //===----------------------------------------------------------------------===//
2727 
2728 bool AMDGPUOperand::isDPPCtrl() const {
2729   bool result = isImm() && getImmTy() == ImmTyDppCtrl && isUInt<9>(getImm());
2730   if (result) {
2731     int64_t Imm = getImm();
2732     return ((Imm >= 0x000) && (Imm <= 0x0ff)) ||
2733            ((Imm >= 0x101) && (Imm <= 0x10f)) ||
2734            ((Imm >= 0x111) && (Imm <= 0x11f)) ||
2735            ((Imm >= 0x121) && (Imm <= 0x12f)) ||
2736            (Imm == 0x130) ||
2737            (Imm == 0x134) ||
2738            (Imm == 0x138) ||
2739            (Imm == 0x13c) ||
2740            (Imm == 0x140) ||
2741            (Imm == 0x141) ||
2742            (Imm == 0x142) ||
2743            (Imm == 0x143);
2744   }
2745   return false;
2746 }
2747 
2748 bool AMDGPUOperand::isGPRIdxMode() const {
2749   return isImm() && isUInt<4>(getImm());
2750 }
2751 
2752 AMDGPUAsmParser::OperandMatchResultTy
2753 AMDGPUAsmParser::parseDPPCtrl(OperandVector &Operands) {
2754   SMLoc S = Parser.getTok().getLoc();
2755   StringRef Prefix;
2756   int64_t Int;
2757 
2758   if (getLexer().getKind() == AsmToken::Identifier) {
2759     Prefix = Parser.getTok().getString();
2760   } else {
2761     return MatchOperand_NoMatch;
2762   }
2763 
2764   if (Prefix == "row_mirror") {
2765     Int = 0x140;
2766     Parser.Lex();
2767   } else if (Prefix == "row_half_mirror") {
2768     Int = 0x141;
2769     Parser.Lex();
2770   } else {
2771     // Check to prevent parseDPPCtrlOps from eating invalid tokens
2772     if (Prefix != "quad_perm"
2773         && Prefix != "row_shl"
2774         && Prefix != "row_shr"
2775         && Prefix != "row_ror"
2776         && Prefix != "wave_shl"
2777         && Prefix != "wave_rol"
2778         && Prefix != "wave_shr"
2779         && Prefix != "wave_ror"
2780         && Prefix != "row_bcast") {
2781       return MatchOperand_NoMatch;
2782     }
2783 
2784     Parser.Lex();
2785     if (getLexer().isNot(AsmToken::Colon))
2786       return MatchOperand_ParseFail;
2787 
2788     if (Prefix == "quad_perm") {
2789       // quad_perm:[%d,%d,%d,%d]
2790       Parser.Lex();
2791       if (getLexer().isNot(AsmToken::LBrac))
2792         return MatchOperand_ParseFail;
2793       Parser.Lex();
2794 
2795       if (getParser().parseAbsoluteExpression(Int) || !(0 <= Int && Int <=3))
2796         return MatchOperand_ParseFail;
2797 
2798       for (int i = 0; i < 3; ++i) {
2799         if (getLexer().isNot(AsmToken::Comma))
2800           return MatchOperand_ParseFail;
2801         Parser.Lex();
2802 
2803         int64_t Temp;
2804         if (getParser().parseAbsoluteExpression(Temp) || !(0 <= Temp && Temp <=3))
2805           return MatchOperand_ParseFail;
2806         const int shift = i*2 + 2;
2807         Int += (Temp << shift);
2808       }
2809 
2810       if (getLexer().isNot(AsmToken::RBrac))
2811         return MatchOperand_ParseFail;
2812       Parser.Lex();
2813 
2814     } else {
2815       // sel:%d
2816       Parser.Lex();
2817       if (getParser().parseAbsoluteExpression(Int))
2818         return MatchOperand_ParseFail;
2819 
2820       if (Prefix == "row_shl" && 1 <= Int && Int <= 15) {
2821         Int |= 0x100;
2822       } else if (Prefix == "row_shr" && 1 <= Int && Int <= 15) {
2823         Int |= 0x110;
2824       } else if (Prefix == "row_ror" && 1 <= Int && Int <= 15) {
2825         Int |= 0x120;
2826       } else if (Prefix == "wave_shl" && 1 == Int) {
2827         Int = 0x130;
2828       } else if (Prefix == "wave_rol" && 1 == Int) {
2829         Int = 0x134;
2830       } else if (Prefix == "wave_shr" && 1 == Int) {
2831         Int = 0x138;
2832       } else if (Prefix == "wave_ror" && 1 == Int) {
2833         Int = 0x13C;
2834       } else if (Prefix == "row_bcast") {
2835         if (Int == 15) {
2836           Int = 0x142;
2837         } else if (Int == 31) {
2838           Int = 0x143;
2839         } else {
2840           return MatchOperand_ParseFail;
2841         }
2842       } else {
2843         return MatchOperand_ParseFail;
2844       }
2845     }
2846   }
2847 
2848   Operands.push_back(AMDGPUOperand::CreateImm(this, Int, S, AMDGPUOperand::ImmTyDppCtrl));
2849   return MatchOperand_Success;
2850 }
2851 
2852 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultRowMask() const {
2853   return AMDGPUOperand::CreateImm(this, 0xf, SMLoc(), AMDGPUOperand::ImmTyDppRowMask);
2854 }
2855 
2856 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultBankMask() const {
2857   return AMDGPUOperand::CreateImm(this, 0xf, SMLoc(), AMDGPUOperand::ImmTyDppBankMask);
2858 }
2859 
2860 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultBoundCtrl() const {
2861   return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTyDppBoundCtrl);
2862 }
2863 
2864 void AMDGPUAsmParser::cvtDPP(MCInst &Inst, const OperandVector &Operands) {
2865   OptionalImmIndexMap OptionalIdx;
2866 
2867   unsigned I = 1;
2868   const MCInstrDesc &Desc = MII.get(Inst.getOpcode());
2869   for (unsigned J = 0; J < Desc.getNumDefs(); ++J) {
2870     ((AMDGPUOperand &)*Operands[I++]).addRegOperands(Inst, 1);
2871   }
2872 
2873   for (unsigned E = Operands.size(); I != E; ++I) {
2874     AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[I]);
2875     // Add the register arguments
2876     if (isRegOrImmWithInputMods(Desc, Inst.getNumOperands())) {
2877       Op.addRegOrImmWithFPInputModsOperands(Inst, 2);
2878     } else if (Op.isDPPCtrl()) {
2879       Op.addImmOperands(Inst, 1);
2880     } else if (Op.isImm()) {
2881       // Handle optional arguments
2882       OptionalIdx[Op.getImmTy()] = I;
2883     } else {
2884       llvm_unreachable("Invalid operand type");
2885     }
2886   }
2887 
2888   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyDppRowMask, 0xf);
2889   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyDppBankMask, 0xf);
2890   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyDppBoundCtrl);
2891 
2892   // special case v_mac_f32:
2893   // it has src2 register operand that is tied to dst operand
2894   if (Inst.getOpcode() == AMDGPU::V_MAC_F32_dpp) {
2895     auto it = Inst.begin();
2896     std::advance(it, AMDGPU::getNamedOperandIdx(Inst.getOpcode(), AMDGPU::OpName::src2));
2897     Inst.insert(it, Inst.getOperand(0)); // src2 = dst
2898   }
2899 }
2900 
2901 //===----------------------------------------------------------------------===//
2902 // sdwa
2903 //===----------------------------------------------------------------------===//
2904 
2905 AMDGPUAsmParser::OperandMatchResultTy
2906 AMDGPUAsmParser::parseSDWASel(OperandVector &Operands, StringRef Prefix,
2907                               AMDGPUOperand::ImmTy Type) {
2908   using namespace llvm::AMDGPU::SDWA;
2909 
2910   SMLoc S = Parser.getTok().getLoc();
2911   StringRef Value;
2912   AMDGPUAsmParser::OperandMatchResultTy res;
2913 
2914   res = parseStringWithPrefix(Prefix, Value);
2915   if (res != MatchOperand_Success) {
2916     return res;
2917   }
2918 
2919   int64_t Int;
2920   Int = StringSwitch<int64_t>(Value)
2921         .Case("BYTE_0", SdwaSel::BYTE_0)
2922         .Case("BYTE_1", SdwaSel::BYTE_1)
2923         .Case("BYTE_2", SdwaSel::BYTE_2)
2924         .Case("BYTE_3", SdwaSel::BYTE_3)
2925         .Case("WORD_0", SdwaSel::WORD_0)
2926         .Case("WORD_1", SdwaSel::WORD_1)
2927         .Case("DWORD", SdwaSel::DWORD)
2928         .Default(0xffffffff);
2929   Parser.Lex(); // eat last token
2930 
2931   if (Int == 0xffffffff) {
2932     return MatchOperand_ParseFail;
2933   }
2934 
2935   Operands.push_back(AMDGPUOperand::CreateImm(this, Int, S, Type));
2936   return MatchOperand_Success;
2937 }
2938 
2939 AMDGPUAsmParser::OperandMatchResultTy
2940 AMDGPUAsmParser::parseSDWADstUnused(OperandVector &Operands) {
2941   using namespace llvm::AMDGPU::SDWA;
2942 
2943   SMLoc S = Parser.getTok().getLoc();
2944   StringRef Value;
2945   AMDGPUAsmParser::OperandMatchResultTy res;
2946 
2947   res = parseStringWithPrefix("dst_unused", Value);
2948   if (res != MatchOperand_Success) {
2949     return res;
2950   }
2951 
2952   int64_t Int;
2953   Int = StringSwitch<int64_t>(Value)
2954         .Case("UNUSED_PAD", DstUnused::UNUSED_PAD)
2955         .Case("UNUSED_SEXT", DstUnused::UNUSED_SEXT)
2956         .Case("UNUSED_PRESERVE", DstUnused::UNUSED_PRESERVE)
2957         .Default(0xffffffff);
2958   Parser.Lex(); // eat last token
2959 
2960   if (Int == 0xffffffff) {
2961     return MatchOperand_ParseFail;
2962   }
2963 
2964   Operands.push_back(AMDGPUOperand::CreateImm(this, Int, S, AMDGPUOperand::ImmTySdwaDstUnused));
2965   return MatchOperand_Success;
2966 }
2967 
2968 void AMDGPUAsmParser::cvtSdwaVOP1(MCInst &Inst, const OperandVector &Operands) {
2969   cvtSDWA(Inst, Operands, SIInstrFlags::VOP1);
2970 }
2971 
2972 void AMDGPUAsmParser::cvtSdwaVOP2(MCInst &Inst, const OperandVector &Operands) {
2973   cvtSDWA(Inst, Operands, SIInstrFlags::VOP2);
2974 }
2975 
2976 void AMDGPUAsmParser::cvtSdwaVOPC(MCInst &Inst, const OperandVector &Operands) {
2977   cvtSDWA(Inst, Operands, SIInstrFlags::VOPC);
2978 }
2979 
2980 void AMDGPUAsmParser::cvtSDWA(MCInst &Inst, const OperandVector &Operands,
2981                               uint64_t BasicInstType) {
2982   OptionalImmIndexMap OptionalIdx;
2983 
2984   unsigned I = 1;
2985   const MCInstrDesc &Desc = MII.get(Inst.getOpcode());
2986   for (unsigned J = 0; J < Desc.getNumDefs(); ++J) {
2987     ((AMDGPUOperand &)*Operands[I++]).addRegOperands(Inst, 1);
2988   }
2989 
2990   for (unsigned E = Operands.size(); I != E; ++I) {
2991     AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[I]);
2992     // Add the register arguments
2993     if (BasicInstType == SIInstrFlags::VOPC &&
2994         Op.isReg() &&
2995         Op.Reg.RegNo == AMDGPU::VCC) {
2996       // VOPC sdwa use "vcc" token as dst. Skip it.
2997       continue;
2998     } else if (isRegOrImmWithInputMods(Desc, Inst.getNumOperands())) {
2999       Op.addRegOrImmWithInputModsOperands(Inst, 2);
3000     } else if (Op.isImm()) {
3001       // Handle optional arguments
3002       OptionalIdx[Op.getImmTy()] = I;
3003     } else {
3004       llvm_unreachable("Invalid operand type");
3005     }
3006   }
3007 
3008   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyClampSI, 0);
3009 
3010   if (Inst.getOpcode() != AMDGPU::V_NOP_sdwa) {
3011     // V_NOP_sdwa has no optional sdwa arguments
3012     switch (BasicInstType) {
3013     case SIInstrFlags::VOP1: {
3014       addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySdwaDstSel, 6);
3015       addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySdwaDstUnused, 2);
3016       addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySdwaSrc0Sel, 6);
3017       break;
3018     }
3019     case SIInstrFlags::VOP2: {
3020       addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySdwaDstSel, 6);
3021       addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySdwaDstUnused, 2);
3022       addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySdwaSrc0Sel, 6);
3023       addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySdwaSrc1Sel, 6);
3024       break;
3025     }
3026     case SIInstrFlags::VOPC: {
3027       addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySdwaSrc0Sel, 6);
3028       addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySdwaSrc1Sel, 6);
3029       break;
3030     }
3031     default:
3032       llvm_unreachable("Invalid instruction type. Only VOP1, VOP2 and VOPC allowed");
3033     }
3034   }
3035 
3036   // special case v_mac_f32:
3037   // it has src2 register operand that is tied to dst operand
3038   if (Inst.getOpcode() == AMDGPU::V_MAC_F32_sdwa) {
3039     auto it = Inst.begin();
3040     std::advance(it, AMDGPU::getNamedOperandIdx(Inst.getOpcode(), AMDGPU::OpName::src2));
3041     Inst.insert(it, Inst.getOperand(0)); // src2 = dst
3042   }
3043 
3044 }
3045 
3046 /// Force static initialization.
3047 extern "C" void LLVMInitializeAMDGPUAsmParser() {
3048   RegisterMCAsmParser<AMDGPUAsmParser> A(getTheAMDGPUTarget());
3049   RegisterMCAsmParser<AMDGPUAsmParser> B(getTheGCNTarget());
3050 }
3051 
3052 #define GET_REGISTER_MATCHER
3053 #define GET_MATCHER_IMPLEMENTATION
3054 #include "AMDGPUGenAsmMatcher.inc"
3055 
3056 
3057 // This fuction should be defined after auto-generated include so that we have
3058 // MatchClassKind enum defined
3059 unsigned AMDGPUAsmParser::validateTargetOperandClass(MCParsedAsmOperand &Op,
3060                                                      unsigned Kind) {
3061   // Tokens like "glc" would be parsed as immediate operands in ParseOperand().
3062   // But MatchInstructionImpl() expects to meet token and fails to validate
3063   // operand. This method checks if we are given immediate operand but expect to
3064   // get corresponding token.
3065   AMDGPUOperand &Operand = (AMDGPUOperand&)Op;
3066   switch (Kind) {
3067   case MCK_addr64:
3068     return Operand.isAddr64() ? Match_Success : Match_InvalidOperand;
3069   case MCK_gds:
3070     return Operand.isGDS() ? Match_Success : Match_InvalidOperand;
3071   case MCK_glc:
3072     return Operand.isGLC() ? Match_Success : Match_InvalidOperand;
3073   case MCK_idxen:
3074     return Operand.isIdxen() ? Match_Success : Match_InvalidOperand;
3075   case MCK_offen:
3076     return Operand.isOffen() ? Match_Success : Match_InvalidOperand;
3077   case MCK_SSrcB32:
3078     // When operands have expression values, they will return true for isToken,
3079     // because it is not possible to distinguish between a token and an
3080     // expression at parse time. MatchInstructionImpl() will always try to
3081     // match an operand as a token, when isToken returns true, and when the
3082     // name of the expression is not a valid token, the match will fail,
3083     // so we need to handle it here.
3084     return Operand.isSSrcB32() ? Match_Success : Match_InvalidOperand;
3085   case MCK_SSrcF32:
3086     return Operand.isSSrcF32() ? Match_Success : Match_InvalidOperand;
3087   case MCK_SoppBrTarget:
3088     return Operand.isSoppBrTarget() ? Match_Success : Match_InvalidOperand;
3089   default: return Match_InvalidOperand;
3090   }
3091 }
3092