1 //===-- AMDGPUAsmParser.cpp - Parse SI asm to MCInst instructions ---------===//
2 //
3 //                     The LLVM Compiler Infrastructure
4 //
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
7 //
8 //===----------------------------------------------------------------------===//
9 
10 #include "AMDKernelCodeT.h"
11 #include "MCTargetDesc/AMDGPUMCTargetDesc.h"
12 #include "MCTargetDesc/AMDGPUTargetStreamer.h"
13 #include "SIDefines.h"
14 #include "Utils/AMDGPUBaseInfo.h"
15 #include "Utils/AMDKernelCodeTUtils.h"
16 #include "Utils/AMDGPUAsmUtils.h"
17 #include "llvm/ADT/APFloat.h"
18 #include "llvm/ADT/APInt.h"
19 #include "llvm/ADT/SmallBitVector.h"
20 #include "llvm/ADT/SmallString.h"
21 #include "llvm/ADT/STLExtras.h"
22 #include "llvm/ADT/StringRef.h"
23 #include "llvm/ADT/StringSwitch.h"
24 #include "llvm/ADT/Twine.h"
25 #include "llvm/CodeGen/MachineValueType.h"
26 #include "llvm/MC/MCAsmInfo.h"
27 #include "llvm/MC/MCContext.h"
28 #include "llvm/MC/MCExpr.h"
29 #include "llvm/MC/MCInst.h"
30 #include "llvm/MC/MCInstrDesc.h"
31 #include "llvm/MC/MCInstrInfo.h"
32 #include "llvm/MC/MCParser/MCAsmLexer.h"
33 #include "llvm/MC/MCParser/MCAsmParser.h"
34 #include "llvm/MC/MCParser/MCAsmParserExtension.h"
35 #include "llvm/MC/MCParser/MCParsedAsmOperand.h"
36 #include "llvm/MC/MCParser/MCTargetAsmParser.h"
37 #include "llvm/MC/MCRegisterInfo.h"
38 #include "llvm/MC/MCStreamer.h"
39 #include "llvm/MC/MCSubtargetInfo.h"
40 #include "llvm/MC/MCSymbol.h"
41 #include "llvm/Support/Casting.h"
42 #include "llvm/Support/Debug.h"
43 #include "llvm/Support/ELF.h"
44 #include "llvm/Support/ErrorHandling.h"
45 #include "llvm/Support/MathExtras.h"
46 #include "llvm/Support/raw_ostream.h"
47 #include "llvm/Support/SMLoc.h"
48 #include "llvm/Support/TargetRegistry.h"
49 #include "llvm/Support/raw_ostream.h"
50 #include "llvm/Support/MathExtras.h"
51 #include <algorithm>
52 #include <cassert>
53 #include <cstdint>
54 #include <cstring>
55 #include <iterator>
56 #include <map>
57 #include <memory>
58 #include <string>
59 #include <vector>
60 
61 using namespace llvm;
62 using namespace llvm::AMDGPU;
63 
64 namespace {
65 
66 class AMDGPUAsmParser;
67 
68 enum RegisterKind { IS_UNKNOWN, IS_VGPR, IS_SGPR, IS_TTMP, IS_SPECIAL };
69 
70 //===----------------------------------------------------------------------===//
71 // Operand
72 //===----------------------------------------------------------------------===//
73 
74 class AMDGPUOperand : public MCParsedAsmOperand {
75   enum KindTy {
76     Token,
77     Immediate,
78     Register,
79     Expression
80   } Kind;
81 
82   SMLoc StartLoc, EndLoc;
83   const AMDGPUAsmParser *AsmParser;
84 
85 public:
86   AMDGPUOperand(enum KindTy Kind_, const AMDGPUAsmParser *AsmParser_)
87     : MCParsedAsmOperand(), Kind(Kind_), AsmParser(AsmParser_) {}
88 
89   typedef std::unique_ptr<AMDGPUOperand> Ptr;
90 
91   struct Modifiers {
92     bool Abs = false;
93     bool Neg = false;
94     bool Sext = false;
95 
96     bool hasFPModifiers() const { return Abs || Neg; }
97     bool hasIntModifiers() const { return Sext; }
98     bool hasModifiers() const { return hasFPModifiers() || hasIntModifiers(); }
99 
100     int64_t getFPModifiersOperand() const {
101       int64_t Operand = 0;
102       Operand |= Abs ? SISrcMods::ABS : 0;
103       Operand |= Neg ? SISrcMods::NEG : 0;
104       return Operand;
105     }
106 
107     int64_t getIntModifiersOperand() const {
108       int64_t Operand = 0;
109       Operand |= Sext ? SISrcMods::SEXT : 0;
110       return Operand;
111     }
112 
113     int64_t getModifiersOperand() const {
114       assert(!(hasFPModifiers() && hasIntModifiers())
115            && "fp and int modifiers should not be used simultaneously");
116       if (hasFPModifiers()) {
117         return getFPModifiersOperand();
118       } else if (hasIntModifiers()) {
119         return getIntModifiersOperand();
120       } else {
121         return 0;
122       }
123     }
124 
125     friend raw_ostream &operator <<(raw_ostream &OS, AMDGPUOperand::Modifiers Mods);
126   };
127 
128   enum ImmTy {
129     ImmTyNone,
130     ImmTyGDS,
131     ImmTyOffen,
132     ImmTyIdxen,
133     ImmTyAddr64,
134     ImmTyOffset,
135     ImmTyOffset0,
136     ImmTyOffset1,
137     ImmTyGLC,
138     ImmTySLC,
139     ImmTyTFE,
140     ImmTyClampSI,
141     ImmTyOModSI,
142     ImmTyDppCtrl,
143     ImmTyDppRowMask,
144     ImmTyDppBankMask,
145     ImmTyDppBoundCtrl,
146     ImmTySdwaDstSel,
147     ImmTySdwaSrc0Sel,
148     ImmTySdwaSrc1Sel,
149     ImmTySdwaDstUnused,
150     ImmTyDMask,
151     ImmTyUNorm,
152     ImmTyDA,
153     ImmTyR128,
154     ImmTyLWE,
155     ImmTyExpTgt,
156     ImmTyExpCompr,
157     ImmTyExpVM,
158     ImmTyHwreg,
159     ImmTyOff,
160     ImmTySendMsg,
161     ImmTyInterpSlot,
162     ImmTyInterpAttr,
163     ImmTyAttrChan
164   };
165 
166   struct TokOp {
167     const char *Data;
168     unsigned Length;
169   };
170 
171   struct ImmOp {
172     int64_t Val;
173     ImmTy Type;
174     bool IsFPImm;
175     Modifiers Mods;
176   };
177 
178   struct RegOp {
179     unsigned RegNo;
180     bool IsForcedVOP3;
181     Modifiers Mods;
182   };
183 
184   union {
185     TokOp Tok;
186     ImmOp Imm;
187     RegOp Reg;
188     const MCExpr *Expr;
189   };
190 
191   bool isToken() const override {
192     if (Kind == Token)
193       return true;
194 
195     if (Kind != Expression || !Expr)
196       return false;
197 
198     // When parsing operands, we can't always tell if something was meant to be
199     // a token, like 'gds', or an expression that references a global variable.
200     // In this case, we assume the string is an expression, and if we need to
201     // interpret is a token, then we treat the symbol name as the token.
202     return isa<MCSymbolRefExpr>(Expr);
203   }
204 
205   bool isImm() const override {
206     return Kind == Immediate;
207   }
208 
209   bool isInlinableImm(MVT type) const;
210   bool isLiteralImm(MVT type) const;
211 
212   bool isRegKind() const {
213     return Kind == Register;
214   }
215 
216   bool isReg() const override {
217     return isRegKind() && !Reg.Mods.hasModifiers();
218   }
219 
220   bool isRegOrImmWithInputMods(MVT type) const {
221     return isRegKind() || isInlinableImm(type);
222   }
223 
224   bool isRegOrImmWithInt16InputMods() const {
225     return isRegOrImmWithInputMods(MVT::i16);
226   }
227 
228   bool isRegOrImmWithInt32InputMods() const {
229     return isRegOrImmWithInputMods(MVT::i32);
230   }
231 
232   bool isRegOrImmWithInt64InputMods() const {
233     return isRegOrImmWithInputMods(MVT::i64);
234   }
235 
236   bool isRegOrImmWithFP16InputMods() const {
237     return isRegOrImmWithInputMods(MVT::f16);
238   }
239 
240   bool isRegOrImmWithFP32InputMods() const {
241     return isRegOrImmWithInputMods(MVT::f32);
242   }
243 
244   bool isRegOrImmWithFP64InputMods() const {
245     return isRegOrImmWithInputMods(MVT::f64);
246   }
247 
248   bool isVReg32OrOff() const {
249     return isOff() || isRegClass(AMDGPU::VGPR_32RegClassID);
250   }
251 
252   bool isImmTy(ImmTy ImmT) const {
253     return isImm() && Imm.Type == ImmT;
254   }
255 
256   bool isImmModifier() const {
257     return isImm() && Imm.Type != ImmTyNone;
258   }
259 
260   bool isClampSI() const { return isImmTy(ImmTyClampSI); }
261   bool isOModSI() const { return isImmTy(ImmTyOModSI); }
262   bool isDMask() const { return isImmTy(ImmTyDMask); }
263   bool isUNorm() const { return isImmTy(ImmTyUNorm); }
264   bool isDA() const { return isImmTy(ImmTyDA); }
265   bool isR128() const { return isImmTy(ImmTyUNorm); }
266   bool isLWE() const { return isImmTy(ImmTyLWE); }
267   bool isOff() const { return isImmTy(ImmTyOff); }
268   bool isExpTgt() const { return isImmTy(ImmTyExpTgt); }
269   bool isExpVM() const { return isImmTy(ImmTyExpVM); }
270   bool isExpCompr() const { return isImmTy(ImmTyExpCompr); }
271   bool isOffen() const { return isImmTy(ImmTyOffen); }
272   bool isIdxen() const { return isImmTy(ImmTyIdxen); }
273   bool isAddr64() const { return isImmTy(ImmTyAddr64); }
274   bool isOffset() const { return isImmTy(ImmTyOffset) && isUInt<16>(getImm()); }
275   bool isOffset0() const { return isImmTy(ImmTyOffset0) && isUInt<16>(getImm()); }
276   bool isOffset1() const { return isImmTy(ImmTyOffset1) && isUInt<8>(getImm()); }
277   bool isGDS() const { return isImmTy(ImmTyGDS); }
278   bool isGLC() const { return isImmTy(ImmTyGLC); }
279   bool isSLC() const { return isImmTy(ImmTySLC); }
280   bool isTFE() const { return isImmTy(ImmTyTFE); }
281   bool isBankMask() const { return isImmTy(ImmTyDppBankMask); }
282   bool isRowMask() const { return isImmTy(ImmTyDppRowMask); }
283   bool isBoundCtrl() const { return isImmTy(ImmTyDppBoundCtrl); }
284   bool isSDWADstSel() const { return isImmTy(ImmTySdwaDstSel); }
285   bool isSDWASrc0Sel() const { return isImmTy(ImmTySdwaSrc0Sel); }
286   bool isSDWASrc1Sel() const { return isImmTy(ImmTySdwaSrc1Sel); }
287   bool isSDWADstUnused() const { return isImmTy(ImmTySdwaDstUnused); }
288   bool isInterpSlot() const { return isImmTy(ImmTyInterpSlot); }
289   bool isInterpAttr() const { return isImmTy(ImmTyInterpAttr); }
290   bool isAttrChan() const { return isImmTy(ImmTyAttrChan); }
291 
292   bool isMod() const {
293     return isClampSI() || isOModSI();
294   }
295 
296   bool isRegOrImm() const {
297     return isReg() || isImm();
298   }
299 
300   bool isRegClass(unsigned RCID) const;
301 
302   bool isSCSrcB16() const {
303     return isRegClass(AMDGPU::SReg_32RegClassID) || isInlinableImm(MVT::i16);
304   }
305 
306   bool isSCSrcB32() const {
307     return isRegClass(AMDGPU::SReg_32RegClassID) || isInlinableImm(MVT::i32);
308   }
309 
310   bool isSCSrcB64() const {
311     return isRegClass(AMDGPU::SReg_64RegClassID) || isInlinableImm(MVT::i64);
312   }
313 
314   bool isSCSrcF16() const {
315     return isRegClass(AMDGPU::SReg_32RegClassID) || isInlinableImm(MVT::f16);
316   }
317 
318   bool isSCSrcF32() const {
319     return isRegClass(AMDGPU::SReg_32RegClassID) || isInlinableImm(MVT::f32);
320   }
321 
322   bool isSCSrcF64() const {
323     return isRegClass(AMDGPU::SReg_64RegClassID) || isInlinableImm(MVT::f64);
324   }
325 
326   bool isSSrcB32() const {
327     return isSCSrcB32() || isLiteralImm(MVT::i32) || isExpr();
328   }
329 
330   bool isSSrcB16() const {
331     return isSCSrcB16() || isLiteralImm(MVT::i16);
332   }
333 
334   bool isSSrcB64() const {
335     // TODO: Find out how SALU supports extension of 32-bit literals to 64 bits.
336     // See isVSrc64().
337     return isSCSrcB64() || isLiteralImm(MVT::i64);
338   }
339 
340   bool isSSrcF32() const {
341     return isSCSrcB32() || isLiteralImm(MVT::f32) || isExpr();
342   }
343 
344   bool isSSrcF64() const {
345     return isSCSrcB64() || isLiteralImm(MVT::f64);
346   }
347 
348   bool isSSrcF16() const {
349     return isSCSrcB16() || isLiteralImm(MVT::f16);
350   }
351 
352   bool isVCSrcB32() const {
353     return isRegClass(AMDGPU::VS_32RegClassID) || isInlinableImm(MVT::i32);
354   }
355 
356   bool isVCSrcB64() const {
357     return isRegClass(AMDGPU::VS_64RegClassID) || isInlinableImm(MVT::i64);
358   }
359 
360   bool isVCSrcB16() const {
361     return isRegClass(AMDGPU::VS_32RegClassID) || isInlinableImm(MVT::i16);
362   }
363 
364   bool isVCSrcF32() const {
365     return isRegClass(AMDGPU::VS_32RegClassID) || isInlinableImm(MVT::f32);
366   }
367 
368   bool isVCSrcF64() const {
369     return isRegClass(AMDGPU::VS_64RegClassID) || isInlinableImm(MVT::f64);
370   }
371 
372   bool isVCSrcF16() const {
373     return isRegClass(AMDGPU::VS_32RegClassID) || isInlinableImm(MVT::f16);
374   }
375 
376   bool isVSrcB32() const {
377     return isVCSrcF32() || isLiteralImm(MVT::i32);
378   }
379 
380   bool isVSrcB64() const {
381     return isVCSrcF64() || isLiteralImm(MVT::i64);
382   }
383 
384   bool isVSrcB16() const {
385     return isVCSrcF16() || isLiteralImm(MVT::i16);
386   }
387 
388   bool isVSrcF32() const {
389     return isVCSrcF32() || isLiteralImm(MVT::f32);
390   }
391 
392   bool isVSrcF64() const {
393     return isVCSrcF64() || isLiteralImm(MVT::f64);
394   }
395 
396   bool isVSrcF16() const {
397     return isVCSrcF16() || isLiteralImm(MVT::f16);
398   }
399 
400   bool isKImmFP32() const {
401     return isLiteralImm(MVT::f32);
402   }
403 
404   bool isKImmFP16() const {
405     return isLiteralImm(MVT::f16);
406   }
407 
408   bool isMem() const override {
409     return false;
410   }
411 
412   bool isExpr() const {
413     return Kind == Expression;
414   }
415 
416   bool isSoppBrTarget() const {
417     return isExpr() || isImm();
418   }
419 
420   bool isSWaitCnt() const;
421   bool isHwreg() const;
422   bool isSendMsg() const;
423   bool isSMRDOffset8() const;
424   bool isSMRDOffset20() const;
425   bool isSMRDLiteralOffset() const;
426   bool isDPPCtrl() const;
427   bool isGPRIdxMode() const;
428 
429   StringRef getExpressionAsToken() const {
430     assert(isExpr());
431     const MCSymbolRefExpr *S = cast<MCSymbolRefExpr>(Expr);
432     return S->getSymbol().getName();
433   }
434 
435   StringRef getToken() const {
436     assert(isToken());
437 
438     if (Kind == Expression)
439       return getExpressionAsToken();
440 
441     return StringRef(Tok.Data, Tok.Length);
442   }
443 
444   int64_t getImm() const {
445     assert(isImm());
446     return Imm.Val;
447   }
448 
449   enum ImmTy getImmTy() const {
450     assert(isImm());
451     return Imm.Type;
452   }
453 
454   unsigned getReg() const override {
455     return Reg.RegNo;
456   }
457 
458   SMLoc getStartLoc() const override {
459     return StartLoc;
460   }
461 
462   SMLoc getEndLoc() const override {
463     return EndLoc;
464   }
465 
466   Modifiers getModifiers() const {
467     assert(isRegKind() || isImmTy(ImmTyNone));
468     return isRegKind() ? Reg.Mods : Imm.Mods;
469   }
470 
471   void setModifiers(Modifiers Mods) {
472     assert(isRegKind() || isImmTy(ImmTyNone));
473     if (isRegKind())
474       Reg.Mods = Mods;
475     else
476       Imm.Mods = Mods;
477   }
478 
479   bool hasModifiers() const {
480     return getModifiers().hasModifiers();
481   }
482 
483   bool hasFPModifiers() const {
484     return getModifiers().hasFPModifiers();
485   }
486 
487   bool hasIntModifiers() const {
488     return getModifiers().hasIntModifiers();
489   }
490 
491   void addImmOperands(MCInst &Inst, unsigned N, bool ApplyModifiers = true) const;
492 
493   void addLiteralImmOperand(MCInst &Inst, int64_t Val) const;
494 
495   template <unsigned Bitwidth>
496   void addKImmFPOperands(MCInst &Inst, unsigned N) const;
497 
498   void addKImmFP16Operands(MCInst &Inst, unsigned N) const {
499     addKImmFPOperands<16>(Inst, N);
500   }
501 
502   void addKImmFP32Operands(MCInst &Inst, unsigned N) const {
503     addKImmFPOperands<32>(Inst, N);
504   }
505 
506   void addRegOperands(MCInst &Inst, unsigned N) const;
507 
508   void addRegOrImmOperands(MCInst &Inst, unsigned N) const {
509     if (isRegKind())
510       addRegOperands(Inst, N);
511     else if (isExpr())
512       Inst.addOperand(MCOperand::createExpr(Expr));
513     else
514       addImmOperands(Inst, N);
515   }
516 
517   void addRegOrImmWithInputModsOperands(MCInst &Inst, unsigned N) const {
518     Modifiers Mods = getModifiers();
519     Inst.addOperand(MCOperand::createImm(Mods.getModifiersOperand()));
520     if (isRegKind()) {
521       addRegOperands(Inst, N);
522     } else {
523       addImmOperands(Inst, N, false);
524     }
525   }
526 
527   void addRegOrImmWithFPInputModsOperands(MCInst &Inst, unsigned N) const {
528     assert(!hasIntModifiers());
529     addRegOrImmWithInputModsOperands(Inst, N);
530   }
531 
532   void addRegOrImmWithIntInputModsOperands(MCInst &Inst, unsigned N) const {
533     assert(!hasFPModifiers());
534     addRegOrImmWithInputModsOperands(Inst, N);
535   }
536 
537   void addSoppBrTargetOperands(MCInst &Inst, unsigned N) const {
538     if (isImm())
539       addImmOperands(Inst, N);
540     else {
541       assert(isExpr());
542       Inst.addOperand(MCOperand::createExpr(Expr));
543     }
544   }
545 
546   static void printImmTy(raw_ostream& OS, ImmTy Type) {
547     switch (Type) {
548     case ImmTyNone: OS << "None"; break;
549     case ImmTyGDS: OS << "GDS"; break;
550     case ImmTyOffen: OS << "Offen"; break;
551     case ImmTyIdxen: OS << "Idxen"; break;
552     case ImmTyAddr64: OS << "Addr64"; break;
553     case ImmTyOffset: OS << "Offset"; break;
554     case ImmTyOffset0: OS << "Offset0"; break;
555     case ImmTyOffset1: OS << "Offset1"; break;
556     case ImmTyGLC: OS << "GLC"; break;
557     case ImmTySLC: OS << "SLC"; break;
558     case ImmTyTFE: OS << "TFE"; break;
559     case ImmTyClampSI: OS << "ClampSI"; break;
560     case ImmTyOModSI: OS << "OModSI"; break;
561     case ImmTyDppCtrl: OS << "DppCtrl"; break;
562     case ImmTyDppRowMask: OS << "DppRowMask"; break;
563     case ImmTyDppBankMask: OS << "DppBankMask"; break;
564     case ImmTyDppBoundCtrl: OS << "DppBoundCtrl"; break;
565     case ImmTySdwaDstSel: OS << "SdwaDstSel"; break;
566     case ImmTySdwaSrc0Sel: OS << "SdwaSrc0Sel"; break;
567     case ImmTySdwaSrc1Sel: OS << "SdwaSrc1Sel"; break;
568     case ImmTySdwaDstUnused: OS << "SdwaDstUnused"; break;
569     case ImmTyDMask: OS << "DMask"; break;
570     case ImmTyUNorm: OS << "UNorm"; break;
571     case ImmTyDA: OS << "DA"; break;
572     case ImmTyR128: OS << "R128"; break;
573     case ImmTyLWE: OS << "LWE"; break;
574     case ImmTyOff: OS << "Off"; break;
575     case ImmTyExpTgt: OS << "ExpTgt"; break;
576     case ImmTyExpCompr: OS << "ExpCompr"; break;
577     case ImmTyExpVM: OS << "ExpVM"; break;
578     case ImmTyHwreg: OS << "Hwreg"; break;
579     case ImmTySendMsg: OS << "SendMsg"; break;
580     case ImmTyInterpSlot: OS << "InterpSlot"; break;
581     case ImmTyInterpAttr: OS << "InterpAttr"; break;
582     case ImmTyAttrChan: OS << "AttrChan"; break;
583     }
584   }
585 
586   void print(raw_ostream &OS) const override {
587     switch (Kind) {
588     case Register:
589       OS << "<register " << getReg() << " mods: " << Reg.Mods << '>';
590       break;
591     case Immediate:
592       OS << '<' << getImm();
593       if (getImmTy() != ImmTyNone) {
594         OS << " type: "; printImmTy(OS, getImmTy());
595       }
596       OS << " mods: " << Imm.Mods << '>';
597       break;
598     case Token:
599       OS << '\'' << getToken() << '\'';
600       break;
601     case Expression:
602       OS << "<expr " << *Expr << '>';
603       break;
604     }
605   }
606 
607   static AMDGPUOperand::Ptr CreateImm(const AMDGPUAsmParser *AsmParser,
608                                       int64_t Val, SMLoc Loc,
609                                       enum ImmTy Type = ImmTyNone,
610                                       bool IsFPImm = false) {
611     auto Op = llvm::make_unique<AMDGPUOperand>(Immediate, AsmParser);
612     Op->Imm.Val = Val;
613     Op->Imm.IsFPImm = IsFPImm;
614     Op->Imm.Type = Type;
615     Op->Imm.Mods = Modifiers();
616     Op->StartLoc = Loc;
617     Op->EndLoc = Loc;
618     return Op;
619   }
620 
621   static AMDGPUOperand::Ptr CreateToken(const AMDGPUAsmParser *AsmParser,
622                                         StringRef Str, SMLoc Loc,
623                                         bool HasExplicitEncodingSize = true) {
624     auto Res = llvm::make_unique<AMDGPUOperand>(Token, AsmParser);
625     Res->Tok.Data = Str.data();
626     Res->Tok.Length = Str.size();
627     Res->StartLoc = Loc;
628     Res->EndLoc = Loc;
629     return Res;
630   }
631 
632   static AMDGPUOperand::Ptr CreateReg(const AMDGPUAsmParser *AsmParser,
633                                       unsigned RegNo, SMLoc S,
634                                       SMLoc E,
635                                       bool ForceVOP3) {
636     auto Op = llvm::make_unique<AMDGPUOperand>(Register, AsmParser);
637     Op->Reg.RegNo = RegNo;
638     Op->Reg.Mods = Modifiers();
639     Op->Reg.IsForcedVOP3 = ForceVOP3;
640     Op->StartLoc = S;
641     Op->EndLoc = E;
642     return Op;
643   }
644 
645   static AMDGPUOperand::Ptr CreateExpr(const AMDGPUAsmParser *AsmParser,
646                                        const class MCExpr *Expr, SMLoc S) {
647     auto Op = llvm::make_unique<AMDGPUOperand>(Expression, AsmParser);
648     Op->Expr = Expr;
649     Op->StartLoc = S;
650     Op->EndLoc = S;
651     return Op;
652   }
653 };
654 
655 raw_ostream &operator <<(raw_ostream &OS, AMDGPUOperand::Modifiers Mods) {
656   OS << "abs:" << Mods.Abs << " neg: " << Mods.Neg << " sext:" << Mods.Sext;
657   return OS;
658 }
659 
660 //===----------------------------------------------------------------------===//
661 // AsmParser
662 //===----------------------------------------------------------------------===//
663 
664 class AMDGPUAsmParser : public MCTargetAsmParser {
665   const MCInstrInfo &MII;
666   MCAsmParser &Parser;
667 
668   unsigned ForcedEncodingSize;
669   bool ForcedDPP;
670   bool ForcedSDWA;
671 
672   /// @name Auto-generated Match Functions
673   /// {
674 
675 #define GET_ASSEMBLER_HEADER
676 #include "AMDGPUGenAsmMatcher.inc"
677 
678   /// }
679 
680 private:
681   bool ParseDirectiveMajorMinor(uint32_t &Major, uint32_t &Minor);
682   bool ParseDirectiveHSACodeObjectVersion();
683   bool ParseDirectiveHSACodeObjectISA();
684   bool ParseDirectiveRuntimeMetadata();
685   bool ParseAMDKernelCodeTValue(StringRef ID, amd_kernel_code_t &Header);
686   bool ParseDirectiveAMDKernelCodeT();
687   bool ParseSectionDirectiveHSAText();
688   bool subtargetHasRegister(const MCRegisterInfo &MRI, unsigned RegNo) const;
689   bool ParseDirectiveAMDGPUHsaKernel();
690   bool ParseDirectiveAMDGPUHsaModuleGlobal();
691   bool ParseDirectiveAMDGPUHsaProgramGlobal();
692   bool ParseSectionDirectiveHSADataGlobalAgent();
693   bool ParseSectionDirectiveHSADataGlobalProgram();
694   bool ParseSectionDirectiveHSARodataReadonlyAgent();
695   bool AddNextRegisterToList(unsigned& Reg, unsigned& RegWidth, RegisterKind RegKind, unsigned Reg1, unsigned RegNum);
696   bool ParseAMDGPURegister(RegisterKind& RegKind, unsigned& Reg, unsigned& RegNum, unsigned& RegWidth);
697   void cvtMubufImpl(MCInst &Inst, const OperandVector &Operands, bool IsAtomic, bool IsAtomicReturn);
698 
699 public:
700   enum AMDGPUMatchResultTy {
701     Match_PreferE32 = FIRST_TARGET_MATCH_RESULT_TY
702   };
703 
704   AMDGPUAsmParser(const MCSubtargetInfo &STI, MCAsmParser &_Parser,
705                const MCInstrInfo &MII,
706                const MCTargetOptions &Options)
707       : MCTargetAsmParser(Options, STI), MII(MII), Parser(_Parser),
708         ForcedEncodingSize(0),
709         ForcedDPP(false),
710         ForcedSDWA(false) {
711     MCAsmParserExtension::Initialize(Parser);
712 
713     if (getSTI().getFeatureBits().none()) {
714       // Set default features.
715       copySTI().ToggleFeature("SOUTHERN_ISLANDS");
716     }
717 
718     setAvailableFeatures(ComputeAvailableFeatures(getSTI().getFeatureBits()));
719 
720     {
721       // TODO: make those pre-defined variables read-only.
722       // Currently there is none suitable machinery in the core llvm-mc for this.
723       // MCSymbol::isRedefinable is intended for another purpose, and
724       // AsmParser::parseDirectiveSet() cannot be specialized for specific target.
725       AMDGPU::IsaVersion Isa = AMDGPU::getIsaVersion(getSTI().getFeatureBits());
726       MCContext &Ctx = getContext();
727       MCSymbol *Sym = Ctx.getOrCreateSymbol(Twine(".option.machine_version_major"));
728       Sym->setVariableValue(MCConstantExpr::create(Isa.Major, Ctx));
729       Sym = Ctx.getOrCreateSymbol(Twine(".option.machine_version_minor"));
730       Sym->setVariableValue(MCConstantExpr::create(Isa.Minor, Ctx));
731       Sym = Ctx.getOrCreateSymbol(Twine(".option.machine_version_stepping"));
732       Sym->setVariableValue(MCConstantExpr::create(Isa.Stepping, Ctx));
733     }
734   }
735 
736   bool isSI() const {
737     return AMDGPU::isSI(getSTI());
738   }
739 
740   bool isCI() const {
741     return AMDGPU::isCI(getSTI());
742   }
743 
744   bool isVI() const {
745     return AMDGPU::isVI(getSTI());
746   }
747 
748   bool hasInv2PiInlineImm() const {
749     return getSTI().getFeatureBits()[AMDGPU::FeatureInv2PiInlineImm];
750   }
751 
752   bool hasSGPR102_SGPR103() const {
753     return !isVI();
754   }
755 
756   AMDGPUTargetStreamer &getTargetStreamer() {
757     MCTargetStreamer &TS = *getParser().getStreamer().getTargetStreamer();
758     return static_cast<AMDGPUTargetStreamer &>(TS);
759   }
760 
761   const MCRegisterInfo *getMRI() const {
762     // We need this const_cast because for some reason getContext() is not const
763     // in MCAsmParser.
764     return const_cast<AMDGPUAsmParser*>(this)->getContext().getRegisterInfo();
765   }
766 
767   const MCInstrInfo *getMII() const {
768     return &MII;
769   }
770 
771   void setForcedEncodingSize(unsigned Size) { ForcedEncodingSize = Size; }
772   void setForcedDPP(bool ForceDPP_) { ForcedDPP = ForceDPP_; }
773   void setForcedSDWA(bool ForceSDWA_) { ForcedSDWA = ForceSDWA_; }
774 
775   unsigned getForcedEncodingSize() const { return ForcedEncodingSize; }
776   bool isForcedVOP3() const { return ForcedEncodingSize == 64; }
777   bool isForcedDPP() const { return ForcedDPP; }
778   bool isForcedSDWA() const { return ForcedSDWA; }
779 
780   std::unique_ptr<AMDGPUOperand> parseRegister();
781   bool ParseRegister(unsigned &RegNo, SMLoc &StartLoc, SMLoc &EndLoc) override;
782   unsigned checkTargetMatchPredicate(MCInst &Inst) override;
783   unsigned validateTargetOperandClass(MCParsedAsmOperand &Op,
784                                       unsigned Kind) override;
785   bool MatchAndEmitInstruction(SMLoc IDLoc, unsigned &Opcode,
786                                OperandVector &Operands, MCStreamer &Out,
787                                uint64_t &ErrorInfo,
788                                bool MatchingInlineAsm) override;
789   bool ParseDirective(AsmToken DirectiveID) override;
790   OperandMatchResultTy parseOperand(OperandVector &Operands, StringRef Mnemonic);
791   StringRef parseMnemonicSuffix(StringRef Name);
792   bool ParseInstruction(ParseInstructionInfo &Info, StringRef Name,
793                         SMLoc NameLoc, OperandVector &Operands) override;
794   //bool ProcessInstruction(MCInst &Inst);
795 
796   OperandMatchResultTy parseIntWithPrefix(const char *Prefix, int64_t &Int);
797   OperandMatchResultTy
798   parseIntWithPrefix(const char *Prefix, OperandVector &Operands,
799                      enum AMDGPUOperand::ImmTy ImmTy = AMDGPUOperand::ImmTyNone,
800                      bool (*ConvertResult)(int64_t &) = nullptr);
801   OperandMatchResultTy
802   parseNamedBit(const char *Name, OperandVector &Operands,
803                 enum AMDGPUOperand::ImmTy ImmTy = AMDGPUOperand::ImmTyNone);
804   OperandMatchResultTy parseStringWithPrefix(StringRef Prefix,
805                                              StringRef &Value);
806 
807   OperandMatchResultTy parseImm(OperandVector &Operands);
808   OperandMatchResultTy parseRegOrImm(OperandVector &Operands);
809   OperandMatchResultTy parseRegOrImmWithFPInputMods(OperandVector &Operands);
810   OperandMatchResultTy parseRegOrImmWithIntInputMods(OperandVector &Operands);
811   OperandMatchResultTy parseVReg32OrOff(OperandVector &Operands);
812 
813   void cvtDSOffset01(MCInst &Inst, const OperandVector &Operands);
814   void cvtDS(MCInst &Inst, const OperandVector &Operands);
815   void cvtExp(MCInst &Inst, const OperandVector &Operands);
816 
817   bool parseCnt(int64_t &IntVal);
818   OperandMatchResultTy parseSWaitCntOps(OperandVector &Operands);
819   OperandMatchResultTy parseHwreg(OperandVector &Operands);
820 
821 private:
822   struct OperandInfoTy {
823     int64_t Id;
824     bool IsSymbolic;
825     OperandInfoTy(int64_t Id_) : Id(Id_), IsSymbolic(false) { }
826   };
827 
828   bool parseSendMsgConstruct(OperandInfoTy &Msg, OperandInfoTy &Operation, int64_t &StreamId);
829   bool parseHwregConstruct(OperandInfoTy &HwReg, int64_t &Offset, int64_t &Width);
830 
831   void errorExpTgt();
832   OperandMatchResultTy parseExpTgtImpl(StringRef Str, uint8_t &Val);
833 
834 public:
835   OperandMatchResultTy parseOptionalOperand(OperandVector &Operands);
836 
837   OperandMatchResultTy parseExpTgt(OperandVector &Operands);
838   OperandMatchResultTy parseSendMsgOp(OperandVector &Operands);
839   OperandMatchResultTy parseInterpSlot(OperandVector &Operands);
840   OperandMatchResultTy parseInterpAttr(OperandVector &Operands);
841   OperandMatchResultTy parseSOppBrTarget(OperandVector &Operands);
842 
843   void cvtMubuf(MCInst &Inst, const OperandVector &Operands) { cvtMubufImpl(Inst, Operands, false, false); }
844   void cvtMubufAtomic(MCInst &Inst, const OperandVector &Operands) { cvtMubufImpl(Inst, Operands, true, false); }
845   void cvtMubufAtomicReturn(MCInst &Inst, const OperandVector &Operands) { cvtMubufImpl(Inst, Operands, true, true); }
846   AMDGPUOperand::Ptr defaultGLC() const;
847   AMDGPUOperand::Ptr defaultSLC() const;
848   AMDGPUOperand::Ptr defaultTFE() const;
849 
850   AMDGPUOperand::Ptr defaultDMask() const;
851   AMDGPUOperand::Ptr defaultUNorm() const;
852   AMDGPUOperand::Ptr defaultDA() const;
853   AMDGPUOperand::Ptr defaultR128() const;
854   AMDGPUOperand::Ptr defaultLWE() const;
855   AMDGPUOperand::Ptr defaultSMRDOffset8() const;
856   AMDGPUOperand::Ptr defaultSMRDOffset20() const;
857   AMDGPUOperand::Ptr defaultSMRDLiteralOffset() const;
858   AMDGPUOperand::Ptr defaultExpTgt() const;
859   AMDGPUOperand::Ptr defaultExpCompr() const;
860   AMDGPUOperand::Ptr defaultExpVM() const;
861 
862   OperandMatchResultTy parseOModOperand(OperandVector &Operands);
863 
864   void cvtId(MCInst &Inst, const OperandVector &Operands);
865   void cvtVOP3_2_mod(MCInst &Inst, const OperandVector &Operands);
866   void cvtVOP3(MCInst &Inst, const OperandVector &Operands);
867 
868   void cvtMIMG(MCInst &Inst, const OperandVector &Operands);
869   void cvtMIMGAtomic(MCInst &Inst, const OperandVector &Operands);
870 
871   OperandMatchResultTy parseDPPCtrl(OperandVector &Operands);
872   AMDGPUOperand::Ptr defaultRowMask() const;
873   AMDGPUOperand::Ptr defaultBankMask() const;
874   AMDGPUOperand::Ptr defaultBoundCtrl() const;
875   void cvtDPP(MCInst &Inst, const OperandVector &Operands);
876 
877   OperandMatchResultTy parseSDWASel(OperandVector &Operands, StringRef Prefix,
878                                     AMDGPUOperand::ImmTy Type);
879   OperandMatchResultTy parseSDWADstUnused(OperandVector &Operands);
880   void cvtSdwaVOP1(MCInst &Inst, const OperandVector &Operands);
881   void cvtSdwaVOP2(MCInst &Inst, const OperandVector &Operands);
882   void cvtSdwaVOPC(MCInst &Inst, const OperandVector &Operands);
883   void cvtSDWA(MCInst &Inst, const OperandVector &Operands,
884                uint64_t BasicInstType);
885 };
886 
887 struct OptionalOperand {
888   const char *Name;
889   AMDGPUOperand::ImmTy Type;
890   bool IsBit;
891   bool (*ConvertResult)(int64_t&);
892 };
893 
894 } // end anonymous namespace
895 
896 // May be called with integer type with equivalent bitwidth.
897 static const fltSemantics *getFltSemantics(unsigned Size) {
898   switch (Size) {
899   case 4:
900     return &APFloat::IEEEsingle();
901   case 8:
902     return &APFloat::IEEEdouble();
903   case 2:
904     return &APFloat::IEEEhalf();
905   default:
906     llvm_unreachable("unsupported fp type");
907   }
908 }
909 
910 static const fltSemantics *getFltSemantics(MVT VT) {
911   return getFltSemantics(VT.getSizeInBits() / 8);
912 }
913 
914 //===----------------------------------------------------------------------===//
915 // Operand
916 //===----------------------------------------------------------------------===//
917 
918 static bool canLosslesslyConvertToFPType(APFloat &FPLiteral, MVT VT) {
919   bool Lost;
920 
921   // Convert literal to single precision
922   APFloat::opStatus Status = FPLiteral.convert(*getFltSemantics(VT),
923                                                APFloat::rmNearestTiesToEven,
924                                                &Lost);
925   // We allow precision lost but not overflow or underflow
926   if (Status != APFloat::opOK &&
927       Lost &&
928       ((Status & APFloat::opOverflow)  != 0 ||
929        (Status & APFloat::opUnderflow) != 0)) {
930     return false;
931   }
932 
933   return true;
934 }
935 
936 bool AMDGPUOperand::isInlinableImm(MVT type) const {
937   if (!isImmTy(ImmTyNone)) {
938     // Only plain immediates are inlinable (e.g. "clamp" attribute is not)
939     return false;
940   }
941   // TODO: We should avoid using host float here. It would be better to
942   // check the float bit values which is what a few other places do.
943   // We've had bot failures before due to weird NaN support on mips hosts.
944 
945   APInt Literal(64, Imm.Val);
946 
947   if (Imm.IsFPImm) { // We got fp literal token
948     if (type == MVT::f64 || type == MVT::i64) { // Expected 64-bit operand
949       return AMDGPU::isInlinableLiteral64(Imm.Val,
950                                           AsmParser->hasInv2PiInlineImm());
951     }
952 
953     APFloat FPLiteral(APFloat::IEEEdouble(), APInt(64, Imm.Val));
954     if (!canLosslesslyConvertToFPType(FPLiteral, type))
955       return false;
956 
957     // Check if single precision literal is inlinable
958     return AMDGPU::isInlinableLiteral32(
959       static_cast<int32_t>(FPLiteral.bitcastToAPInt().getZExtValue()),
960       AsmParser->hasInv2PiInlineImm());
961   }
962 
963 
964   // We got int literal token.
965   if (type == MVT::f64 || type == MVT::i64) { // Expected 64-bit operand
966     return AMDGPU::isInlinableLiteral64(Imm.Val,
967                                         AsmParser->hasInv2PiInlineImm());
968   }
969 
970   if (type.getScalarSizeInBits() == 16) {
971     return AMDGPU::isInlinableLiteral16(
972       static_cast<int16_t>(Literal.getLoBits(16).getSExtValue()),
973       AsmParser->hasInv2PiInlineImm());
974   }
975 
976   return AMDGPU::isInlinableLiteral32(
977     static_cast<int32_t>(Literal.getLoBits(32).getZExtValue()),
978     AsmParser->hasInv2PiInlineImm());
979 }
980 
981 bool AMDGPUOperand::isLiteralImm(MVT type) const {
982   // Check that this imediate can be added as literal
983   if (!isImmTy(ImmTyNone)) {
984     return false;
985   }
986 
987   if (!Imm.IsFPImm) {
988     // We got int literal token.
989 
990     unsigned Size = type.getSizeInBits();
991     if (Size == 64)
992       Size = 32;
993 
994     // FIXME: 64-bit operands can zero extend, sign extend, or pad zeroes for FP
995     // types.
996     return isUIntN(Size, Imm.Val) || isIntN(Size, Imm.Val);
997   }
998 
999   // We got fp literal token
1000   if (type == MVT::f64) { // Expected 64-bit fp operand
1001     // We would set low 64-bits of literal to zeroes but we accept this literals
1002     return true;
1003   }
1004 
1005   if (type == MVT::i64) { // Expected 64-bit int operand
1006     // We don't allow fp literals in 64-bit integer instructions. It is
1007     // unclear how we should encode them.
1008     return false;
1009   }
1010 
1011   APFloat FPLiteral(APFloat::IEEEdouble(), APInt(64, Imm.Val));
1012   return canLosslesslyConvertToFPType(FPLiteral, type);
1013 }
1014 
1015 bool AMDGPUOperand::isRegClass(unsigned RCID) const {
1016   return isReg() && AsmParser->getMRI()->getRegClass(RCID).contains(getReg());
1017 }
1018 
1019 void AMDGPUOperand::addImmOperands(MCInst &Inst, unsigned N, bool ApplyModifiers) const {
1020   int64_t Val = Imm.Val;
1021   if (isImmTy(ImmTyNone) && ApplyModifiers && Imm.Mods.hasFPModifiers() && Imm.Mods.Neg) {
1022     // Apply modifiers to immediate value. Only negate can get here
1023     if (Imm.IsFPImm) {
1024       APFloat F(BitsToDouble(Val));
1025       F.changeSign();
1026       Val = F.bitcastToAPInt().getZExtValue();
1027     } else {
1028       Val = -Val;
1029     }
1030   }
1031 
1032   if (AMDGPU::isSISrcOperand(AsmParser->getMII()->get(Inst.getOpcode()),
1033                              Inst.getNumOperands())) {
1034     addLiteralImmOperand(Inst, Val);
1035   } else {
1036     Inst.addOperand(MCOperand::createImm(Val));
1037   }
1038 }
1039 
1040 void AMDGPUOperand::addLiteralImmOperand(MCInst &Inst, int64_t Val) const {
1041   const auto& InstDesc = AsmParser->getMII()->get(Inst.getOpcode());
1042   auto OpNum = Inst.getNumOperands();
1043   // Check that this operand accepts literals
1044   assert(AMDGPU::isSISrcOperand(InstDesc, OpNum));
1045 
1046   auto OpSize = AMDGPU::getOperandSize(InstDesc, OpNum); // expected operand size
1047 
1048   if (Imm.IsFPImm) { // We got fp literal token
1049     APInt Literal(64, Val);
1050 
1051     switch (OpSize) {
1052     case 8: {
1053       if (AMDGPU::isInlinableLiteral64(Literal.getZExtValue(),
1054                                        AsmParser->hasInv2PiInlineImm())) {
1055         Inst.addOperand(MCOperand::createImm(Literal.getZExtValue()));
1056         return;
1057       }
1058 
1059       // Non-inlineable
1060       if (AMDGPU::isSISrcFPOperand(InstDesc, OpNum)) { // Expected 64-bit fp operand
1061         // For fp operands we check if low 32 bits are zeros
1062         if (Literal.getLoBits(32) != 0) {
1063           const_cast<AMDGPUAsmParser *>(AsmParser)->Warning(Inst.getLoc(),
1064           "Can't encode literal as exact 64-bit floating-point operand. "
1065           "Low 32-bits will be set to zero");
1066         }
1067 
1068         Inst.addOperand(MCOperand::createImm(Literal.lshr(32).getZExtValue()));
1069         return;
1070       }
1071 
1072       // We don't allow fp literals in 64-bit integer instructions. It is
1073       // unclear how we should encode them. This case should be checked earlier
1074       // in predicate methods (isLiteralImm())
1075       llvm_unreachable("fp literal in 64-bit integer instruction.");
1076     }
1077     case 4:
1078     case 2: {
1079       bool lost;
1080       APFloat FPLiteral(APFloat::IEEEdouble(), Literal);
1081       // Convert literal to single precision
1082       FPLiteral.convert(*getFltSemantics(OpSize),
1083                         APFloat::rmNearestTiesToEven, &lost);
1084       // We allow precision lost but not overflow or underflow. This should be
1085       // checked earlier in isLiteralImm()
1086       Inst.addOperand(MCOperand::createImm(FPLiteral.bitcastToAPInt().getZExtValue()));
1087       return;
1088     }
1089     default:
1090       llvm_unreachable("invalid operand size");
1091     }
1092 
1093     return;
1094   }
1095 
1096    // We got int literal token.
1097   // Only sign extend inline immediates.
1098   // FIXME: No errors on truncation
1099   switch (OpSize) {
1100   case 4: {
1101     if (isInt<32>(Val) &&
1102         AMDGPU::isInlinableLiteral32(static_cast<int32_t>(Val),
1103                                      AsmParser->hasInv2PiInlineImm())) {
1104       Inst.addOperand(MCOperand::createImm(Val));
1105       return;
1106     }
1107 
1108     Inst.addOperand(MCOperand::createImm(Val & 0xffffffff));
1109     return;
1110   }
1111   case 8: {
1112     if (AMDGPU::isInlinableLiteral64(Val,
1113                                      AsmParser->hasInv2PiInlineImm())) {
1114       Inst.addOperand(MCOperand::createImm(Val));
1115       return;
1116     }
1117 
1118     Inst.addOperand(MCOperand::createImm(Lo_32(Val)));
1119     return;
1120   }
1121   case 2: {
1122     if (isInt<16>(Val) &&
1123         AMDGPU::isInlinableLiteral16(static_cast<int16_t>(Val),
1124                                      AsmParser->hasInv2PiInlineImm())) {
1125       Inst.addOperand(MCOperand::createImm(Val));
1126       return;
1127     }
1128 
1129     Inst.addOperand(MCOperand::createImm(Val & 0xffff));
1130     return;
1131   }
1132   default:
1133     llvm_unreachable("invalid operand size");
1134   }
1135 }
1136 
1137 template <unsigned Bitwidth>
1138 void AMDGPUOperand::addKImmFPOperands(MCInst &Inst, unsigned N) const {
1139   APInt Literal(64, Imm.Val);
1140 
1141   if (!Imm.IsFPImm) {
1142     // We got int literal token.
1143     Inst.addOperand(MCOperand::createImm(Literal.getLoBits(Bitwidth).getZExtValue()));
1144     return;
1145   }
1146 
1147   bool Lost;
1148   APFloat FPLiteral(APFloat::IEEEdouble(), Literal);
1149   FPLiteral.convert(*getFltSemantics(Bitwidth / 8),
1150                     APFloat::rmNearestTiesToEven, &Lost);
1151   Inst.addOperand(MCOperand::createImm(FPLiteral.bitcastToAPInt().getZExtValue()));
1152 }
1153 
1154 void AMDGPUOperand::addRegOperands(MCInst &Inst, unsigned N) const {
1155   Inst.addOperand(MCOperand::createReg(AMDGPU::getMCReg(getReg(), AsmParser->getSTI())));
1156 }
1157 
1158 //===----------------------------------------------------------------------===//
1159 // AsmParser
1160 //===----------------------------------------------------------------------===//
1161 
1162 static int getRegClass(RegisterKind Is, unsigned RegWidth) {
1163   if (Is == IS_VGPR) {
1164     switch (RegWidth) {
1165       default: return -1;
1166       case 1: return AMDGPU::VGPR_32RegClassID;
1167       case 2: return AMDGPU::VReg_64RegClassID;
1168       case 3: return AMDGPU::VReg_96RegClassID;
1169       case 4: return AMDGPU::VReg_128RegClassID;
1170       case 8: return AMDGPU::VReg_256RegClassID;
1171       case 16: return AMDGPU::VReg_512RegClassID;
1172     }
1173   } else if (Is == IS_TTMP) {
1174     switch (RegWidth) {
1175       default: return -1;
1176       case 1: return AMDGPU::TTMP_32RegClassID;
1177       case 2: return AMDGPU::TTMP_64RegClassID;
1178       case 4: return AMDGPU::TTMP_128RegClassID;
1179     }
1180   } else if (Is == IS_SGPR) {
1181     switch (RegWidth) {
1182       default: return -1;
1183       case 1: return AMDGPU::SGPR_32RegClassID;
1184       case 2: return AMDGPU::SGPR_64RegClassID;
1185       case 4: return AMDGPU::SGPR_128RegClassID;
1186       case 8: return AMDGPU::SReg_256RegClassID;
1187       case 16: return AMDGPU::SReg_512RegClassID;
1188     }
1189   }
1190   return -1;
1191 }
1192 
1193 static unsigned getSpecialRegForName(StringRef RegName) {
1194   return StringSwitch<unsigned>(RegName)
1195     .Case("exec", AMDGPU::EXEC)
1196     .Case("vcc", AMDGPU::VCC)
1197     .Case("flat_scratch", AMDGPU::FLAT_SCR)
1198     .Case("m0", AMDGPU::M0)
1199     .Case("scc", AMDGPU::SCC)
1200     .Case("tba", AMDGPU::TBA)
1201     .Case("tma", AMDGPU::TMA)
1202     .Case("flat_scratch_lo", AMDGPU::FLAT_SCR_LO)
1203     .Case("flat_scratch_hi", AMDGPU::FLAT_SCR_HI)
1204     .Case("vcc_lo", AMDGPU::VCC_LO)
1205     .Case("vcc_hi", AMDGPU::VCC_HI)
1206     .Case("exec_lo", AMDGPU::EXEC_LO)
1207     .Case("exec_hi", AMDGPU::EXEC_HI)
1208     .Case("tma_lo", AMDGPU::TMA_LO)
1209     .Case("tma_hi", AMDGPU::TMA_HI)
1210     .Case("tba_lo", AMDGPU::TBA_LO)
1211     .Case("tba_hi", AMDGPU::TBA_HI)
1212     .Default(0);
1213 }
1214 
1215 bool AMDGPUAsmParser::ParseRegister(unsigned &RegNo, SMLoc &StartLoc, SMLoc &EndLoc) {
1216   auto R = parseRegister();
1217   if (!R) return true;
1218   assert(R->isReg());
1219   RegNo = R->getReg();
1220   StartLoc = R->getStartLoc();
1221   EndLoc = R->getEndLoc();
1222   return false;
1223 }
1224 
1225 bool AMDGPUAsmParser::AddNextRegisterToList(unsigned& Reg, unsigned& RegWidth, RegisterKind RegKind, unsigned Reg1, unsigned RegNum)
1226 {
1227   switch (RegKind) {
1228   case IS_SPECIAL:
1229     if (Reg == AMDGPU::EXEC_LO && Reg1 == AMDGPU::EXEC_HI) { Reg = AMDGPU::EXEC; RegWidth = 2; return true; }
1230     if (Reg == AMDGPU::FLAT_SCR_LO && Reg1 == AMDGPU::FLAT_SCR_HI) { Reg = AMDGPU::FLAT_SCR; RegWidth = 2; return true; }
1231     if (Reg == AMDGPU::VCC_LO && Reg1 == AMDGPU::VCC_HI) { Reg = AMDGPU::VCC; RegWidth = 2; return true; }
1232     if (Reg == AMDGPU::TBA_LO && Reg1 == AMDGPU::TBA_HI) { Reg = AMDGPU::TBA; RegWidth = 2; return true; }
1233     if (Reg == AMDGPU::TMA_LO && Reg1 == AMDGPU::TMA_HI) { Reg = AMDGPU::TMA; RegWidth = 2; return true; }
1234     return false;
1235   case IS_VGPR:
1236   case IS_SGPR:
1237   case IS_TTMP:
1238     if (Reg1 != Reg + RegWidth) { return false; }
1239     RegWidth++;
1240     return true;
1241   default:
1242     llvm_unreachable("unexpected register kind");
1243   }
1244 }
1245 
1246 bool AMDGPUAsmParser::ParseAMDGPURegister(RegisterKind& RegKind, unsigned& Reg, unsigned& RegNum, unsigned& RegWidth)
1247 {
1248   const MCRegisterInfo *TRI = getContext().getRegisterInfo();
1249   if (getLexer().is(AsmToken::Identifier)) {
1250     StringRef RegName = Parser.getTok().getString();
1251     if ((Reg = getSpecialRegForName(RegName))) {
1252       Parser.Lex();
1253       RegKind = IS_SPECIAL;
1254     } else {
1255       unsigned RegNumIndex = 0;
1256       if (RegName[0] == 'v') {
1257         RegNumIndex = 1;
1258         RegKind = IS_VGPR;
1259       } else if (RegName[0] == 's') {
1260         RegNumIndex = 1;
1261         RegKind = IS_SGPR;
1262       } else if (RegName.startswith("ttmp")) {
1263         RegNumIndex = strlen("ttmp");
1264         RegKind = IS_TTMP;
1265       } else {
1266         return false;
1267       }
1268       if (RegName.size() > RegNumIndex) {
1269         // Single 32-bit register: vXX.
1270         if (RegName.substr(RegNumIndex).getAsInteger(10, RegNum))
1271           return false;
1272         Parser.Lex();
1273         RegWidth = 1;
1274       } else {
1275         // Range of registers: v[XX:YY]. ":YY" is optional.
1276         Parser.Lex();
1277         int64_t RegLo, RegHi;
1278         if (getLexer().isNot(AsmToken::LBrac))
1279           return false;
1280         Parser.Lex();
1281 
1282         if (getParser().parseAbsoluteExpression(RegLo))
1283           return false;
1284 
1285         const bool isRBrace = getLexer().is(AsmToken::RBrac);
1286         if (!isRBrace && getLexer().isNot(AsmToken::Colon))
1287           return false;
1288         Parser.Lex();
1289 
1290         if (isRBrace) {
1291           RegHi = RegLo;
1292         } else {
1293           if (getParser().parseAbsoluteExpression(RegHi))
1294             return false;
1295 
1296           if (getLexer().isNot(AsmToken::RBrac))
1297             return false;
1298           Parser.Lex();
1299         }
1300         RegNum = (unsigned) RegLo;
1301         RegWidth = (RegHi - RegLo) + 1;
1302       }
1303     }
1304   } else if (getLexer().is(AsmToken::LBrac)) {
1305     // List of consecutive registers: [s0,s1,s2,s3]
1306     Parser.Lex();
1307     if (!ParseAMDGPURegister(RegKind, Reg, RegNum, RegWidth))
1308       return false;
1309     if (RegWidth != 1)
1310       return false;
1311     RegisterKind RegKind1;
1312     unsigned Reg1, RegNum1, RegWidth1;
1313     do {
1314       if (getLexer().is(AsmToken::Comma)) {
1315         Parser.Lex();
1316       } else if (getLexer().is(AsmToken::RBrac)) {
1317         Parser.Lex();
1318         break;
1319       } else if (ParseAMDGPURegister(RegKind1, Reg1, RegNum1, RegWidth1)) {
1320         if (RegWidth1 != 1) {
1321           return false;
1322         }
1323         if (RegKind1 != RegKind) {
1324           return false;
1325         }
1326         if (!AddNextRegisterToList(Reg, RegWidth, RegKind1, Reg1, RegNum1)) {
1327           return false;
1328         }
1329       } else {
1330         return false;
1331       }
1332     } while (true);
1333   } else {
1334     return false;
1335   }
1336   switch (RegKind) {
1337   case IS_SPECIAL:
1338     RegNum = 0;
1339     RegWidth = 1;
1340     break;
1341   case IS_VGPR:
1342   case IS_SGPR:
1343   case IS_TTMP:
1344   {
1345     unsigned Size = 1;
1346     if (RegKind == IS_SGPR || RegKind == IS_TTMP) {
1347       // SGPR and TTMP registers must be are aligned. Max required alignment is 4 dwords.
1348       Size = std::min(RegWidth, 4u);
1349     }
1350     if (RegNum % Size != 0)
1351       return false;
1352     RegNum = RegNum / Size;
1353     int RCID = getRegClass(RegKind, RegWidth);
1354     if (RCID == -1)
1355       return false;
1356     const MCRegisterClass RC = TRI->getRegClass(RCID);
1357     if (RegNum >= RC.getNumRegs())
1358       return false;
1359     Reg = RC.getRegister(RegNum);
1360     break;
1361   }
1362 
1363   default:
1364     llvm_unreachable("unexpected register kind");
1365   }
1366 
1367   if (!subtargetHasRegister(*TRI, Reg))
1368     return false;
1369   return true;
1370 }
1371 
1372 std::unique_ptr<AMDGPUOperand> AMDGPUAsmParser::parseRegister() {
1373   const auto &Tok = Parser.getTok();
1374   SMLoc StartLoc = Tok.getLoc();
1375   SMLoc EndLoc = Tok.getEndLoc();
1376   RegisterKind RegKind;
1377   unsigned Reg, RegNum, RegWidth;
1378 
1379   if (!ParseAMDGPURegister(RegKind, Reg, RegNum, RegWidth)) {
1380     return nullptr;
1381   }
1382   return AMDGPUOperand::CreateReg(this, Reg, StartLoc, EndLoc, false);
1383 }
1384 
1385 OperandMatchResultTy
1386 AMDGPUAsmParser::parseImm(OperandVector &Operands) {
1387   // TODO: add syntactic sugar for 1/(2*PI)
1388   bool Minus = false;
1389   if (getLexer().getKind() == AsmToken::Minus) {
1390     Minus = true;
1391     Parser.Lex();
1392   }
1393 
1394   SMLoc S = Parser.getTok().getLoc();
1395   switch(getLexer().getKind()) {
1396   case AsmToken::Integer: {
1397     int64_t IntVal;
1398     if (getParser().parseAbsoluteExpression(IntVal))
1399       return MatchOperand_ParseFail;
1400     if (Minus)
1401       IntVal *= -1;
1402     Operands.push_back(AMDGPUOperand::CreateImm(this, IntVal, S));
1403     return MatchOperand_Success;
1404   }
1405   case AsmToken::Real: {
1406     int64_t IntVal;
1407     if (getParser().parseAbsoluteExpression(IntVal))
1408       return MatchOperand_ParseFail;
1409 
1410     APFloat F(BitsToDouble(IntVal));
1411     if (Minus)
1412       F.changeSign();
1413     Operands.push_back(
1414         AMDGPUOperand::CreateImm(this, F.bitcastToAPInt().getZExtValue(), S,
1415                                  AMDGPUOperand::ImmTyNone, true));
1416     return MatchOperand_Success;
1417   }
1418   default:
1419     return Minus ? MatchOperand_ParseFail : MatchOperand_NoMatch;
1420   }
1421 }
1422 
1423 OperandMatchResultTy
1424 AMDGPUAsmParser::parseRegOrImm(OperandVector &Operands) {
1425   auto res = parseImm(Operands);
1426   if (res != MatchOperand_NoMatch) {
1427     return res;
1428   }
1429 
1430   if (auto R = parseRegister()) {
1431     assert(R->isReg());
1432     R->Reg.IsForcedVOP3 = isForcedVOP3();
1433     Operands.push_back(std::move(R));
1434     return MatchOperand_Success;
1435   }
1436   return MatchOperand_ParseFail;
1437 }
1438 
1439 OperandMatchResultTy
1440 AMDGPUAsmParser::parseRegOrImmWithFPInputMods(OperandVector &Operands) {
1441   // XXX: During parsing we can't determine if minus sign means
1442   // negate-modifier or negative immediate value.
1443   // By default we suppose it is modifier.
1444   bool Negate = false, Abs = false, Abs2 = false;
1445 
1446   if (getLexer().getKind()== AsmToken::Minus) {
1447     Parser.Lex();
1448     Negate = true;
1449   }
1450 
1451   if (getLexer().getKind() == AsmToken::Identifier && Parser.getTok().getString() == "abs") {
1452     Parser.Lex();
1453     Abs2 = true;
1454     if (getLexer().isNot(AsmToken::LParen)) {
1455       Error(Parser.getTok().getLoc(), "expected left paren after abs");
1456       return MatchOperand_ParseFail;
1457     }
1458     Parser.Lex();
1459   }
1460 
1461   if (getLexer().getKind() == AsmToken::Pipe) {
1462     if (Abs2) {
1463       Error(Parser.getTok().getLoc(), "expected register or immediate");
1464       return MatchOperand_ParseFail;
1465     }
1466     Parser.Lex();
1467     Abs = true;
1468   }
1469 
1470   auto Res = parseRegOrImm(Operands);
1471   if (Res != MatchOperand_Success) {
1472     return Res;
1473   }
1474 
1475   AMDGPUOperand::Modifiers Mods;
1476   if (Negate) {
1477     Mods.Neg = true;
1478   }
1479   if (Abs) {
1480     if (getLexer().getKind() != AsmToken::Pipe) {
1481       Error(Parser.getTok().getLoc(), "expected vertical bar");
1482       return MatchOperand_ParseFail;
1483     }
1484     Parser.Lex();
1485     Mods.Abs = true;
1486   }
1487   if (Abs2) {
1488     if (getLexer().isNot(AsmToken::RParen)) {
1489       Error(Parser.getTok().getLoc(), "expected closing parentheses");
1490       return MatchOperand_ParseFail;
1491     }
1492     Parser.Lex();
1493     Mods.Abs = true;
1494   }
1495 
1496   if (Mods.hasFPModifiers()) {
1497     AMDGPUOperand &Op = static_cast<AMDGPUOperand &>(*Operands.back());
1498     Op.setModifiers(Mods);
1499   }
1500   return MatchOperand_Success;
1501 }
1502 
1503 OperandMatchResultTy
1504 AMDGPUAsmParser::parseRegOrImmWithIntInputMods(OperandVector &Operands) {
1505   bool Sext = false;
1506 
1507   if (getLexer().getKind() == AsmToken::Identifier && Parser.getTok().getString() == "sext") {
1508     Parser.Lex();
1509     Sext = true;
1510     if (getLexer().isNot(AsmToken::LParen)) {
1511       Error(Parser.getTok().getLoc(), "expected left paren after sext");
1512       return MatchOperand_ParseFail;
1513     }
1514     Parser.Lex();
1515   }
1516 
1517   auto Res = parseRegOrImm(Operands);
1518   if (Res != MatchOperand_Success) {
1519     return Res;
1520   }
1521 
1522   AMDGPUOperand::Modifiers Mods;
1523   if (Sext) {
1524     if (getLexer().isNot(AsmToken::RParen)) {
1525       Error(Parser.getTok().getLoc(), "expected closing parentheses");
1526       return MatchOperand_ParseFail;
1527     }
1528     Parser.Lex();
1529     Mods.Sext = true;
1530   }
1531 
1532   if (Mods.hasIntModifiers()) {
1533     AMDGPUOperand &Op = static_cast<AMDGPUOperand &>(*Operands.back());
1534     Op.setModifiers(Mods);
1535   }
1536 
1537   return MatchOperand_Success;
1538 }
1539 
1540 OperandMatchResultTy AMDGPUAsmParser::parseVReg32OrOff(OperandVector &Operands) {
1541   std::unique_ptr<AMDGPUOperand> Reg = parseRegister();
1542   if (Reg) {
1543     Operands.push_back(std::move(Reg));
1544     return MatchOperand_Success;
1545   }
1546 
1547   const AsmToken &Tok = Parser.getTok();
1548   if (Tok.getString() == "off") {
1549     Operands.push_back(AMDGPUOperand::CreateImm(this, 0, Tok.getLoc(),
1550                                                 AMDGPUOperand::ImmTyOff, false));
1551     Parser.Lex();
1552     return MatchOperand_Success;
1553   }
1554 
1555   return MatchOperand_NoMatch;
1556 }
1557 
1558 unsigned AMDGPUAsmParser::checkTargetMatchPredicate(MCInst &Inst) {
1559 
1560   uint64_t TSFlags = MII.get(Inst.getOpcode()).TSFlags;
1561 
1562   if ((getForcedEncodingSize() == 32 && (TSFlags & SIInstrFlags::VOP3)) ||
1563       (getForcedEncodingSize() == 64 && !(TSFlags & SIInstrFlags::VOP3)) ||
1564       (isForcedDPP() && !(TSFlags & SIInstrFlags::DPP)) ||
1565       (isForcedSDWA() && !(TSFlags & SIInstrFlags::SDWA)) )
1566     return Match_InvalidOperand;
1567 
1568   if ((TSFlags & SIInstrFlags::VOP3) &&
1569       (TSFlags & SIInstrFlags::VOPAsmPrefer32Bit) &&
1570       getForcedEncodingSize() != 64)
1571     return Match_PreferE32;
1572 
1573   if (Inst.getOpcode() == AMDGPU::V_MAC_F32_sdwa ||
1574       Inst.getOpcode() == AMDGPU::V_MAC_F16_sdwa) {
1575     // v_mac_f32/16 allow only dst_sel == DWORD;
1576     auto OpNum =
1577         AMDGPU::getNamedOperandIdx(Inst.getOpcode(), AMDGPU::OpName::dst_sel);
1578     const auto &Op = Inst.getOperand(OpNum);
1579     if (!Op.isImm() || Op.getImm() != AMDGPU::SDWA::SdwaSel::DWORD) {
1580       return Match_InvalidOperand;
1581     }
1582   }
1583 
1584   return Match_Success;
1585 }
1586 
1587 bool AMDGPUAsmParser::MatchAndEmitInstruction(SMLoc IDLoc, unsigned &Opcode,
1588                                               OperandVector &Operands,
1589                                               MCStreamer &Out,
1590                                               uint64_t &ErrorInfo,
1591                                               bool MatchingInlineAsm) {
1592   // What asm variants we should check
1593   std::vector<unsigned> MatchedVariants;
1594   if (getForcedEncodingSize() == 32) {
1595     MatchedVariants = {AMDGPUAsmVariants::DEFAULT};
1596   } else if (isForcedVOP3()) {
1597     MatchedVariants = {AMDGPUAsmVariants::VOP3};
1598   } else if (isForcedSDWA()) {
1599     MatchedVariants = {AMDGPUAsmVariants::SDWA};
1600   } else if (isForcedDPP()) {
1601     MatchedVariants = {AMDGPUAsmVariants::DPP};
1602   } else {
1603     MatchedVariants = {AMDGPUAsmVariants::DEFAULT,
1604                        AMDGPUAsmVariants::VOP3,
1605                        AMDGPUAsmVariants::SDWA,
1606                        AMDGPUAsmVariants::DPP};
1607   }
1608 
1609   MCInst Inst;
1610   unsigned Result = Match_Success;
1611   for (auto Variant : MatchedVariants) {
1612     uint64_t EI;
1613     auto R = MatchInstructionImpl(Operands, Inst, EI, MatchingInlineAsm,
1614                                   Variant);
1615     // We order match statuses from least to most specific. We use most specific
1616     // status as resulting
1617     // Match_MnemonicFail < Match_InvalidOperand < Match_MissingFeature < Match_PreferE32
1618     if ((R == Match_Success) ||
1619         (R == Match_PreferE32) ||
1620         (R == Match_MissingFeature && Result != Match_PreferE32) ||
1621         (R == Match_InvalidOperand && Result != Match_MissingFeature
1622                                    && Result != Match_PreferE32) ||
1623         (R == Match_MnemonicFail   && Result != Match_InvalidOperand
1624                                    && Result != Match_MissingFeature
1625                                    && Result != Match_PreferE32)) {
1626       Result = R;
1627       ErrorInfo = EI;
1628     }
1629     if (R == Match_Success)
1630       break;
1631   }
1632 
1633   switch (Result) {
1634   default: break;
1635   case Match_Success:
1636     Inst.setLoc(IDLoc);
1637     Out.EmitInstruction(Inst, getSTI());
1638     return false;
1639 
1640   case Match_MissingFeature:
1641     return Error(IDLoc, "instruction not supported on this GPU");
1642 
1643   case Match_MnemonicFail:
1644     return Error(IDLoc, "unrecognized instruction mnemonic");
1645 
1646   case Match_InvalidOperand: {
1647     SMLoc ErrorLoc = IDLoc;
1648     if (ErrorInfo != ~0ULL) {
1649       if (ErrorInfo >= Operands.size()) {
1650         return Error(IDLoc, "too few operands for instruction");
1651       }
1652       ErrorLoc = ((AMDGPUOperand &)*Operands[ErrorInfo]).getStartLoc();
1653       if (ErrorLoc == SMLoc())
1654         ErrorLoc = IDLoc;
1655     }
1656     return Error(ErrorLoc, "invalid operand for instruction");
1657   }
1658 
1659   case Match_PreferE32:
1660     return Error(IDLoc, "internal error: instruction without _e64 suffix "
1661                         "should be encoded as e32");
1662   }
1663   llvm_unreachable("Implement any new match types added!");
1664 }
1665 
1666 bool AMDGPUAsmParser::ParseDirectiveMajorMinor(uint32_t &Major,
1667                                                uint32_t &Minor) {
1668   if (getLexer().isNot(AsmToken::Integer))
1669     return TokError("invalid major version");
1670 
1671   Major = getLexer().getTok().getIntVal();
1672   Lex();
1673 
1674   if (getLexer().isNot(AsmToken::Comma))
1675     return TokError("minor version number required, comma expected");
1676   Lex();
1677 
1678   if (getLexer().isNot(AsmToken::Integer))
1679     return TokError("invalid minor version");
1680 
1681   Minor = getLexer().getTok().getIntVal();
1682   Lex();
1683 
1684   return false;
1685 }
1686 
1687 bool AMDGPUAsmParser::ParseDirectiveHSACodeObjectVersion() {
1688 
1689   uint32_t Major;
1690   uint32_t Minor;
1691 
1692   if (ParseDirectiveMajorMinor(Major, Minor))
1693     return true;
1694 
1695   getTargetStreamer().EmitDirectiveHSACodeObjectVersion(Major, Minor);
1696   return false;
1697 }
1698 
1699 bool AMDGPUAsmParser::ParseDirectiveHSACodeObjectISA() {
1700   uint32_t Major;
1701   uint32_t Minor;
1702   uint32_t Stepping;
1703   StringRef VendorName;
1704   StringRef ArchName;
1705 
1706   // If this directive has no arguments, then use the ISA version for the
1707   // targeted GPU.
1708   if (getLexer().is(AsmToken::EndOfStatement)) {
1709     AMDGPU::IsaVersion Isa = AMDGPU::getIsaVersion(getSTI().getFeatureBits());
1710     getTargetStreamer().EmitDirectiveHSACodeObjectISA(Isa.Major, Isa.Minor,
1711                                                       Isa.Stepping,
1712                                                       "AMD", "AMDGPU");
1713     return false;
1714   }
1715 
1716   if (ParseDirectiveMajorMinor(Major, Minor))
1717     return true;
1718 
1719   if (getLexer().isNot(AsmToken::Comma))
1720     return TokError("stepping version number required, comma expected");
1721   Lex();
1722 
1723   if (getLexer().isNot(AsmToken::Integer))
1724     return TokError("invalid stepping version");
1725 
1726   Stepping = getLexer().getTok().getIntVal();
1727   Lex();
1728 
1729   if (getLexer().isNot(AsmToken::Comma))
1730     return TokError("vendor name required, comma expected");
1731   Lex();
1732 
1733   if (getLexer().isNot(AsmToken::String))
1734     return TokError("invalid vendor name");
1735 
1736   VendorName = getLexer().getTok().getStringContents();
1737   Lex();
1738 
1739   if (getLexer().isNot(AsmToken::Comma))
1740     return TokError("arch name required, comma expected");
1741   Lex();
1742 
1743   if (getLexer().isNot(AsmToken::String))
1744     return TokError("invalid arch name");
1745 
1746   ArchName = getLexer().getTok().getStringContents();
1747   Lex();
1748 
1749   getTargetStreamer().EmitDirectiveHSACodeObjectISA(Major, Minor, Stepping,
1750                                                     VendorName, ArchName);
1751   return false;
1752 }
1753 
1754 bool AMDGPUAsmParser::ParseDirectiveRuntimeMetadata() {
1755   std::string Metadata;
1756   raw_string_ostream MS(Metadata);
1757 
1758   getLexer().setSkipSpace(false);
1759 
1760   bool FoundEnd = false;
1761   while (!getLexer().is(AsmToken::Eof)) {
1762     while (getLexer().is(AsmToken::Space)) {
1763       MS << ' ';
1764       Lex();
1765     }
1766 
1767     if (getLexer().is(AsmToken::Identifier)) {
1768       StringRef ID = getLexer().getTok().getIdentifier();
1769       if (ID == ".end_amdgpu_runtime_metadata") {
1770         Lex();
1771         FoundEnd = true;
1772         break;
1773       }
1774     }
1775 
1776     MS << Parser.parseStringToEndOfStatement()
1777        << getContext().getAsmInfo()->getSeparatorString();
1778 
1779     Parser.eatToEndOfStatement();
1780   }
1781 
1782   getLexer().setSkipSpace(true);
1783 
1784   if (getLexer().is(AsmToken::Eof) && !FoundEnd)
1785     return TokError("expected directive .end_amdgpu_runtime_metadata not found");
1786 
1787   MS.flush();
1788 
1789   getTargetStreamer().EmitRuntimeMetadata(Metadata);
1790 
1791   return false;
1792 }
1793 
1794 bool AMDGPUAsmParser::ParseAMDKernelCodeTValue(StringRef ID,
1795                                                amd_kernel_code_t &Header) {
1796   SmallString<40> ErrStr;
1797   raw_svector_ostream Err(ErrStr);
1798   if (!parseAmdKernelCodeField(ID, getParser(), Header, Err)) {
1799     return TokError(Err.str());
1800   }
1801   Lex();
1802   return false;
1803 }
1804 
1805 bool AMDGPUAsmParser::ParseDirectiveAMDKernelCodeT() {
1806   amd_kernel_code_t Header;
1807   AMDGPU::initDefaultAMDKernelCodeT(Header, getSTI().getFeatureBits());
1808 
1809   while (true) {
1810     // Lex EndOfStatement.  This is in a while loop, because lexing a comment
1811     // will set the current token to EndOfStatement.
1812     while(getLexer().is(AsmToken::EndOfStatement))
1813       Lex();
1814 
1815     if (getLexer().isNot(AsmToken::Identifier))
1816       return TokError("expected value identifier or .end_amd_kernel_code_t");
1817 
1818     StringRef ID = getLexer().getTok().getIdentifier();
1819     Lex();
1820 
1821     if (ID == ".end_amd_kernel_code_t")
1822       break;
1823 
1824     if (ParseAMDKernelCodeTValue(ID, Header))
1825       return true;
1826   }
1827 
1828   getTargetStreamer().EmitAMDKernelCodeT(Header);
1829 
1830   return false;
1831 }
1832 
1833 bool AMDGPUAsmParser::ParseSectionDirectiveHSAText() {
1834   getParser().getStreamer().SwitchSection(
1835       AMDGPU::getHSATextSection(getContext()));
1836   return false;
1837 }
1838 
1839 bool AMDGPUAsmParser::ParseDirectiveAMDGPUHsaKernel() {
1840   if (getLexer().isNot(AsmToken::Identifier))
1841     return TokError("expected symbol name");
1842 
1843   StringRef KernelName = Parser.getTok().getString();
1844 
1845   getTargetStreamer().EmitAMDGPUSymbolType(KernelName,
1846                                            ELF::STT_AMDGPU_HSA_KERNEL);
1847   Lex();
1848   return false;
1849 }
1850 
1851 bool AMDGPUAsmParser::ParseDirectiveAMDGPUHsaModuleGlobal() {
1852   if (getLexer().isNot(AsmToken::Identifier))
1853     return TokError("expected symbol name");
1854 
1855   StringRef GlobalName = Parser.getTok().getIdentifier();
1856 
1857   getTargetStreamer().EmitAMDGPUHsaModuleScopeGlobal(GlobalName);
1858   Lex();
1859   return false;
1860 }
1861 
1862 bool AMDGPUAsmParser::ParseDirectiveAMDGPUHsaProgramGlobal() {
1863   if (getLexer().isNot(AsmToken::Identifier))
1864     return TokError("expected symbol name");
1865 
1866   StringRef GlobalName = Parser.getTok().getIdentifier();
1867 
1868   getTargetStreamer().EmitAMDGPUHsaProgramScopeGlobal(GlobalName);
1869   Lex();
1870   return false;
1871 }
1872 
1873 bool AMDGPUAsmParser::ParseSectionDirectiveHSADataGlobalAgent() {
1874   getParser().getStreamer().SwitchSection(
1875       AMDGPU::getHSADataGlobalAgentSection(getContext()));
1876   return false;
1877 }
1878 
1879 bool AMDGPUAsmParser::ParseSectionDirectiveHSADataGlobalProgram() {
1880   getParser().getStreamer().SwitchSection(
1881       AMDGPU::getHSADataGlobalProgramSection(getContext()));
1882   return false;
1883 }
1884 
1885 bool AMDGPUAsmParser::ParseSectionDirectiveHSARodataReadonlyAgent() {
1886   getParser().getStreamer().SwitchSection(
1887       AMDGPU::getHSARodataReadonlyAgentSection(getContext()));
1888   return false;
1889 }
1890 
1891 bool AMDGPUAsmParser::ParseDirective(AsmToken DirectiveID) {
1892   StringRef IDVal = DirectiveID.getString();
1893 
1894   if (IDVal == ".hsa_code_object_version")
1895     return ParseDirectiveHSACodeObjectVersion();
1896 
1897   if (IDVal == ".hsa_code_object_isa")
1898     return ParseDirectiveHSACodeObjectISA();
1899 
1900   if (IDVal == ".amdgpu_runtime_metadata")
1901     return ParseDirectiveRuntimeMetadata();
1902 
1903   if (IDVal == ".amd_kernel_code_t")
1904     return ParseDirectiveAMDKernelCodeT();
1905 
1906   if (IDVal == ".hsatext")
1907     return ParseSectionDirectiveHSAText();
1908 
1909   if (IDVal == ".amdgpu_hsa_kernel")
1910     return ParseDirectiveAMDGPUHsaKernel();
1911 
1912   if (IDVal == ".amdgpu_hsa_module_global")
1913     return ParseDirectiveAMDGPUHsaModuleGlobal();
1914 
1915   if (IDVal == ".amdgpu_hsa_program_global")
1916     return ParseDirectiveAMDGPUHsaProgramGlobal();
1917 
1918   if (IDVal == ".hsadata_global_agent")
1919     return ParseSectionDirectiveHSADataGlobalAgent();
1920 
1921   if (IDVal == ".hsadata_global_program")
1922     return ParseSectionDirectiveHSADataGlobalProgram();
1923 
1924   if (IDVal == ".hsarodata_readonly_agent")
1925     return ParseSectionDirectiveHSARodataReadonlyAgent();
1926 
1927   return true;
1928 }
1929 
1930 bool AMDGPUAsmParser::subtargetHasRegister(const MCRegisterInfo &MRI,
1931                                            unsigned RegNo) const {
1932   if (isCI())
1933     return true;
1934 
1935   if (isSI()) {
1936     // No flat_scr
1937     switch (RegNo) {
1938     case AMDGPU::FLAT_SCR:
1939     case AMDGPU::FLAT_SCR_LO:
1940     case AMDGPU::FLAT_SCR_HI:
1941       return false;
1942     default:
1943       return true;
1944     }
1945   }
1946 
1947   // VI only has 102 SGPRs, so make sure we aren't trying to use the 2 more that
1948   // SI/CI have.
1949   for (MCRegAliasIterator R(AMDGPU::SGPR102_SGPR103, &MRI, true);
1950        R.isValid(); ++R) {
1951     if (*R == RegNo)
1952       return false;
1953   }
1954 
1955   return true;
1956 }
1957 
1958 OperandMatchResultTy
1959 AMDGPUAsmParser::parseOperand(OperandVector &Operands, StringRef Mnemonic) {
1960 
1961   // Try to parse with a custom parser
1962   OperandMatchResultTy ResTy = MatchOperandParserImpl(Operands, Mnemonic);
1963 
1964   // If we successfully parsed the operand or if there as an error parsing,
1965   // we are done.
1966   //
1967   // If we are parsing after we reach EndOfStatement then this means we
1968   // are appending default values to the Operands list.  This is only done
1969   // by custom parser, so we shouldn't continue on to the generic parsing.
1970   if (ResTy == MatchOperand_Success || ResTy == MatchOperand_ParseFail ||
1971       getLexer().is(AsmToken::EndOfStatement))
1972     return ResTy;
1973 
1974   ResTy = parseRegOrImm(Operands);
1975 
1976   if (ResTy == MatchOperand_Success)
1977     return ResTy;
1978 
1979   if (getLexer().getKind() == AsmToken::Identifier) {
1980     // If this identifier is a symbol, we want to create an expression for it.
1981     // It is a little difficult to distinguish between a symbol name, and
1982     // an instruction flag like 'gds'.  In order to do this, we parse
1983     // all tokens as expressions and then treate the symbol name as the token
1984     // string when we want to interpret the operand as a token.
1985     const auto &Tok = Parser.getTok();
1986     SMLoc S = Tok.getLoc();
1987     const MCExpr *Expr = nullptr;
1988     if (!Parser.parseExpression(Expr)) {
1989       Operands.push_back(AMDGPUOperand::CreateExpr(this, Expr, S));
1990       return MatchOperand_Success;
1991     }
1992 
1993     Operands.push_back(AMDGPUOperand::CreateToken(this, Tok.getString(), Tok.getLoc()));
1994     Parser.Lex();
1995     return MatchOperand_Success;
1996   }
1997   return MatchOperand_NoMatch;
1998 }
1999 
2000 StringRef AMDGPUAsmParser::parseMnemonicSuffix(StringRef Name) {
2001   // Clear any forced encodings from the previous instruction.
2002   setForcedEncodingSize(0);
2003   setForcedDPP(false);
2004   setForcedSDWA(false);
2005 
2006   if (Name.endswith("_e64")) {
2007     setForcedEncodingSize(64);
2008     return Name.substr(0, Name.size() - 4);
2009   } else if (Name.endswith("_e32")) {
2010     setForcedEncodingSize(32);
2011     return Name.substr(0, Name.size() - 4);
2012   } else if (Name.endswith("_dpp")) {
2013     setForcedDPP(true);
2014     return Name.substr(0, Name.size() - 4);
2015   } else if (Name.endswith("_sdwa")) {
2016     setForcedSDWA(true);
2017     return Name.substr(0, Name.size() - 5);
2018   }
2019   return Name;
2020 }
2021 
2022 bool AMDGPUAsmParser::ParseInstruction(ParseInstructionInfo &Info,
2023                                        StringRef Name,
2024                                        SMLoc NameLoc, OperandVector &Operands) {
2025   // Add the instruction mnemonic
2026   Name = parseMnemonicSuffix(Name);
2027   Operands.push_back(AMDGPUOperand::CreateToken(this, Name, NameLoc));
2028 
2029   while (!getLexer().is(AsmToken::EndOfStatement)) {
2030     OperandMatchResultTy Res = parseOperand(Operands, Name);
2031 
2032     // Eat the comma or space if there is one.
2033     if (getLexer().is(AsmToken::Comma))
2034       Parser.Lex();
2035 
2036     switch (Res) {
2037       case MatchOperand_Success: break;
2038       case MatchOperand_ParseFail:
2039         Error(getLexer().getLoc(), "failed parsing operand.");
2040         while (!getLexer().is(AsmToken::EndOfStatement)) {
2041           Parser.Lex();
2042         }
2043         return true;
2044       case MatchOperand_NoMatch:
2045         Error(getLexer().getLoc(), "not a valid operand.");
2046         while (!getLexer().is(AsmToken::EndOfStatement)) {
2047           Parser.Lex();
2048         }
2049         return true;
2050     }
2051   }
2052 
2053   return false;
2054 }
2055 
2056 //===----------------------------------------------------------------------===//
2057 // Utility functions
2058 //===----------------------------------------------------------------------===//
2059 
2060 OperandMatchResultTy
2061 AMDGPUAsmParser::parseIntWithPrefix(const char *Prefix, int64_t &Int) {
2062   switch(getLexer().getKind()) {
2063     default: return MatchOperand_NoMatch;
2064     case AsmToken::Identifier: {
2065       StringRef Name = Parser.getTok().getString();
2066       if (!Name.equals(Prefix)) {
2067         return MatchOperand_NoMatch;
2068       }
2069 
2070       Parser.Lex();
2071       if (getLexer().isNot(AsmToken::Colon))
2072         return MatchOperand_ParseFail;
2073 
2074       Parser.Lex();
2075       if (getLexer().isNot(AsmToken::Integer))
2076         return MatchOperand_ParseFail;
2077 
2078       if (getParser().parseAbsoluteExpression(Int))
2079         return MatchOperand_ParseFail;
2080       break;
2081     }
2082   }
2083   return MatchOperand_Success;
2084 }
2085 
2086 OperandMatchResultTy
2087 AMDGPUAsmParser::parseIntWithPrefix(const char *Prefix, OperandVector &Operands,
2088                                     enum AMDGPUOperand::ImmTy ImmTy,
2089                                     bool (*ConvertResult)(int64_t&)) {
2090   SMLoc S = Parser.getTok().getLoc();
2091   int64_t Value = 0;
2092 
2093   OperandMatchResultTy Res = parseIntWithPrefix(Prefix, Value);
2094   if (Res != MatchOperand_Success)
2095     return Res;
2096 
2097   if (ConvertResult && !ConvertResult(Value)) {
2098     return MatchOperand_ParseFail;
2099   }
2100 
2101   Operands.push_back(AMDGPUOperand::CreateImm(this, Value, S, ImmTy));
2102   return MatchOperand_Success;
2103 }
2104 
2105 OperandMatchResultTy
2106 AMDGPUAsmParser::parseNamedBit(const char *Name, OperandVector &Operands,
2107                                enum AMDGPUOperand::ImmTy ImmTy) {
2108   int64_t Bit = 0;
2109   SMLoc S = Parser.getTok().getLoc();
2110 
2111   // We are at the end of the statement, and this is a default argument, so
2112   // use a default value.
2113   if (getLexer().isNot(AsmToken::EndOfStatement)) {
2114     switch(getLexer().getKind()) {
2115       case AsmToken::Identifier: {
2116         StringRef Tok = Parser.getTok().getString();
2117         if (Tok == Name) {
2118           Bit = 1;
2119           Parser.Lex();
2120         } else if (Tok.startswith("no") && Tok.endswith(Name)) {
2121           Bit = 0;
2122           Parser.Lex();
2123         } else {
2124           return MatchOperand_NoMatch;
2125         }
2126         break;
2127       }
2128       default:
2129         return MatchOperand_NoMatch;
2130     }
2131   }
2132 
2133   Operands.push_back(AMDGPUOperand::CreateImm(this, Bit, S, ImmTy));
2134   return MatchOperand_Success;
2135 }
2136 
2137 typedef std::map<enum AMDGPUOperand::ImmTy, unsigned> OptionalImmIndexMap;
2138 
2139 void addOptionalImmOperand(MCInst& Inst, const OperandVector& Operands,
2140                            OptionalImmIndexMap& OptionalIdx,
2141                            enum AMDGPUOperand::ImmTy ImmT, int64_t Default = 0) {
2142   auto i = OptionalIdx.find(ImmT);
2143   if (i != OptionalIdx.end()) {
2144     unsigned Idx = i->second;
2145     ((AMDGPUOperand &)*Operands[Idx]).addImmOperands(Inst, 1);
2146   } else {
2147     Inst.addOperand(MCOperand::createImm(Default));
2148   }
2149 }
2150 
2151 OperandMatchResultTy
2152 AMDGPUAsmParser::parseStringWithPrefix(StringRef Prefix, StringRef &Value) {
2153   if (getLexer().isNot(AsmToken::Identifier)) {
2154     return MatchOperand_NoMatch;
2155   }
2156   StringRef Tok = Parser.getTok().getString();
2157   if (Tok != Prefix) {
2158     return MatchOperand_NoMatch;
2159   }
2160 
2161   Parser.Lex();
2162   if (getLexer().isNot(AsmToken::Colon)) {
2163     return MatchOperand_ParseFail;
2164   }
2165 
2166   Parser.Lex();
2167   if (getLexer().isNot(AsmToken::Identifier)) {
2168     return MatchOperand_ParseFail;
2169   }
2170 
2171   Value = Parser.getTok().getString();
2172   return MatchOperand_Success;
2173 }
2174 
2175 //===----------------------------------------------------------------------===//
2176 // ds
2177 //===----------------------------------------------------------------------===//
2178 
2179 void AMDGPUAsmParser::cvtDSOffset01(MCInst &Inst,
2180                                     const OperandVector &Operands) {
2181   OptionalImmIndexMap OptionalIdx;
2182 
2183   for (unsigned i = 1, e = Operands.size(); i != e; ++i) {
2184     AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[i]);
2185 
2186     // Add the register arguments
2187     if (Op.isReg()) {
2188       Op.addRegOperands(Inst, 1);
2189       continue;
2190     }
2191 
2192     // Handle optional arguments
2193     OptionalIdx[Op.getImmTy()] = i;
2194   }
2195 
2196   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyOffset0);
2197   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyOffset1);
2198   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyGDS);
2199 
2200   Inst.addOperand(MCOperand::createReg(AMDGPU::M0)); // m0
2201 }
2202 
2203 void AMDGPUAsmParser::cvtDS(MCInst &Inst, const OperandVector &Operands) {
2204   std::map<enum AMDGPUOperand::ImmTy, unsigned> OptionalIdx;
2205   bool GDSOnly = false;
2206 
2207   for (unsigned i = 1, e = Operands.size(); i != e; ++i) {
2208     AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[i]);
2209 
2210     // Add the register arguments
2211     if (Op.isReg()) {
2212       Op.addRegOperands(Inst, 1);
2213       continue;
2214     }
2215 
2216     if (Op.isToken() && Op.getToken() == "gds") {
2217       GDSOnly = true;
2218       continue;
2219     }
2220 
2221     // Handle optional arguments
2222     OptionalIdx[Op.getImmTy()] = i;
2223   }
2224 
2225   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyOffset);
2226   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyGDS);
2227 
2228   if (!GDSOnly) {
2229     addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyGDS);
2230   }
2231   Inst.addOperand(MCOperand::createReg(AMDGPU::M0)); // m0
2232 }
2233 
2234 void AMDGPUAsmParser::cvtExp(MCInst &Inst, const OperandVector &Operands) {
2235   OptionalImmIndexMap OptionalIdx;
2236 
2237   unsigned EnMask = 0;
2238   int SrcIdx = 0;
2239 
2240   for (unsigned i = 1, e = Operands.size(); i != e; ++i) {
2241     AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[i]);
2242 
2243     // Add the register arguments
2244     if (Op.isReg()) {
2245       EnMask |= (1 << SrcIdx);
2246       Op.addRegOperands(Inst, 1);
2247       ++SrcIdx;
2248       continue;
2249     }
2250 
2251     if (Op.isOff()) {
2252       ++SrcIdx;
2253       Inst.addOperand(MCOperand::createReg(AMDGPU::NoRegister));
2254       continue;
2255     }
2256 
2257     if (Op.isImm() && Op.getImmTy() == AMDGPUOperand::ImmTyExpTgt) {
2258       Op.addImmOperands(Inst, 1);
2259       continue;
2260     }
2261 
2262     if (Op.isToken() && Op.getToken() == "done")
2263       continue;
2264 
2265     // Handle optional arguments
2266     OptionalIdx[Op.getImmTy()] = i;
2267   }
2268 
2269   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyExpVM);
2270   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyExpCompr);
2271 
2272   Inst.addOperand(MCOperand::createImm(EnMask));
2273 }
2274 
2275 //===----------------------------------------------------------------------===//
2276 // s_waitcnt
2277 //===----------------------------------------------------------------------===//
2278 
2279 bool AMDGPUAsmParser::parseCnt(int64_t &IntVal) {
2280   StringRef CntName = Parser.getTok().getString();
2281   int64_t CntVal;
2282 
2283   Parser.Lex();
2284   if (getLexer().isNot(AsmToken::LParen))
2285     return true;
2286 
2287   Parser.Lex();
2288   if (getLexer().isNot(AsmToken::Integer))
2289     return true;
2290 
2291   if (getParser().parseAbsoluteExpression(CntVal))
2292     return true;
2293 
2294   if (getLexer().isNot(AsmToken::RParen))
2295     return true;
2296 
2297   Parser.Lex();
2298   if (getLexer().is(AsmToken::Amp) || getLexer().is(AsmToken::Comma))
2299     Parser.Lex();
2300 
2301   IsaVersion IV = getIsaVersion(getSTI().getFeatureBits());
2302   if (CntName == "vmcnt")
2303     IntVal = encodeVmcnt(IV, IntVal, CntVal);
2304   else if (CntName == "expcnt")
2305     IntVal = encodeExpcnt(IV, IntVal, CntVal);
2306   else if (CntName == "lgkmcnt")
2307     IntVal = encodeLgkmcnt(IV, IntVal, CntVal);
2308   else
2309     return true;
2310 
2311   return false;
2312 }
2313 
2314 OperandMatchResultTy
2315 AMDGPUAsmParser::parseSWaitCntOps(OperandVector &Operands) {
2316   IsaVersion IV = getIsaVersion(getSTI().getFeatureBits());
2317   int64_t Waitcnt = getWaitcntBitMask(IV);
2318   SMLoc S = Parser.getTok().getLoc();
2319 
2320   switch(getLexer().getKind()) {
2321     default: return MatchOperand_ParseFail;
2322     case AsmToken::Integer:
2323       // The operand can be an integer value.
2324       if (getParser().parseAbsoluteExpression(Waitcnt))
2325         return MatchOperand_ParseFail;
2326       break;
2327 
2328     case AsmToken::Identifier:
2329       do {
2330         if (parseCnt(Waitcnt))
2331           return MatchOperand_ParseFail;
2332       } while(getLexer().isNot(AsmToken::EndOfStatement));
2333       break;
2334   }
2335   Operands.push_back(AMDGPUOperand::CreateImm(this, Waitcnt, S));
2336   return MatchOperand_Success;
2337 }
2338 
2339 bool AMDGPUAsmParser::parseHwregConstruct(OperandInfoTy &HwReg, int64_t &Offset, int64_t &Width) {
2340   using namespace llvm::AMDGPU::Hwreg;
2341 
2342   if (Parser.getTok().getString() != "hwreg")
2343     return true;
2344   Parser.Lex();
2345 
2346   if (getLexer().isNot(AsmToken::LParen))
2347     return true;
2348   Parser.Lex();
2349 
2350   if (getLexer().is(AsmToken::Identifier)) {
2351     HwReg.IsSymbolic = true;
2352     HwReg.Id = ID_UNKNOWN_;
2353     const StringRef tok = Parser.getTok().getString();
2354     for (int i = ID_SYMBOLIC_FIRST_; i < ID_SYMBOLIC_LAST_; ++i) {
2355       if (tok == IdSymbolic[i]) {
2356         HwReg.Id = i;
2357         break;
2358       }
2359     }
2360     Parser.Lex();
2361   } else {
2362     HwReg.IsSymbolic = false;
2363     if (getLexer().isNot(AsmToken::Integer))
2364       return true;
2365     if (getParser().parseAbsoluteExpression(HwReg.Id))
2366       return true;
2367   }
2368 
2369   if (getLexer().is(AsmToken::RParen)) {
2370     Parser.Lex();
2371     return false;
2372   }
2373 
2374   // optional params
2375   if (getLexer().isNot(AsmToken::Comma))
2376     return true;
2377   Parser.Lex();
2378 
2379   if (getLexer().isNot(AsmToken::Integer))
2380     return true;
2381   if (getParser().parseAbsoluteExpression(Offset))
2382     return true;
2383 
2384   if (getLexer().isNot(AsmToken::Comma))
2385     return true;
2386   Parser.Lex();
2387 
2388   if (getLexer().isNot(AsmToken::Integer))
2389     return true;
2390   if (getParser().parseAbsoluteExpression(Width))
2391     return true;
2392 
2393   if (getLexer().isNot(AsmToken::RParen))
2394     return true;
2395   Parser.Lex();
2396 
2397   return false;
2398 }
2399 
2400 OperandMatchResultTy
2401 AMDGPUAsmParser::parseHwreg(OperandVector &Operands) {
2402   using namespace llvm::AMDGPU::Hwreg;
2403 
2404   int64_t Imm16Val = 0;
2405   SMLoc S = Parser.getTok().getLoc();
2406 
2407   switch(getLexer().getKind()) {
2408     default: return MatchOperand_NoMatch;
2409     case AsmToken::Integer:
2410       // The operand can be an integer value.
2411       if (getParser().parseAbsoluteExpression(Imm16Val))
2412         return MatchOperand_NoMatch;
2413       if (Imm16Val < 0 || !isUInt<16>(Imm16Val)) {
2414         Error(S, "invalid immediate: only 16-bit values are legal");
2415         // Do not return error code, but create an imm operand anyway and proceed
2416         // to the next operand, if any. That avoids unneccessary error messages.
2417       }
2418       break;
2419 
2420     case AsmToken::Identifier: {
2421         OperandInfoTy HwReg(ID_UNKNOWN_);
2422         int64_t Offset = OFFSET_DEFAULT_;
2423         int64_t Width = WIDTH_M1_DEFAULT_ + 1;
2424         if (parseHwregConstruct(HwReg, Offset, Width))
2425           return MatchOperand_ParseFail;
2426         if (HwReg.Id < 0 || !isUInt<ID_WIDTH_>(HwReg.Id)) {
2427           if (HwReg.IsSymbolic)
2428             Error(S, "invalid symbolic name of hardware register");
2429           else
2430             Error(S, "invalid code of hardware register: only 6-bit values are legal");
2431         }
2432         if (Offset < 0 || !isUInt<OFFSET_WIDTH_>(Offset))
2433           Error(S, "invalid bit offset: only 5-bit values are legal");
2434         if ((Width-1) < 0 || !isUInt<WIDTH_M1_WIDTH_>(Width-1))
2435           Error(S, "invalid bitfield width: only values from 1 to 32 are legal");
2436         Imm16Val = (HwReg.Id << ID_SHIFT_) | (Offset << OFFSET_SHIFT_) | ((Width-1) << WIDTH_M1_SHIFT_);
2437       }
2438       break;
2439   }
2440   Operands.push_back(AMDGPUOperand::CreateImm(this, Imm16Val, S, AMDGPUOperand::ImmTyHwreg));
2441   return MatchOperand_Success;
2442 }
2443 
2444 bool AMDGPUOperand::isSWaitCnt() const {
2445   return isImm();
2446 }
2447 
2448 bool AMDGPUOperand::isHwreg() const {
2449   return isImmTy(ImmTyHwreg);
2450 }
2451 
2452 bool AMDGPUAsmParser::parseSendMsgConstruct(OperandInfoTy &Msg, OperandInfoTy &Operation, int64_t &StreamId) {
2453   using namespace llvm::AMDGPU::SendMsg;
2454 
2455   if (Parser.getTok().getString() != "sendmsg")
2456     return true;
2457   Parser.Lex();
2458 
2459   if (getLexer().isNot(AsmToken::LParen))
2460     return true;
2461   Parser.Lex();
2462 
2463   if (getLexer().is(AsmToken::Identifier)) {
2464     Msg.IsSymbolic = true;
2465     Msg.Id = ID_UNKNOWN_;
2466     const std::string tok = Parser.getTok().getString();
2467     for (int i = ID_GAPS_FIRST_; i < ID_GAPS_LAST_; ++i) {
2468       switch(i) {
2469         default: continue; // Omit gaps.
2470         case ID_INTERRUPT: case ID_GS: case ID_GS_DONE:  case ID_SYSMSG: break;
2471       }
2472       if (tok == IdSymbolic[i]) {
2473         Msg.Id = i;
2474         break;
2475       }
2476     }
2477     Parser.Lex();
2478   } else {
2479     Msg.IsSymbolic = false;
2480     if (getLexer().isNot(AsmToken::Integer))
2481       return true;
2482     if (getParser().parseAbsoluteExpression(Msg.Id))
2483       return true;
2484     if (getLexer().is(AsmToken::Integer))
2485       if (getParser().parseAbsoluteExpression(Msg.Id))
2486         Msg.Id = ID_UNKNOWN_;
2487   }
2488   if (Msg.Id == ID_UNKNOWN_) // Don't know how to parse the rest.
2489     return false;
2490 
2491   if (!(Msg.Id == ID_GS || Msg.Id == ID_GS_DONE || Msg.Id == ID_SYSMSG)) {
2492     if (getLexer().isNot(AsmToken::RParen))
2493       return true;
2494     Parser.Lex();
2495     return false;
2496   }
2497 
2498   if (getLexer().isNot(AsmToken::Comma))
2499     return true;
2500   Parser.Lex();
2501 
2502   assert(Msg.Id == ID_GS || Msg.Id == ID_GS_DONE || Msg.Id == ID_SYSMSG);
2503   Operation.Id = ID_UNKNOWN_;
2504   if (getLexer().is(AsmToken::Identifier)) {
2505     Operation.IsSymbolic = true;
2506     const char* const *S = (Msg.Id == ID_SYSMSG) ? OpSysSymbolic : OpGsSymbolic;
2507     const int F = (Msg.Id == ID_SYSMSG) ? OP_SYS_FIRST_ : OP_GS_FIRST_;
2508     const int L = (Msg.Id == ID_SYSMSG) ? OP_SYS_LAST_ : OP_GS_LAST_;
2509     const StringRef Tok = Parser.getTok().getString();
2510     for (int i = F; i < L; ++i) {
2511       if (Tok == S[i]) {
2512         Operation.Id = i;
2513         break;
2514       }
2515     }
2516     Parser.Lex();
2517   } else {
2518     Operation.IsSymbolic = false;
2519     if (getLexer().isNot(AsmToken::Integer))
2520       return true;
2521     if (getParser().parseAbsoluteExpression(Operation.Id))
2522       return true;
2523   }
2524 
2525   if ((Msg.Id == ID_GS || Msg.Id == ID_GS_DONE) && Operation.Id != OP_GS_NOP) {
2526     // Stream id is optional.
2527     if (getLexer().is(AsmToken::RParen)) {
2528       Parser.Lex();
2529       return false;
2530     }
2531 
2532     if (getLexer().isNot(AsmToken::Comma))
2533       return true;
2534     Parser.Lex();
2535 
2536     if (getLexer().isNot(AsmToken::Integer))
2537       return true;
2538     if (getParser().parseAbsoluteExpression(StreamId))
2539       return true;
2540   }
2541 
2542   if (getLexer().isNot(AsmToken::RParen))
2543     return true;
2544   Parser.Lex();
2545   return false;
2546 }
2547 
2548 OperandMatchResultTy AMDGPUAsmParser::parseInterpSlot(OperandVector &Operands) {
2549   if (getLexer().getKind() != AsmToken::Identifier)
2550     return MatchOperand_NoMatch;
2551 
2552   StringRef Str = Parser.getTok().getString();
2553   int Slot = StringSwitch<int>(Str)
2554     .Case("p10", 0)
2555     .Case("p20", 1)
2556     .Case("p0", 2)
2557     .Default(-1);
2558 
2559   SMLoc S = Parser.getTok().getLoc();
2560   if (Slot == -1)
2561     return MatchOperand_ParseFail;
2562 
2563   Parser.Lex();
2564   Operands.push_back(AMDGPUOperand::CreateImm(this, Slot, S,
2565                                               AMDGPUOperand::ImmTyInterpSlot));
2566   return MatchOperand_Success;
2567 }
2568 
2569 OperandMatchResultTy AMDGPUAsmParser::parseInterpAttr(OperandVector &Operands) {
2570   if (getLexer().getKind() != AsmToken::Identifier)
2571     return MatchOperand_NoMatch;
2572 
2573   StringRef Str = Parser.getTok().getString();
2574   if (!Str.startswith("attr"))
2575     return MatchOperand_NoMatch;
2576 
2577   StringRef Chan = Str.take_back(2);
2578   int AttrChan = StringSwitch<int>(Chan)
2579     .Case(".x", 0)
2580     .Case(".y", 1)
2581     .Case(".z", 2)
2582     .Case(".w", 3)
2583     .Default(-1);
2584   if (AttrChan == -1)
2585     return MatchOperand_ParseFail;
2586 
2587   Str = Str.drop_back(2).drop_front(4);
2588 
2589   uint8_t Attr;
2590   if (Str.getAsInteger(10, Attr))
2591     return MatchOperand_ParseFail;
2592 
2593   SMLoc S = Parser.getTok().getLoc();
2594   Parser.Lex();
2595   if (Attr > 63) {
2596     Error(S, "out of bounds attr");
2597     return MatchOperand_Success;
2598   }
2599 
2600   SMLoc SChan = SMLoc::getFromPointer(Chan.data());
2601 
2602   Operands.push_back(AMDGPUOperand::CreateImm(this, Attr, S,
2603                                               AMDGPUOperand::ImmTyInterpAttr));
2604   Operands.push_back(AMDGPUOperand::CreateImm(this, AttrChan, SChan,
2605                                               AMDGPUOperand::ImmTyAttrChan));
2606   return MatchOperand_Success;
2607 }
2608 
2609 void AMDGPUAsmParser::errorExpTgt() {
2610   Error(Parser.getTok().getLoc(), "invalid exp target");
2611 }
2612 
2613 OperandMatchResultTy AMDGPUAsmParser::parseExpTgtImpl(StringRef Str,
2614                                                       uint8_t &Val) {
2615   if (Str == "null") {
2616     Val = 9;
2617     return MatchOperand_Success;
2618   }
2619 
2620   if (Str.startswith("mrt")) {
2621     Str = Str.drop_front(3);
2622     if (Str == "z") { // == mrtz
2623       Val = 8;
2624       return MatchOperand_Success;
2625     }
2626 
2627     if (Str.getAsInteger(10, Val))
2628       return MatchOperand_ParseFail;
2629 
2630     if (Val > 7)
2631       errorExpTgt();
2632 
2633     return MatchOperand_Success;
2634   }
2635 
2636   if (Str.startswith("pos")) {
2637     Str = Str.drop_front(3);
2638     if (Str.getAsInteger(10, Val))
2639       return MatchOperand_ParseFail;
2640 
2641     if (Val > 3)
2642       errorExpTgt();
2643 
2644     Val += 12;
2645     return MatchOperand_Success;
2646   }
2647 
2648   if (Str.startswith("param")) {
2649     Str = Str.drop_front(5);
2650     if (Str.getAsInteger(10, Val))
2651       return MatchOperand_ParseFail;
2652 
2653     if (Val >= 32)
2654       errorExpTgt();
2655 
2656     Val += 32;
2657     return MatchOperand_Success;
2658   }
2659 
2660   if (Str.startswith("invalid_target_")) {
2661     Str = Str.drop_front(15);
2662     if (Str.getAsInteger(10, Val))
2663       return MatchOperand_ParseFail;
2664 
2665     errorExpTgt();
2666     return MatchOperand_Success;
2667   }
2668 
2669   return MatchOperand_NoMatch;
2670 }
2671 
2672 OperandMatchResultTy AMDGPUAsmParser::parseExpTgt(OperandVector &Operands) {
2673   uint8_t Val;
2674   StringRef Str = Parser.getTok().getString();
2675 
2676   auto Res = parseExpTgtImpl(Str, Val);
2677   if (Res != MatchOperand_Success)
2678     return Res;
2679 
2680   SMLoc S = Parser.getTok().getLoc();
2681   Parser.Lex();
2682 
2683   Operands.push_back(AMDGPUOperand::CreateImm(this, Val, S,
2684                                               AMDGPUOperand::ImmTyExpTgt));
2685   return MatchOperand_Success;
2686 }
2687 
2688 OperandMatchResultTy
2689 AMDGPUAsmParser::parseSendMsgOp(OperandVector &Operands) {
2690   using namespace llvm::AMDGPU::SendMsg;
2691 
2692   int64_t Imm16Val = 0;
2693   SMLoc S = Parser.getTok().getLoc();
2694 
2695   switch(getLexer().getKind()) {
2696   default:
2697     return MatchOperand_NoMatch;
2698   case AsmToken::Integer:
2699     // The operand can be an integer value.
2700     if (getParser().parseAbsoluteExpression(Imm16Val))
2701       return MatchOperand_NoMatch;
2702     if (Imm16Val < 0 || !isUInt<16>(Imm16Val)) {
2703       Error(S, "invalid immediate: only 16-bit values are legal");
2704       // Do not return error code, but create an imm operand anyway and proceed
2705       // to the next operand, if any. That avoids unneccessary error messages.
2706     }
2707     break;
2708   case AsmToken::Identifier: {
2709       OperandInfoTy Msg(ID_UNKNOWN_);
2710       OperandInfoTy Operation(OP_UNKNOWN_);
2711       int64_t StreamId = STREAM_ID_DEFAULT_;
2712       if (parseSendMsgConstruct(Msg, Operation, StreamId))
2713         return MatchOperand_ParseFail;
2714       do {
2715         // Validate and encode message ID.
2716         if (! ((ID_INTERRUPT <= Msg.Id && Msg.Id <= ID_GS_DONE)
2717                 || Msg.Id == ID_SYSMSG)) {
2718           if (Msg.IsSymbolic)
2719             Error(S, "invalid/unsupported symbolic name of message");
2720           else
2721             Error(S, "invalid/unsupported code of message");
2722           break;
2723         }
2724         Imm16Val = (Msg.Id << ID_SHIFT_);
2725         // Validate and encode operation ID.
2726         if (Msg.Id == ID_GS || Msg.Id == ID_GS_DONE) {
2727           if (! (OP_GS_FIRST_ <= Operation.Id && Operation.Id < OP_GS_LAST_)) {
2728             if (Operation.IsSymbolic)
2729               Error(S, "invalid symbolic name of GS_OP");
2730             else
2731               Error(S, "invalid code of GS_OP: only 2-bit values are legal");
2732             break;
2733           }
2734           if (Operation.Id == OP_GS_NOP
2735               && Msg.Id != ID_GS_DONE) {
2736             Error(S, "invalid GS_OP: NOP is for GS_DONE only");
2737             break;
2738           }
2739           Imm16Val |= (Operation.Id << OP_SHIFT_);
2740         }
2741         if (Msg.Id == ID_SYSMSG) {
2742           if (! (OP_SYS_FIRST_ <= Operation.Id && Operation.Id < OP_SYS_LAST_)) {
2743             if (Operation.IsSymbolic)
2744               Error(S, "invalid/unsupported symbolic name of SYSMSG_OP");
2745             else
2746               Error(S, "invalid/unsupported code of SYSMSG_OP");
2747             break;
2748           }
2749           Imm16Val |= (Operation.Id << OP_SHIFT_);
2750         }
2751         // Validate and encode stream ID.
2752         if ((Msg.Id == ID_GS || Msg.Id == ID_GS_DONE) && Operation.Id != OP_GS_NOP) {
2753           if (! (STREAM_ID_FIRST_ <= StreamId && StreamId < STREAM_ID_LAST_)) {
2754             Error(S, "invalid stream id: only 2-bit values are legal");
2755             break;
2756           }
2757           Imm16Val |= (StreamId << STREAM_ID_SHIFT_);
2758         }
2759       } while (false);
2760     }
2761     break;
2762   }
2763   Operands.push_back(AMDGPUOperand::CreateImm(this, Imm16Val, S, AMDGPUOperand::ImmTySendMsg));
2764   return MatchOperand_Success;
2765 }
2766 
2767 bool AMDGPUOperand::isSendMsg() const {
2768   return isImmTy(ImmTySendMsg);
2769 }
2770 
2771 //===----------------------------------------------------------------------===//
2772 // sopp branch targets
2773 //===----------------------------------------------------------------------===//
2774 
2775 OperandMatchResultTy
2776 AMDGPUAsmParser::parseSOppBrTarget(OperandVector &Operands) {
2777   SMLoc S = Parser.getTok().getLoc();
2778 
2779   switch (getLexer().getKind()) {
2780     default: return MatchOperand_ParseFail;
2781     case AsmToken::Integer: {
2782       int64_t Imm;
2783       if (getParser().parseAbsoluteExpression(Imm))
2784         return MatchOperand_ParseFail;
2785       Operands.push_back(AMDGPUOperand::CreateImm(this, Imm, S));
2786       return MatchOperand_Success;
2787     }
2788 
2789     case AsmToken::Identifier:
2790       Operands.push_back(AMDGPUOperand::CreateExpr(this,
2791           MCSymbolRefExpr::create(getContext().getOrCreateSymbol(
2792                                   Parser.getTok().getString()), getContext()), S));
2793       Parser.Lex();
2794       return MatchOperand_Success;
2795   }
2796 }
2797 
2798 //===----------------------------------------------------------------------===//
2799 // mubuf
2800 //===----------------------------------------------------------------------===//
2801 
2802 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultGLC() const {
2803   return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTyGLC);
2804 }
2805 
2806 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultSLC() const {
2807   return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTySLC);
2808 }
2809 
2810 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultTFE() const {
2811   return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTyTFE);
2812 }
2813 
2814 void AMDGPUAsmParser::cvtMubufImpl(MCInst &Inst,
2815                                const OperandVector &Operands,
2816                                bool IsAtomic, bool IsAtomicReturn) {
2817   OptionalImmIndexMap OptionalIdx;
2818   assert(IsAtomicReturn ? IsAtomic : true);
2819 
2820   for (unsigned i = 1, e = Operands.size(); i != e; ++i) {
2821     AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[i]);
2822 
2823     // Add the register arguments
2824     if (Op.isReg()) {
2825       Op.addRegOperands(Inst, 1);
2826       continue;
2827     }
2828 
2829     // Handle the case where soffset is an immediate
2830     if (Op.isImm() && Op.getImmTy() == AMDGPUOperand::ImmTyNone) {
2831       Op.addImmOperands(Inst, 1);
2832       continue;
2833     }
2834 
2835     // Handle tokens like 'offen' which are sometimes hard-coded into the
2836     // asm string.  There are no MCInst operands for these.
2837     if (Op.isToken()) {
2838       continue;
2839     }
2840     assert(Op.isImm());
2841 
2842     // Handle optional arguments
2843     OptionalIdx[Op.getImmTy()] = i;
2844   }
2845 
2846   // Copy $vdata_in operand and insert as $vdata for MUBUF_Atomic RTN insns.
2847   if (IsAtomicReturn) {
2848     MCInst::iterator I = Inst.begin(); // $vdata_in is always at the beginning.
2849     Inst.insert(I, *I);
2850   }
2851 
2852   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyOffset);
2853   if (!IsAtomic) { // glc is hard-coded.
2854     addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyGLC);
2855   }
2856   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySLC);
2857   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyTFE);
2858 }
2859 
2860 //===----------------------------------------------------------------------===//
2861 // mimg
2862 //===----------------------------------------------------------------------===//
2863 
2864 void AMDGPUAsmParser::cvtMIMG(MCInst &Inst, const OperandVector &Operands) {
2865   unsigned I = 1;
2866   const MCInstrDesc &Desc = MII.get(Inst.getOpcode());
2867   for (unsigned J = 0; J < Desc.getNumDefs(); ++J) {
2868     ((AMDGPUOperand &)*Operands[I++]).addRegOperands(Inst, 1);
2869   }
2870 
2871   OptionalImmIndexMap OptionalIdx;
2872 
2873   for (unsigned E = Operands.size(); I != E; ++I) {
2874     AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[I]);
2875 
2876     // Add the register arguments
2877     if (Op.isRegOrImm()) {
2878       Op.addRegOrImmOperands(Inst, 1);
2879       continue;
2880     } else if (Op.isImmModifier()) {
2881       OptionalIdx[Op.getImmTy()] = I;
2882     } else {
2883       llvm_unreachable("unexpected operand type");
2884     }
2885   }
2886 
2887   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyDMask);
2888   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyUNorm);
2889   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyGLC);
2890   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyDA);
2891   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyR128);
2892   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyTFE);
2893   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyLWE);
2894   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySLC);
2895 }
2896 
2897 void AMDGPUAsmParser::cvtMIMGAtomic(MCInst &Inst, const OperandVector &Operands) {
2898   unsigned I = 1;
2899   const MCInstrDesc &Desc = MII.get(Inst.getOpcode());
2900   for (unsigned J = 0; J < Desc.getNumDefs(); ++J) {
2901     ((AMDGPUOperand &)*Operands[I++]).addRegOperands(Inst, 1);
2902   }
2903 
2904   // Add src, same as dst
2905   ((AMDGPUOperand &)*Operands[I]).addRegOperands(Inst, 1);
2906 
2907   OptionalImmIndexMap OptionalIdx;
2908 
2909   for (unsigned E = Operands.size(); I != E; ++I) {
2910     AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[I]);
2911 
2912     // Add the register arguments
2913     if (Op.isRegOrImm()) {
2914       Op.addRegOrImmOperands(Inst, 1);
2915       continue;
2916     } else if (Op.isImmModifier()) {
2917       OptionalIdx[Op.getImmTy()] = I;
2918     } else {
2919       llvm_unreachable("unexpected operand type");
2920     }
2921   }
2922 
2923   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyDMask);
2924   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyUNorm);
2925   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyGLC);
2926   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyDA);
2927   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyR128);
2928   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyTFE);
2929   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyLWE);
2930   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySLC);
2931 }
2932 
2933 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultDMask() const {
2934   return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTyDMask);
2935 }
2936 
2937 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultUNorm() const {
2938   return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTyUNorm);
2939 }
2940 
2941 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultDA() const {
2942   return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTyDA);
2943 }
2944 
2945 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultR128() const {
2946   return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTyR128);
2947 }
2948 
2949 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultLWE() const {
2950   return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTyLWE);
2951 }
2952 
2953 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultExpTgt() const {
2954   return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTyExpTgt);
2955 }
2956 
2957 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultExpCompr() const {
2958   return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTyExpCompr);
2959 }
2960 
2961 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultExpVM() const {
2962   return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTyExpVM);
2963 }
2964 
2965 //===----------------------------------------------------------------------===//
2966 // smrd
2967 //===----------------------------------------------------------------------===//
2968 
2969 bool AMDGPUOperand::isSMRDOffset8() const {
2970   return isImm() && isUInt<8>(getImm());
2971 }
2972 
2973 bool AMDGPUOperand::isSMRDOffset20() const {
2974   return isImm() && isUInt<20>(getImm());
2975 }
2976 
2977 bool AMDGPUOperand::isSMRDLiteralOffset() const {
2978   // 32-bit literals are only supported on CI and we only want to use them
2979   // when the offset is > 8-bits.
2980   return isImm() && !isUInt<8>(getImm()) && isUInt<32>(getImm());
2981 }
2982 
2983 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultSMRDOffset8() const {
2984   return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTyOffset);
2985 }
2986 
2987 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultSMRDOffset20() const {
2988   return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTyOffset);
2989 }
2990 
2991 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultSMRDLiteralOffset() const {
2992   return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTyOffset);
2993 }
2994 
2995 //===----------------------------------------------------------------------===//
2996 // vop3
2997 //===----------------------------------------------------------------------===//
2998 
2999 static bool ConvertOmodMul(int64_t &Mul) {
3000   if (Mul != 1 && Mul != 2 && Mul != 4)
3001     return false;
3002 
3003   Mul >>= 1;
3004   return true;
3005 }
3006 
3007 static bool ConvertOmodDiv(int64_t &Div) {
3008   if (Div == 1) {
3009     Div = 0;
3010     return true;
3011   }
3012 
3013   if (Div == 2) {
3014     Div = 3;
3015     return true;
3016   }
3017 
3018   return false;
3019 }
3020 
3021 static bool ConvertBoundCtrl(int64_t &BoundCtrl) {
3022   if (BoundCtrl == 0) {
3023     BoundCtrl = 1;
3024     return true;
3025   }
3026 
3027   if (BoundCtrl == -1) {
3028     BoundCtrl = 0;
3029     return true;
3030   }
3031 
3032   return false;
3033 }
3034 
3035 // Note: the order in this table matches the order of operands in AsmString.
3036 static const OptionalOperand AMDGPUOptionalOperandTable[] = {
3037   {"offen",   AMDGPUOperand::ImmTyOffen, true, nullptr},
3038   {"idxen",   AMDGPUOperand::ImmTyIdxen, true, nullptr},
3039   {"addr64",  AMDGPUOperand::ImmTyAddr64, true, nullptr},
3040   {"offset0", AMDGPUOperand::ImmTyOffset0, false, nullptr},
3041   {"offset1", AMDGPUOperand::ImmTyOffset1, false, nullptr},
3042   {"gds",     AMDGPUOperand::ImmTyGDS, true, nullptr},
3043   {"offset",  AMDGPUOperand::ImmTyOffset, false, nullptr},
3044   {"glc",     AMDGPUOperand::ImmTyGLC, true, nullptr},
3045   {"slc",     AMDGPUOperand::ImmTySLC, true, nullptr},
3046   {"tfe",     AMDGPUOperand::ImmTyTFE, true, nullptr},
3047   {"clamp",   AMDGPUOperand::ImmTyClampSI, true, nullptr},
3048   {"omod",    AMDGPUOperand::ImmTyOModSI, false, ConvertOmodMul},
3049   {"unorm",   AMDGPUOperand::ImmTyUNorm, true, nullptr},
3050   {"da",      AMDGPUOperand::ImmTyDA,    true, nullptr},
3051   {"r128",    AMDGPUOperand::ImmTyR128,  true, nullptr},
3052   {"lwe",     AMDGPUOperand::ImmTyLWE,   true, nullptr},
3053   {"dmask",   AMDGPUOperand::ImmTyDMask, false, nullptr},
3054   {"row_mask",   AMDGPUOperand::ImmTyDppRowMask, false, nullptr},
3055   {"bank_mask",  AMDGPUOperand::ImmTyDppBankMask, false, nullptr},
3056   {"bound_ctrl", AMDGPUOperand::ImmTyDppBoundCtrl, false, ConvertBoundCtrl},
3057   {"dst_sel",    AMDGPUOperand::ImmTySdwaDstSel, false, nullptr},
3058   {"src0_sel",   AMDGPUOperand::ImmTySdwaSrc0Sel, false, nullptr},
3059   {"src1_sel",   AMDGPUOperand::ImmTySdwaSrc1Sel, false, nullptr},
3060   {"dst_unused", AMDGPUOperand::ImmTySdwaDstUnused, false, nullptr},
3061   {"vm", AMDGPUOperand::ImmTyExpVM, true, nullptr},
3062 };
3063 
3064 OperandMatchResultTy AMDGPUAsmParser::parseOptionalOperand(OperandVector &Operands) {
3065   OperandMatchResultTy res;
3066   for (const OptionalOperand &Op : AMDGPUOptionalOperandTable) {
3067     // try to parse any optional operand here
3068     if (Op.IsBit) {
3069       res = parseNamedBit(Op.Name, Operands, Op.Type);
3070     } else if (Op.Type == AMDGPUOperand::ImmTyOModSI) {
3071       res = parseOModOperand(Operands);
3072     } else if (Op.Type == AMDGPUOperand::ImmTySdwaDstSel ||
3073                Op.Type == AMDGPUOperand::ImmTySdwaSrc0Sel ||
3074                Op.Type == AMDGPUOperand::ImmTySdwaSrc1Sel) {
3075       res = parseSDWASel(Operands, Op.Name, Op.Type);
3076     } else if (Op.Type == AMDGPUOperand::ImmTySdwaDstUnused) {
3077       res = parseSDWADstUnused(Operands);
3078     } else {
3079       res = parseIntWithPrefix(Op.Name, Operands, Op.Type, Op.ConvertResult);
3080     }
3081     if (res != MatchOperand_NoMatch) {
3082       return res;
3083     }
3084   }
3085   return MatchOperand_NoMatch;
3086 }
3087 
3088 OperandMatchResultTy AMDGPUAsmParser::parseOModOperand(OperandVector &Operands) {
3089   StringRef Name = Parser.getTok().getString();
3090   if (Name == "mul") {
3091     return parseIntWithPrefix("mul", Operands,
3092                               AMDGPUOperand::ImmTyOModSI, ConvertOmodMul);
3093   }
3094 
3095   if (Name == "div") {
3096     return parseIntWithPrefix("div", Operands,
3097                               AMDGPUOperand::ImmTyOModSI, ConvertOmodDiv);
3098   }
3099 
3100   return MatchOperand_NoMatch;
3101 }
3102 
3103 void AMDGPUAsmParser::cvtId(MCInst &Inst, const OperandVector &Operands) {
3104   unsigned I = 1;
3105   const MCInstrDesc &Desc = MII.get(Inst.getOpcode());
3106   for (unsigned J = 0; J < Desc.getNumDefs(); ++J) {
3107     ((AMDGPUOperand &)*Operands[I++]).addRegOperands(Inst, 1);
3108   }
3109   for (unsigned E = Operands.size(); I != E; ++I)
3110     ((AMDGPUOperand &)*Operands[I]).addRegOrImmOperands(Inst, 1);
3111 }
3112 
3113 void AMDGPUAsmParser::cvtVOP3_2_mod(MCInst &Inst, const OperandVector &Operands) {
3114   uint64_t TSFlags = MII.get(Inst.getOpcode()).TSFlags;
3115   if (TSFlags & SIInstrFlags::VOP3) {
3116     cvtVOP3(Inst, Operands);
3117   } else {
3118     cvtId(Inst, Operands);
3119   }
3120 }
3121 
3122 static bool isRegOrImmWithInputMods(const MCInstrDesc &Desc, unsigned OpNum) {
3123       // 1. This operand is input modifiers
3124   return Desc.OpInfo[OpNum].OperandType == AMDGPU::OPERAND_INPUT_MODS
3125       // 2. This is not last operand
3126       && Desc.NumOperands > (OpNum + 1)
3127       // 3. Next operand is register class
3128       && Desc.OpInfo[OpNum + 1].RegClass != -1
3129       // 4. Next register is not tied to any other operand
3130       && Desc.getOperandConstraint(OpNum + 1, MCOI::OperandConstraint::TIED_TO) == -1;
3131 }
3132 
3133 void AMDGPUAsmParser::cvtVOP3(MCInst &Inst, const OperandVector &Operands) {
3134   OptionalImmIndexMap OptionalIdx;
3135   unsigned I = 1;
3136   const MCInstrDesc &Desc = MII.get(Inst.getOpcode());
3137   for (unsigned J = 0; J < Desc.getNumDefs(); ++J) {
3138     ((AMDGPUOperand &)*Operands[I++]).addRegOperands(Inst, 1);
3139   }
3140 
3141   for (unsigned E = Operands.size(); I != E; ++I) {
3142     AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[I]);
3143     if (isRegOrImmWithInputMods(Desc, Inst.getNumOperands())) {
3144       Op.addRegOrImmWithFPInputModsOperands(Inst, 2);
3145     } else if (Op.isImm()) {
3146       OptionalIdx[Op.getImmTy()] = I;
3147     } else {
3148       llvm_unreachable("unhandled operand type");
3149     }
3150   }
3151 
3152   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyClampSI);
3153   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyOModSI);
3154 
3155   // special case v_mac_{f16, f32}:
3156   // it has src2 register operand that is tied to dst operand
3157   // we don't allow modifiers for this operand in assembler so src2_modifiers
3158   // should be 0
3159   if (Inst.getOpcode() == AMDGPU::V_MAC_F32_e64_si ||
3160       Inst.getOpcode() == AMDGPU::V_MAC_F32_e64_vi ||
3161       Inst.getOpcode() == AMDGPU::V_MAC_F16_e64_vi) {
3162     auto it = Inst.begin();
3163     std::advance(
3164       it,
3165       AMDGPU::getNamedOperandIdx(Inst.getOpcode() == AMDGPU::V_MAC_F16_e64_vi ?
3166                                      AMDGPU::V_MAC_F16_e64 :
3167                                      AMDGPU::V_MAC_F32_e64,
3168                                  AMDGPU::OpName::src2_modifiers));
3169     it = Inst.insert(it, MCOperand::createImm(0)); // no modifiers for src2
3170     ++it;
3171     Inst.insert(it, Inst.getOperand(0)); // src2 = dst
3172   }
3173 }
3174 
3175 //===----------------------------------------------------------------------===//
3176 // dpp
3177 //===----------------------------------------------------------------------===//
3178 
3179 bool AMDGPUOperand::isDPPCtrl() const {
3180   bool result = isImm() && getImmTy() == ImmTyDppCtrl && isUInt<9>(getImm());
3181   if (result) {
3182     int64_t Imm = getImm();
3183     return ((Imm >= 0x000) && (Imm <= 0x0ff)) ||
3184            ((Imm >= 0x101) && (Imm <= 0x10f)) ||
3185            ((Imm >= 0x111) && (Imm <= 0x11f)) ||
3186            ((Imm >= 0x121) && (Imm <= 0x12f)) ||
3187            (Imm == 0x130) ||
3188            (Imm == 0x134) ||
3189            (Imm == 0x138) ||
3190            (Imm == 0x13c) ||
3191            (Imm == 0x140) ||
3192            (Imm == 0x141) ||
3193            (Imm == 0x142) ||
3194            (Imm == 0x143);
3195   }
3196   return false;
3197 }
3198 
3199 bool AMDGPUOperand::isGPRIdxMode() const {
3200   return isImm() && isUInt<4>(getImm());
3201 }
3202 
3203 OperandMatchResultTy
3204 AMDGPUAsmParser::parseDPPCtrl(OperandVector &Operands) {
3205   SMLoc S = Parser.getTok().getLoc();
3206   StringRef Prefix;
3207   int64_t Int;
3208 
3209   if (getLexer().getKind() == AsmToken::Identifier) {
3210     Prefix = Parser.getTok().getString();
3211   } else {
3212     return MatchOperand_NoMatch;
3213   }
3214 
3215   if (Prefix == "row_mirror") {
3216     Int = 0x140;
3217     Parser.Lex();
3218   } else if (Prefix == "row_half_mirror") {
3219     Int = 0x141;
3220     Parser.Lex();
3221   } else {
3222     // Check to prevent parseDPPCtrlOps from eating invalid tokens
3223     if (Prefix != "quad_perm"
3224         && Prefix != "row_shl"
3225         && Prefix != "row_shr"
3226         && Prefix != "row_ror"
3227         && Prefix != "wave_shl"
3228         && Prefix != "wave_rol"
3229         && Prefix != "wave_shr"
3230         && Prefix != "wave_ror"
3231         && Prefix != "row_bcast") {
3232       return MatchOperand_NoMatch;
3233     }
3234 
3235     Parser.Lex();
3236     if (getLexer().isNot(AsmToken::Colon))
3237       return MatchOperand_ParseFail;
3238 
3239     if (Prefix == "quad_perm") {
3240       // quad_perm:[%d,%d,%d,%d]
3241       Parser.Lex();
3242       if (getLexer().isNot(AsmToken::LBrac))
3243         return MatchOperand_ParseFail;
3244       Parser.Lex();
3245 
3246       if (getParser().parseAbsoluteExpression(Int) || !(0 <= Int && Int <=3))
3247         return MatchOperand_ParseFail;
3248 
3249       for (int i = 0; i < 3; ++i) {
3250         if (getLexer().isNot(AsmToken::Comma))
3251           return MatchOperand_ParseFail;
3252         Parser.Lex();
3253 
3254         int64_t Temp;
3255         if (getParser().parseAbsoluteExpression(Temp) || !(0 <= Temp && Temp <=3))
3256           return MatchOperand_ParseFail;
3257         const int shift = i*2 + 2;
3258         Int += (Temp << shift);
3259       }
3260 
3261       if (getLexer().isNot(AsmToken::RBrac))
3262         return MatchOperand_ParseFail;
3263       Parser.Lex();
3264 
3265     } else {
3266       // sel:%d
3267       Parser.Lex();
3268       if (getParser().parseAbsoluteExpression(Int))
3269         return MatchOperand_ParseFail;
3270 
3271       if (Prefix == "row_shl" && 1 <= Int && Int <= 15) {
3272         Int |= 0x100;
3273       } else if (Prefix == "row_shr" && 1 <= Int && Int <= 15) {
3274         Int |= 0x110;
3275       } else if (Prefix == "row_ror" && 1 <= Int && Int <= 15) {
3276         Int |= 0x120;
3277       } else if (Prefix == "wave_shl" && 1 == Int) {
3278         Int = 0x130;
3279       } else if (Prefix == "wave_rol" && 1 == Int) {
3280         Int = 0x134;
3281       } else if (Prefix == "wave_shr" && 1 == Int) {
3282         Int = 0x138;
3283       } else if (Prefix == "wave_ror" && 1 == Int) {
3284         Int = 0x13C;
3285       } else if (Prefix == "row_bcast") {
3286         if (Int == 15) {
3287           Int = 0x142;
3288         } else if (Int == 31) {
3289           Int = 0x143;
3290         } else {
3291           return MatchOperand_ParseFail;
3292         }
3293       } else {
3294         return MatchOperand_ParseFail;
3295       }
3296     }
3297   }
3298 
3299   Operands.push_back(AMDGPUOperand::CreateImm(this, Int, S, AMDGPUOperand::ImmTyDppCtrl));
3300   return MatchOperand_Success;
3301 }
3302 
3303 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultRowMask() const {
3304   return AMDGPUOperand::CreateImm(this, 0xf, SMLoc(), AMDGPUOperand::ImmTyDppRowMask);
3305 }
3306 
3307 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultBankMask() const {
3308   return AMDGPUOperand::CreateImm(this, 0xf, SMLoc(), AMDGPUOperand::ImmTyDppBankMask);
3309 }
3310 
3311 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultBoundCtrl() const {
3312   return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTyDppBoundCtrl);
3313 }
3314 
3315 void AMDGPUAsmParser::cvtDPP(MCInst &Inst, const OperandVector &Operands) {
3316   OptionalImmIndexMap OptionalIdx;
3317 
3318   unsigned I = 1;
3319   const MCInstrDesc &Desc = MII.get(Inst.getOpcode());
3320   for (unsigned J = 0; J < Desc.getNumDefs(); ++J) {
3321     ((AMDGPUOperand &)*Operands[I++]).addRegOperands(Inst, 1);
3322   }
3323 
3324   for (unsigned E = Operands.size(); I != E; ++I) {
3325     AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[I]);
3326     // Add the register arguments
3327     if (isRegOrImmWithInputMods(Desc, Inst.getNumOperands())) {
3328       Op.addRegOrImmWithFPInputModsOperands(Inst, 2);
3329     } else if (Op.isDPPCtrl()) {
3330       Op.addImmOperands(Inst, 1);
3331     } else if (Op.isImm()) {
3332       // Handle optional arguments
3333       OptionalIdx[Op.getImmTy()] = I;
3334     } else {
3335       llvm_unreachable("Invalid operand type");
3336     }
3337   }
3338 
3339   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyDppRowMask, 0xf);
3340   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyDppBankMask, 0xf);
3341   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyDppBoundCtrl);
3342 
3343   // special case v_mac_{f16, f32}:
3344   // it has src2 register operand that is tied to dst operand
3345   if (Inst.getOpcode() == AMDGPU::V_MAC_F32_dpp ||
3346       Inst.getOpcode() == AMDGPU::V_MAC_F16_dpp) {
3347     auto it = Inst.begin();
3348     std::advance(
3349         it, AMDGPU::getNamedOperandIdx(Inst.getOpcode(), AMDGPU::OpName::src2));
3350     Inst.insert(it, Inst.getOperand(0)); // src2 = dst
3351   }
3352 }
3353 
3354 //===----------------------------------------------------------------------===//
3355 // sdwa
3356 //===----------------------------------------------------------------------===//
3357 
3358 OperandMatchResultTy
3359 AMDGPUAsmParser::parseSDWASel(OperandVector &Operands, StringRef Prefix,
3360                               AMDGPUOperand::ImmTy Type) {
3361   using namespace llvm::AMDGPU::SDWA;
3362 
3363   SMLoc S = Parser.getTok().getLoc();
3364   StringRef Value;
3365   OperandMatchResultTy res;
3366 
3367   res = parseStringWithPrefix(Prefix, Value);
3368   if (res != MatchOperand_Success) {
3369     return res;
3370   }
3371 
3372   int64_t Int;
3373   Int = StringSwitch<int64_t>(Value)
3374         .Case("BYTE_0", SdwaSel::BYTE_0)
3375         .Case("BYTE_1", SdwaSel::BYTE_1)
3376         .Case("BYTE_2", SdwaSel::BYTE_2)
3377         .Case("BYTE_3", SdwaSel::BYTE_3)
3378         .Case("WORD_0", SdwaSel::WORD_0)
3379         .Case("WORD_1", SdwaSel::WORD_1)
3380         .Case("DWORD", SdwaSel::DWORD)
3381         .Default(0xffffffff);
3382   Parser.Lex(); // eat last token
3383 
3384   if (Int == 0xffffffff) {
3385     return MatchOperand_ParseFail;
3386   }
3387 
3388   Operands.push_back(AMDGPUOperand::CreateImm(this, Int, S, Type));
3389   return MatchOperand_Success;
3390 }
3391 
3392 OperandMatchResultTy
3393 AMDGPUAsmParser::parseSDWADstUnused(OperandVector &Operands) {
3394   using namespace llvm::AMDGPU::SDWA;
3395 
3396   SMLoc S = Parser.getTok().getLoc();
3397   StringRef Value;
3398   OperandMatchResultTy res;
3399 
3400   res = parseStringWithPrefix("dst_unused", Value);
3401   if (res != MatchOperand_Success) {
3402     return res;
3403   }
3404 
3405   int64_t Int;
3406   Int = StringSwitch<int64_t>(Value)
3407         .Case("UNUSED_PAD", DstUnused::UNUSED_PAD)
3408         .Case("UNUSED_SEXT", DstUnused::UNUSED_SEXT)
3409         .Case("UNUSED_PRESERVE", DstUnused::UNUSED_PRESERVE)
3410         .Default(0xffffffff);
3411   Parser.Lex(); // eat last token
3412 
3413   if (Int == 0xffffffff) {
3414     return MatchOperand_ParseFail;
3415   }
3416 
3417   Operands.push_back(AMDGPUOperand::CreateImm(this, Int, S, AMDGPUOperand::ImmTySdwaDstUnused));
3418   return MatchOperand_Success;
3419 }
3420 
3421 void AMDGPUAsmParser::cvtSdwaVOP1(MCInst &Inst, const OperandVector &Operands) {
3422   cvtSDWA(Inst, Operands, SIInstrFlags::VOP1);
3423 }
3424 
3425 void AMDGPUAsmParser::cvtSdwaVOP2(MCInst &Inst, const OperandVector &Operands) {
3426   cvtSDWA(Inst, Operands, SIInstrFlags::VOP2);
3427 }
3428 
3429 void AMDGPUAsmParser::cvtSdwaVOPC(MCInst &Inst, const OperandVector &Operands) {
3430   cvtSDWA(Inst, Operands, SIInstrFlags::VOPC);
3431 }
3432 
3433 void AMDGPUAsmParser::cvtSDWA(MCInst &Inst, const OperandVector &Operands,
3434                               uint64_t BasicInstType) {
3435   OptionalImmIndexMap OptionalIdx;
3436 
3437   unsigned I = 1;
3438   const MCInstrDesc &Desc = MII.get(Inst.getOpcode());
3439   for (unsigned J = 0; J < Desc.getNumDefs(); ++J) {
3440     ((AMDGPUOperand &)*Operands[I++]).addRegOperands(Inst, 1);
3441   }
3442 
3443   for (unsigned E = Operands.size(); I != E; ++I) {
3444     AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[I]);
3445     // Add the register arguments
3446     if (BasicInstType == SIInstrFlags::VOPC &&
3447         Op.isReg() &&
3448         Op.Reg.RegNo == AMDGPU::VCC) {
3449       // VOPC sdwa use "vcc" token as dst. Skip it.
3450       continue;
3451     } else if (isRegOrImmWithInputMods(Desc, Inst.getNumOperands())) {
3452       Op.addRegOrImmWithInputModsOperands(Inst, 2);
3453     } else if (Op.isImm()) {
3454       // Handle optional arguments
3455       OptionalIdx[Op.getImmTy()] = I;
3456     } else {
3457       llvm_unreachable("Invalid operand type");
3458     }
3459   }
3460 
3461   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyClampSI, 0);
3462 
3463   if (Inst.getOpcode() != AMDGPU::V_NOP_sdwa) {
3464     // V_NOP_sdwa has no optional sdwa arguments
3465     switch (BasicInstType) {
3466     case SIInstrFlags::VOP1:
3467       addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySdwaDstSel, 6);
3468       addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySdwaDstUnused, 2);
3469       addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySdwaSrc0Sel, 6);
3470       break;
3471 
3472     case SIInstrFlags::VOP2:
3473       addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySdwaDstSel, 6);
3474       addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySdwaDstUnused, 2);
3475       addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySdwaSrc0Sel, 6);
3476       addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySdwaSrc1Sel, 6);
3477       break;
3478 
3479     case SIInstrFlags::VOPC:
3480       addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySdwaSrc0Sel, 6);
3481       addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySdwaSrc1Sel, 6);
3482       break;
3483 
3484     default:
3485       llvm_unreachable("Invalid instruction type. Only VOP1, VOP2 and VOPC allowed");
3486     }
3487   }
3488 
3489   // special case v_mac_{f16, f32}:
3490   // it has src2 register operand that is tied to dst operand
3491   if (Inst.getOpcode() == AMDGPU::V_MAC_F32_sdwa ||
3492       Inst.getOpcode() == AMDGPU::V_MAC_F16_sdwa)  {
3493     auto it = Inst.begin();
3494     std::advance(
3495         it, AMDGPU::getNamedOperandIdx(Inst.getOpcode(), AMDGPU::OpName::src2));
3496     Inst.insert(it, Inst.getOperand(0)); // src2 = dst
3497   }
3498 
3499 }
3500 
3501 /// Force static initialization.
3502 extern "C" void LLVMInitializeAMDGPUAsmParser() {
3503   RegisterMCAsmParser<AMDGPUAsmParser> A(getTheAMDGPUTarget());
3504   RegisterMCAsmParser<AMDGPUAsmParser> B(getTheGCNTarget());
3505 }
3506 
3507 #define GET_REGISTER_MATCHER
3508 #define GET_MATCHER_IMPLEMENTATION
3509 #include "AMDGPUGenAsmMatcher.inc"
3510 
3511 // This fuction should be defined after auto-generated include so that we have
3512 // MatchClassKind enum defined
3513 unsigned AMDGPUAsmParser::validateTargetOperandClass(MCParsedAsmOperand &Op,
3514                                                      unsigned Kind) {
3515   // Tokens like "glc" would be parsed as immediate operands in ParseOperand().
3516   // But MatchInstructionImpl() expects to meet token and fails to validate
3517   // operand. This method checks if we are given immediate operand but expect to
3518   // get corresponding token.
3519   AMDGPUOperand &Operand = (AMDGPUOperand&)Op;
3520   switch (Kind) {
3521   case MCK_addr64:
3522     return Operand.isAddr64() ? Match_Success : Match_InvalidOperand;
3523   case MCK_gds:
3524     return Operand.isGDS() ? Match_Success : Match_InvalidOperand;
3525   case MCK_glc:
3526     return Operand.isGLC() ? Match_Success : Match_InvalidOperand;
3527   case MCK_idxen:
3528     return Operand.isIdxen() ? Match_Success : Match_InvalidOperand;
3529   case MCK_offen:
3530     return Operand.isOffen() ? Match_Success : Match_InvalidOperand;
3531   case MCK_SSrcB32:
3532     // When operands have expression values, they will return true for isToken,
3533     // because it is not possible to distinguish between a token and an
3534     // expression at parse time. MatchInstructionImpl() will always try to
3535     // match an operand as a token, when isToken returns true, and when the
3536     // name of the expression is not a valid token, the match will fail,
3537     // so we need to handle it here.
3538     return Operand.isSSrcB32() ? Match_Success : Match_InvalidOperand;
3539   case MCK_SSrcF32:
3540     return Operand.isSSrcF32() ? Match_Success : Match_InvalidOperand;
3541   case MCK_SoppBrTarget:
3542     return Operand.isSoppBrTarget() ? Match_Success : Match_InvalidOperand;
3543   case MCK_VReg32OrOff:
3544     return Operand.isVReg32OrOff() ? Match_Success : Match_InvalidOperand;
3545   case MCK_InterpSlot:
3546     return Operand.isInterpSlot() ? Match_Success : Match_InvalidOperand;
3547   case MCK_Attr:
3548     return Operand.isInterpAttr() ? Match_Success : Match_InvalidOperand;
3549   case MCK_AttrChan:
3550     return Operand.isAttrChan() ? Match_Success : Match_InvalidOperand;
3551   default:
3552     return Match_InvalidOperand;
3553   }
3554 }
3555