1 //===-- AMDGPUAsmParser.cpp - Parse SI asm to MCInst instructions ---------===//
2 //
3 //                     The LLVM Compiler Infrastructure
4 //
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
7 //
8 //===----------------------------------------------------------------------===//
9 
10 #include "AMDKernelCodeT.h"
11 #include "MCTargetDesc/AMDGPUMCTargetDesc.h"
12 #include "MCTargetDesc/AMDGPUTargetStreamer.h"
13 #include "SIDefines.h"
14 #include "Utils/AMDGPUBaseInfo.h"
15 #include "Utils/AMDKernelCodeTUtils.h"
16 #include "Utils/AMDGPUAsmUtils.h"
17 #include "llvm/ADT/APFloat.h"
18 #include "llvm/ADT/APInt.h"
19 #include "llvm/ADT/SmallBitVector.h"
20 #include "llvm/ADT/SmallString.h"
21 #include "llvm/ADT/STLExtras.h"
22 #include "llvm/ADT/StringRef.h"
23 #include "llvm/ADT/StringSwitch.h"
24 #include "llvm/ADT/Twine.h"
25 #include "llvm/CodeGen/MachineValueType.h"
26 #include "llvm/MC/MCAsmInfo.h"
27 #include "llvm/MC/MCContext.h"
28 #include "llvm/MC/MCExpr.h"
29 #include "llvm/MC/MCInst.h"
30 #include "llvm/MC/MCInstrDesc.h"
31 #include "llvm/MC/MCInstrInfo.h"
32 #include "llvm/MC/MCParser/MCAsmLexer.h"
33 #include "llvm/MC/MCParser/MCAsmParser.h"
34 #include "llvm/MC/MCParser/MCAsmParserExtension.h"
35 #include "llvm/MC/MCParser/MCParsedAsmOperand.h"
36 #include "llvm/MC/MCParser/MCTargetAsmParser.h"
37 #include "llvm/MC/MCRegisterInfo.h"
38 #include "llvm/MC/MCStreamer.h"
39 #include "llvm/MC/MCSubtargetInfo.h"
40 #include "llvm/MC/MCSymbol.h"
41 #include "llvm/Support/Casting.h"
42 #include "llvm/Support/Debug.h"
43 #include "llvm/Support/ELF.h"
44 #include "llvm/Support/ErrorHandling.h"
45 #include "llvm/Support/MathExtras.h"
46 #include "llvm/Support/raw_ostream.h"
47 #include "llvm/Support/SMLoc.h"
48 #include "llvm/Support/TargetRegistry.h"
49 #include "llvm/Support/raw_ostream.h"
50 #include "llvm/Support/MathExtras.h"
51 #include <algorithm>
52 #include <cassert>
53 #include <cstdint>
54 #include <cstring>
55 #include <iterator>
56 #include <map>
57 #include <memory>
58 #include <string>
59 #include <vector>
60 
61 using namespace llvm;
62 using namespace llvm::AMDGPU;
63 
64 namespace {
65 
66 class AMDGPUAsmParser;
67 
68 enum RegisterKind { IS_UNKNOWN, IS_VGPR, IS_SGPR, IS_TTMP, IS_SPECIAL };
69 
70 //===----------------------------------------------------------------------===//
71 // Operand
72 //===----------------------------------------------------------------------===//
73 
74 class AMDGPUOperand : public MCParsedAsmOperand {
75   enum KindTy {
76     Token,
77     Immediate,
78     Register,
79     Expression
80   } Kind;
81 
82   SMLoc StartLoc, EndLoc;
83   const AMDGPUAsmParser *AsmParser;
84 
85 public:
86   AMDGPUOperand(enum KindTy Kind_, const AMDGPUAsmParser *AsmParser_)
87     : MCParsedAsmOperand(), Kind(Kind_), AsmParser(AsmParser_) {}
88 
89   typedef std::unique_ptr<AMDGPUOperand> Ptr;
90 
91   struct Modifiers {
92     bool Abs = false;
93     bool Neg = false;
94     bool Sext = false;
95 
96     bool hasFPModifiers() const { return Abs || Neg; }
97     bool hasIntModifiers() const { return Sext; }
98     bool hasModifiers() const { return hasFPModifiers() || hasIntModifiers(); }
99 
100     int64_t getFPModifiersOperand() const {
101       int64_t Operand = 0;
102       Operand |= Abs ? SISrcMods::ABS : 0;
103       Operand |= Neg ? SISrcMods::NEG : 0;
104       return Operand;
105     }
106 
107     int64_t getIntModifiersOperand() const {
108       int64_t Operand = 0;
109       Operand |= Sext ? SISrcMods::SEXT : 0;
110       return Operand;
111     }
112 
113     int64_t getModifiersOperand() const {
114       assert(!(hasFPModifiers() && hasIntModifiers())
115            && "fp and int modifiers should not be used simultaneously");
116       if (hasFPModifiers()) {
117         return getFPModifiersOperand();
118       } else if (hasIntModifiers()) {
119         return getIntModifiersOperand();
120       } else {
121         return 0;
122       }
123     }
124 
125     friend raw_ostream &operator <<(raw_ostream &OS, AMDGPUOperand::Modifiers Mods);
126   };
127 
128   enum ImmTy {
129     ImmTyNone,
130     ImmTyGDS,
131     ImmTyOffen,
132     ImmTyIdxen,
133     ImmTyAddr64,
134     ImmTyOffset,
135     ImmTyOffset0,
136     ImmTyOffset1,
137     ImmTyGLC,
138     ImmTySLC,
139     ImmTyTFE,
140     ImmTyClampSI,
141     ImmTyOModSI,
142     ImmTyDppCtrl,
143     ImmTyDppRowMask,
144     ImmTyDppBankMask,
145     ImmTyDppBoundCtrl,
146     ImmTySdwaDstSel,
147     ImmTySdwaSrc0Sel,
148     ImmTySdwaSrc1Sel,
149     ImmTySdwaDstUnused,
150     ImmTyDMask,
151     ImmTyUNorm,
152     ImmTyDA,
153     ImmTyR128,
154     ImmTyLWE,
155     ImmTyExpTgt,
156     ImmTyExpCompr,
157     ImmTyExpVM,
158     ImmTyHwreg,
159     ImmTyOff,
160     ImmTySendMsg,
161     ImmTyInterpSlot,
162     ImmTyInterpAttr,
163     ImmTyAttrChan
164   };
165 
166   struct TokOp {
167     const char *Data;
168     unsigned Length;
169   };
170 
171   struct ImmOp {
172     int64_t Val;
173     ImmTy Type;
174     bool IsFPImm;
175     Modifiers Mods;
176   };
177 
178   struct RegOp {
179     unsigned RegNo;
180     bool IsForcedVOP3;
181     Modifiers Mods;
182   };
183 
184   union {
185     TokOp Tok;
186     ImmOp Imm;
187     RegOp Reg;
188     const MCExpr *Expr;
189   };
190 
191   bool isToken() const override {
192     if (Kind == Token)
193       return true;
194 
195     if (Kind != Expression || !Expr)
196       return false;
197 
198     // When parsing operands, we can't always tell if something was meant to be
199     // a token, like 'gds', or an expression that references a global variable.
200     // In this case, we assume the string is an expression, and if we need to
201     // interpret is a token, then we treat the symbol name as the token.
202     return isa<MCSymbolRefExpr>(Expr);
203   }
204 
205   bool isImm() const override {
206     return Kind == Immediate;
207   }
208 
209   bool isInlinableImm(MVT type) const;
210   bool isLiteralImm(MVT type) const;
211 
212   bool isRegKind() const {
213     return Kind == Register;
214   }
215 
216   bool isReg() const override {
217     return isRegKind() && !Reg.Mods.hasModifiers();
218   }
219 
220   bool isRegOrImmWithInputMods(MVT type) const {
221     return isRegKind() || isInlinableImm(type);
222   }
223 
224   bool isRegOrImmWithInt16InputMods() const {
225     return isRegOrImmWithInputMods(MVT::i16);
226   }
227 
228   bool isRegOrImmWithInt32InputMods() const {
229     return isRegOrImmWithInputMods(MVT::i32);
230   }
231 
232   bool isRegOrImmWithInt64InputMods() const {
233     return isRegOrImmWithInputMods(MVT::i64);
234   }
235 
236   bool isRegOrImmWithFP16InputMods() const {
237     return isRegOrImmWithInputMods(MVT::f16);
238   }
239 
240   bool isRegOrImmWithFP32InputMods() const {
241     return isRegOrImmWithInputMods(MVT::f32);
242   }
243 
244   bool isRegOrImmWithFP64InputMods() const {
245     return isRegOrImmWithInputMods(MVT::f64);
246   }
247 
248   bool isVReg32OrOff() const {
249     return isOff() || isRegClass(AMDGPU::VGPR_32RegClassID);
250   }
251 
252   bool isImmTy(ImmTy ImmT) const {
253     return isImm() && Imm.Type == ImmT;
254   }
255 
256   bool isImmModifier() const {
257     return isImm() && Imm.Type != ImmTyNone;
258   }
259 
260   bool isClampSI() const { return isImmTy(ImmTyClampSI); }
261   bool isOModSI() const { return isImmTy(ImmTyOModSI); }
262   bool isDMask() const { return isImmTy(ImmTyDMask); }
263   bool isUNorm() const { return isImmTy(ImmTyUNorm); }
264   bool isDA() const { return isImmTy(ImmTyDA); }
265   bool isR128() const { return isImmTy(ImmTyUNorm); }
266   bool isLWE() const { return isImmTy(ImmTyLWE); }
267   bool isOff() const { return isImmTy(ImmTyOff); }
268   bool isExpTgt() const { return isImmTy(ImmTyExpTgt); }
269   bool isExpVM() const { return isImmTy(ImmTyExpVM); }
270   bool isExpCompr() const { return isImmTy(ImmTyExpCompr); }
271   bool isOffen() const { return isImmTy(ImmTyOffen); }
272   bool isIdxen() const { return isImmTy(ImmTyIdxen); }
273   bool isAddr64() const { return isImmTy(ImmTyAddr64); }
274   bool isOffset() const { return isImmTy(ImmTyOffset) && isUInt<16>(getImm()); }
275   bool isOffset0() const { return isImmTy(ImmTyOffset0) && isUInt<16>(getImm()); }
276   bool isOffset1() const { return isImmTy(ImmTyOffset1) && isUInt<8>(getImm()); }
277   bool isGDS() const { return isImmTy(ImmTyGDS); }
278   bool isGLC() const { return isImmTy(ImmTyGLC); }
279   bool isSLC() const { return isImmTy(ImmTySLC); }
280   bool isTFE() const { return isImmTy(ImmTyTFE); }
281   bool isBankMask() const { return isImmTy(ImmTyDppBankMask); }
282   bool isRowMask() const { return isImmTy(ImmTyDppRowMask); }
283   bool isBoundCtrl() const { return isImmTy(ImmTyDppBoundCtrl); }
284   bool isSDWADstSel() const { return isImmTy(ImmTySdwaDstSel); }
285   bool isSDWASrc0Sel() const { return isImmTy(ImmTySdwaSrc0Sel); }
286   bool isSDWASrc1Sel() const { return isImmTy(ImmTySdwaSrc1Sel); }
287   bool isSDWADstUnused() const { return isImmTy(ImmTySdwaDstUnused); }
288   bool isInterpSlot() const { return isImmTy(ImmTyInterpSlot); }
289   bool isInterpAttr() const { return isImmTy(ImmTyInterpAttr); }
290   bool isAttrChan() const { return isImmTy(ImmTyAttrChan); }
291 
292   bool isMod() const {
293     return isClampSI() || isOModSI();
294   }
295 
296   bool isRegOrImm() const {
297     return isReg() || isImm();
298   }
299 
300   bool isRegClass(unsigned RCID) const;
301 
302   bool isSCSrcB16() const {
303     return isRegClass(AMDGPU::SReg_32RegClassID) || isInlinableImm(MVT::i16);
304   }
305 
306   bool isSCSrcB32() const {
307     return isRegClass(AMDGPU::SReg_32RegClassID) || isInlinableImm(MVT::i32);
308   }
309 
310   bool isSCSrcB64() const {
311     return isRegClass(AMDGPU::SReg_64RegClassID) || isInlinableImm(MVT::i64);
312   }
313 
314   bool isSCSrcF16() const {
315     return isRegClass(AMDGPU::SReg_32RegClassID) || isInlinableImm(MVT::f16);
316   }
317 
318   bool isSCSrcF32() const {
319     return isRegClass(AMDGPU::SReg_32RegClassID) || isInlinableImm(MVT::f32);
320   }
321 
322   bool isSCSrcF64() const {
323     return isRegClass(AMDGPU::SReg_64RegClassID) || isInlinableImm(MVT::f64);
324   }
325 
326   bool isSSrcB32() const {
327     return isSCSrcB32() || isLiteralImm(MVT::i32) || isExpr();
328   }
329 
330   bool isSSrcB16() const {
331     return isSCSrcB16() || isLiteralImm(MVT::i16);
332   }
333 
334   bool isSSrcB64() const {
335     // TODO: Find out how SALU supports extension of 32-bit literals to 64 bits.
336     // See isVSrc64().
337     return isSCSrcB64() || isLiteralImm(MVT::i64);
338   }
339 
340   bool isSSrcF32() const {
341     return isSCSrcB32() || isLiteralImm(MVT::f32) || isExpr();
342   }
343 
344   bool isSSrcF64() const {
345     return isSCSrcB64() || isLiteralImm(MVT::f64);
346   }
347 
348   bool isSSrcF16() const {
349     return isSCSrcB16() || isLiteralImm(MVT::f16);
350   }
351 
352   bool isVCSrcB32() const {
353     return isRegClass(AMDGPU::VS_32RegClassID) || isInlinableImm(MVT::i32);
354   }
355 
356   bool isVCSrcB64() const {
357     return isRegClass(AMDGPU::VS_64RegClassID) || isInlinableImm(MVT::i64);
358   }
359 
360   bool isVCSrcB16() const {
361     return isRegClass(AMDGPU::VS_32RegClassID) || isInlinableImm(MVT::i16);
362   }
363 
364   bool isVCSrcF32() const {
365     return isRegClass(AMDGPU::VS_32RegClassID) || isInlinableImm(MVT::f32);
366   }
367 
368   bool isVCSrcF64() const {
369     return isRegClass(AMDGPU::VS_64RegClassID) || isInlinableImm(MVT::f64);
370   }
371 
372   bool isVCSrcF16() const {
373     return isRegClass(AMDGPU::VS_32RegClassID) || isInlinableImm(MVT::f16);
374   }
375 
376   bool isVSrcB32() const {
377     return isVCSrcF32() || isLiteralImm(MVT::i32);
378   }
379 
380   bool isVSrcB64() const {
381     return isVCSrcF64() || isLiteralImm(MVT::i64);
382   }
383 
384   bool isVSrcB16() const {
385     return isVCSrcF16() || isLiteralImm(MVT::i16);
386   }
387 
388   bool isVSrcF32() const {
389     return isVCSrcF32() || isLiteralImm(MVT::f32);
390   }
391 
392   bool isVSrcF64() const {
393     return isVCSrcF64() || isLiteralImm(MVT::f64);
394   }
395 
396   bool isVSrcF16() const {
397     return isVCSrcF16() || isLiteralImm(MVT::f16);
398   }
399 
400   bool isKImmFP32() const {
401     return isLiteralImm(MVT::f32);
402   }
403 
404   bool isKImmFP16() const {
405     return isLiteralImm(MVT::f16);
406   }
407 
408   bool isMem() const override {
409     return false;
410   }
411 
412   bool isExpr() const {
413     return Kind == Expression;
414   }
415 
416   bool isSoppBrTarget() const {
417     return isExpr() || isImm();
418   }
419 
420   bool isSWaitCnt() const;
421   bool isHwreg() const;
422   bool isSendMsg() const;
423   bool isSMRDOffset8() const;
424   bool isSMRDOffset20() const;
425   bool isSMRDLiteralOffset() const;
426   bool isDPPCtrl() const;
427   bool isGPRIdxMode() const;
428 
429   StringRef getExpressionAsToken() const {
430     assert(isExpr());
431     const MCSymbolRefExpr *S = cast<MCSymbolRefExpr>(Expr);
432     return S->getSymbol().getName();
433   }
434 
435   StringRef getToken() const {
436     assert(isToken());
437 
438     if (Kind == Expression)
439       return getExpressionAsToken();
440 
441     return StringRef(Tok.Data, Tok.Length);
442   }
443 
444   int64_t getImm() const {
445     assert(isImm());
446     return Imm.Val;
447   }
448 
449   enum ImmTy getImmTy() const {
450     assert(isImm());
451     return Imm.Type;
452   }
453 
454   unsigned getReg() const override {
455     return Reg.RegNo;
456   }
457 
458   SMLoc getStartLoc() const override {
459     return StartLoc;
460   }
461 
462   SMLoc getEndLoc() const override {
463     return EndLoc;
464   }
465 
466   Modifiers getModifiers() const {
467     assert(isRegKind() || isImmTy(ImmTyNone));
468     return isRegKind() ? Reg.Mods : Imm.Mods;
469   }
470 
471   void setModifiers(Modifiers Mods) {
472     assert(isRegKind() || isImmTy(ImmTyNone));
473     if (isRegKind())
474       Reg.Mods = Mods;
475     else
476       Imm.Mods = Mods;
477   }
478 
479   bool hasModifiers() const {
480     return getModifiers().hasModifiers();
481   }
482 
483   bool hasFPModifiers() const {
484     return getModifiers().hasFPModifiers();
485   }
486 
487   bool hasIntModifiers() const {
488     return getModifiers().hasIntModifiers();
489   }
490 
491   void addImmOperands(MCInst &Inst, unsigned N, bool ApplyModifiers = true) const;
492 
493   void addLiteralImmOperand(MCInst &Inst, int64_t Val) const;
494 
495   template <unsigned Bitwidth>
496   void addKImmFPOperands(MCInst &Inst, unsigned N) const;
497 
498   void addKImmFP16Operands(MCInst &Inst, unsigned N) const {
499     addKImmFPOperands<16>(Inst, N);
500   }
501 
502   void addKImmFP32Operands(MCInst &Inst, unsigned N) const {
503     addKImmFPOperands<32>(Inst, N);
504   }
505 
506   void addRegOperands(MCInst &Inst, unsigned N) const;
507 
508   void addRegOrImmOperands(MCInst &Inst, unsigned N) const {
509     if (isRegKind())
510       addRegOperands(Inst, N);
511     else if (isExpr())
512       Inst.addOperand(MCOperand::createExpr(Expr));
513     else
514       addImmOperands(Inst, N);
515   }
516 
517   void addRegOrImmWithInputModsOperands(MCInst &Inst, unsigned N) const {
518     Modifiers Mods = getModifiers();
519     Inst.addOperand(MCOperand::createImm(Mods.getModifiersOperand()));
520     if (isRegKind()) {
521       addRegOperands(Inst, N);
522     } else {
523       addImmOperands(Inst, N, false);
524     }
525   }
526 
527   void addRegOrImmWithFPInputModsOperands(MCInst &Inst, unsigned N) const {
528     assert(!hasIntModifiers());
529     addRegOrImmWithInputModsOperands(Inst, N);
530   }
531 
532   void addRegOrImmWithIntInputModsOperands(MCInst &Inst, unsigned N) const {
533     assert(!hasFPModifiers());
534     addRegOrImmWithInputModsOperands(Inst, N);
535   }
536 
537   void addSoppBrTargetOperands(MCInst &Inst, unsigned N) const {
538     if (isImm())
539       addImmOperands(Inst, N);
540     else {
541       assert(isExpr());
542       Inst.addOperand(MCOperand::createExpr(Expr));
543     }
544   }
545 
546   static void printImmTy(raw_ostream& OS, ImmTy Type) {
547     switch (Type) {
548     case ImmTyNone: OS << "None"; break;
549     case ImmTyGDS: OS << "GDS"; break;
550     case ImmTyOffen: OS << "Offen"; break;
551     case ImmTyIdxen: OS << "Idxen"; break;
552     case ImmTyAddr64: OS << "Addr64"; break;
553     case ImmTyOffset: OS << "Offset"; break;
554     case ImmTyOffset0: OS << "Offset0"; break;
555     case ImmTyOffset1: OS << "Offset1"; break;
556     case ImmTyGLC: OS << "GLC"; break;
557     case ImmTySLC: OS << "SLC"; break;
558     case ImmTyTFE: OS << "TFE"; break;
559     case ImmTyClampSI: OS << "ClampSI"; break;
560     case ImmTyOModSI: OS << "OModSI"; break;
561     case ImmTyDppCtrl: OS << "DppCtrl"; break;
562     case ImmTyDppRowMask: OS << "DppRowMask"; break;
563     case ImmTyDppBankMask: OS << "DppBankMask"; break;
564     case ImmTyDppBoundCtrl: OS << "DppBoundCtrl"; break;
565     case ImmTySdwaDstSel: OS << "SdwaDstSel"; break;
566     case ImmTySdwaSrc0Sel: OS << "SdwaSrc0Sel"; break;
567     case ImmTySdwaSrc1Sel: OS << "SdwaSrc1Sel"; break;
568     case ImmTySdwaDstUnused: OS << "SdwaDstUnused"; break;
569     case ImmTyDMask: OS << "DMask"; break;
570     case ImmTyUNorm: OS << "UNorm"; break;
571     case ImmTyDA: OS << "DA"; break;
572     case ImmTyR128: OS << "R128"; break;
573     case ImmTyLWE: OS << "LWE"; break;
574     case ImmTyOff: OS << "Off"; break;
575     case ImmTyExpTgt: OS << "ExpTgt"; break;
576     case ImmTyExpCompr: OS << "ExpCompr"; break;
577     case ImmTyExpVM: OS << "ExpVM"; break;
578     case ImmTyHwreg: OS << "Hwreg"; break;
579     case ImmTySendMsg: OS << "SendMsg"; break;
580     case ImmTyInterpSlot: OS << "InterpSlot"; break;
581     case ImmTyInterpAttr: OS << "InterpAttr"; break;
582     case ImmTyAttrChan: OS << "AttrChan"; break;
583     }
584   }
585 
586   void print(raw_ostream &OS) const override {
587     switch (Kind) {
588     case Register:
589       OS << "<register " << getReg() << " mods: " << Reg.Mods << '>';
590       break;
591     case Immediate:
592       OS << '<' << getImm();
593       if (getImmTy() != ImmTyNone) {
594         OS << " type: "; printImmTy(OS, getImmTy());
595       }
596       OS << " mods: " << Imm.Mods << '>';
597       break;
598     case Token:
599       OS << '\'' << getToken() << '\'';
600       break;
601     case Expression:
602       OS << "<expr " << *Expr << '>';
603       break;
604     }
605   }
606 
607   static AMDGPUOperand::Ptr CreateImm(const AMDGPUAsmParser *AsmParser,
608                                       int64_t Val, SMLoc Loc,
609                                       enum ImmTy Type = ImmTyNone,
610                                       bool IsFPImm = false) {
611     auto Op = llvm::make_unique<AMDGPUOperand>(Immediate, AsmParser);
612     Op->Imm.Val = Val;
613     Op->Imm.IsFPImm = IsFPImm;
614     Op->Imm.Type = Type;
615     Op->Imm.Mods = Modifiers();
616     Op->StartLoc = Loc;
617     Op->EndLoc = Loc;
618     return Op;
619   }
620 
621   static AMDGPUOperand::Ptr CreateToken(const AMDGPUAsmParser *AsmParser,
622                                         StringRef Str, SMLoc Loc,
623                                         bool HasExplicitEncodingSize = true) {
624     auto Res = llvm::make_unique<AMDGPUOperand>(Token, AsmParser);
625     Res->Tok.Data = Str.data();
626     Res->Tok.Length = Str.size();
627     Res->StartLoc = Loc;
628     Res->EndLoc = Loc;
629     return Res;
630   }
631 
632   static AMDGPUOperand::Ptr CreateReg(const AMDGPUAsmParser *AsmParser,
633                                       unsigned RegNo, SMLoc S,
634                                       SMLoc E,
635                                       bool ForceVOP3) {
636     auto Op = llvm::make_unique<AMDGPUOperand>(Register, AsmParser);
637     Op->Reg.RegNo = RegNo;
638     Op->Reg.Mods = Modifiers();
639     Op->Reg.IsForcedVOP3 = ForceVOP3;
640     Op->StartLoc = S;
641     Op->EndLoc = E;
642     return Op;
643   }
644 
645   static AMDGPUOperand::Ptr CreateExpr(const AMDGPUAsmParser *AsmParser,
646                                        const class MCExpr *Expr, SMLoc S) {
647     auto Op = llvm::make_unique<AMDGPUOperand>(Expression, AsmParser);
648     Op->Expr = Expr;
649     Op->StartLoc = S;
650     Op->EndLoc = S;
651     return Op;
652   }
653 };
654 
655 raw_ostream &operator <<(raw_ostream &OS, AMDGPUOperand::Modifiers Mods) {
656   OS << "abs:" << Mods.Abs << " neg: " << Mods.Neg << " sext:" << Mods.Sext;
657   return OS;
658 }
659 
660 //===----------------------------------------------------------------------===//
661 // AsmParser
662 //===----------------------------------------------------------------------===//
663 
664 class AMDGPUAsmParser : public MCTargetAsmParser {
665   const MCInstrInfo &MII;
666   MCAsmParser &Parser;
667 
668   unsigned ForcedEncodingSize;
669   bool ForcedDPP;
670   bool ForcedSDWA;
671 
672   /// @name Auto-generated Match Functions
673   /// {
674 
675 #define GET_ASSEMBLER_HEADER
676 #include "AMDGPUGenAsmMatcher.inc"
677 
678   /// }
679 
680 private:
681   bool ParseDirectiveMajorMinor(uint32_t &Major, uint32_t &Minor);
682   bool ParseDirectiveHSACodeObjectVersion();
683   bool ParseDirectiveHSACodeObjectISA();
684   bool ParseDirectiveRuntimeMetadata();
685   bool ParseAMDKernelCodeTValue(StringRef ID, amd_kernel_code_t &Header);
686   bool ParseDirectiveAMDKernelCodeT();
687   bool ParseSectionDirectiveHSAText();
688   bool subtargetHasRegister(const MCRegisterInfo &MRI, unsigned RegNo) const;
689   bool ParseDirectiveAMDGPUHsaKernel();
690   bool ParseDirectiveAMDGPUHsaModuleGlobal();
691   bool ParseDirectiveAMDGPUHsaProgramGlobal();
692   bool ParseSectionDirectiveHSADataGlobalAgent();
693   bool ParseSectionDirectiveHSADataGlobalProgram();
694   bool ParseSectionDirectiveHSARodataReadonlyAgent();
695   bool AddNextRegisterToList(unsigned& Reg, unsigned& RegWidth, RegisterKind RegKind, unsigned Reg1, unsigned RegNum);
696   bool ParseAMDGPURegister(RegisterKind& RegKind, unsigned& Reg, unsigned& RegNum, unsigned& RegWidth);
697   void cvtMubufImpl(MCInst &Inst, const OperandVector &Operands, bool IsAtomic, bool IsAtomicReturn);
698 
699 public:
700   enum AMDGPUMatchResultTy {
701     Match_PreferE32 = FIRST_TARGET_MATCH_RESULT_TY
702   };
703 
704   AMDGPUAsmParser(const MCSubtargetInfo &STI, MCAsmParser &_Parser,
705                const MCInstrInfo &MII,
706                const MCTargetOptions &Options)
707       : MCTargetAsmParser(Options, STI), MII(MII), Parser(_Parser),
708         ForcedEncodingSize(0),
709         ForcedDPP(false),
710         ForcedSDWA(false) {
711     MCAsmParserExtension::Initialize(Parser);
712 
713     if (getSTI().getFeatureBits().none()) {
714       // Set default features.
715       copySTI().ToggleFeature("SOUTHERN_ISLANDS");
716     }
717 
718     setAvailableFeatures(ComputeAvailableFeatures(getSTI().getFeatureBits()));
719 
720     {
721       // TODO: make those pre-defined variables read-only.
722       // Currently there is none suitable machinery in the core llvm-mc for this.
723       // MCSymbol::isRedefinable is intended for another purpose, and
724       // AsmParser::parseDirectiveSet() cannot be specialized for specific target.
725       AMDGPU::IsaVersion Isa = AMDGPU::getIsaVersion(getSTI().getFeatureBits());
726       MCContext &Ctx = getContext();
727       MCSymbol *Sym = Ctx.getOrCreateSymbol(Twine(".option.machine_version_major"));
728       Sym->setVariableValue(MCConstantExpr::create(Isa.Major, Ctx));
729       Sym = Ctx.getOrCreateSymbol(Twine(".option.machine_version_minor"));
730       Sym->setVariableValue(MCConstantExpr::create(Isa.Minor, Ctx));
731       Sym = Ctx.getOrCreateSymbol(Twine(".option.machine_version_stepping"));
732       Sym->setVariableValue(MCConstantExpr::create(Isa.Stepping, Ctx));
733     }
734   }
735 
736   bool isSI() const {
737     return AMDGPU::isSI(getSTI());
738   }
739 
740   bool isCI() const {
741     return AMDGPU::isCI(getSTI());
742   }
743 
744   bool isVI() const {
745     return AMDGPU::isVI(getSTI());
746   }
747 
748   bool hasInv2PiInlineImm() const {
749     return getSTI().getFeatureBits()[AMDGPU::FeatureInv2PiInlineImm];
750   }
751 
752   bool hasSGPR102_SGPR103() const {
753     return !isVI();
754   }
755 
756   AMDGPUTargetStreamer &getTargetStreamer() {
757     MCTargetStreamer &TS = *getParser().getStreamer().getTargetStreamer();
758     return static_cast<AMDGPUTargetStreamer &>(TS);
759   }
760 
761   const MCRegisterInfo *getMRI() const {
762     // We need this const_cast because for some reason getContext() is not const
763     // in MCAsmParser.
764     return const_cast<AMDGPUAsmParser*>(this)->getContext().getRegisterInfo();
765   }
766 
767   const MCInstrInfo *getMII() const {
768     return &MII;
769   }
770 
771   void setForcedEncodingSize(unsigned Size) { ForcedEncodingSize = Size; }
772   void setForcedDPP(bool ForceDPP_) { ForcedDPP = ForceDPP_; }
773   void setForcedSDWA(bool ForceSDWA_) { ForcedSDWA = ForceSDWA_; }
774 
775   unsigned getForcedEncodingSize() const { return ForcedEncodingSize; }
776   bool isForcedVOP3() const { return ForcedEncodingSize == 64; }
777   bool isForcedDPP() const { return ForcedDPP; }
778   bool isForcedSDWA() const { return ForcedSDWA; }
779 
780   std::unique_ptr<AMDGPUOperand> parseRegister();
781   bool ParseRegister(unsigned &RegNo, SMLoc &StartLoc, SMLoc &EndLoc) override;
782   unsigned checkTargetMatchPredicate(MCInst &Inst) override;
783   unsigned validateTargetOperandClass(MCParsedAsmOperand &Op,
784                                       unsigned Kind) override;
785   bool MatchAndEmitInstruction(SMLoc IDLoc, unsigned &Opcode,
786                                OperandVector &Operands, MCStreamer &Out,
787                                uint64_t &ErrorInfo,
788                                bool MatchingInlineAsm) override;
789   bool ParseDirective(AsmToken DirectiveID) override;
790   OperandMatchResultTy parseOperand(OperandVector &Operands, StringRef Mnemonic);
791   StringRef parseMnemonicSuffix(StringRef Name);
792   bool ParseInstruction(ParseInstructionInfo &Info, StringRef Name,
793                         SMLoc NameLoc, OperandVector &Operands) override;
794   //bool ProcessInstruction(MCInst &Inst);
795 
796   OperandMatchResultTy parseIntWithPrefix(const char *Prefix, int64_t &Int);
797   OperandMatchResultTy
798   parseIntWithPrefix(const char *Prefix, OperandVector &Operands,
799                      enum AMDGPUOperand::ImmTy ImmTy = AMDGPUOperand::ImmTyNone,
800                      bool (*ConvertResult)(int64_t &) = nullptr);
801   OperandMatchResultTy
802   parseNamedBit(const char *Name, OperandVector &Operands,
803                 enum AMDGPUOperand::ImmTy ImmTy = AMDGPUOperand::ImmTyNone);
804   OperandMatchResultTy parseStringWithPrefix(StringRef Prefix,
805                                              StringRef &Value);
806 
807   OperandMatchResultTy parseImm(OperandVector &Operands);
808   OperandMatchResultTy parseRegOrImm(OperandVector &Operands);
809   OperandMatchResultTy parseRegOrImmWithFPInputMods(OperandVector &Operands);
810   OperandMatchResultTy parseRegOrImmWithIntInputMods(OperandVector &Operands);
811   OperandMatchResultTy parseVReg32OrOff(OperandVector &Operands);
812 
813   void cvtDSOffset01(MCInst &Inst, const OperandVector &Operands);
814   void cvtDS(MCInst &Inst, const OperandVector &Operands);
815   void cvtExp(MCInst &Inst, const OperandVector &Operands);
816 
817   bool parseCnt(int64_t &IntVal);
818   OperandMatchResultTy parseSWaitCntOps(OperandVector &Operands);
819   OperandMatchResultTy parseHwreg(OperandVector &Operands);
820 
821 private:
822   struct OperandInfoTy {
823     int64_t Id;
824     bool IsSymbolic;
825     OperandInfoTy(int64_t Id_) : Id(Id_), IsSymbolic(false) { }
826   };
827 
828   bool parseSendMsgConstruct(OperandInfoTy &Msg, OperandInfoTy &Operation, int64_t &StreamId);
829   bool parseHwregConstruct(OperandInfoTy &HwReg, int64_t &Offset, int64_t &Width);
830 
831   void errorExpTgt();
832   OperandMatchResultTy parseExpTgtImpl(StringRef Str, uint8_t &Val);
833 
834 public:
835   OperandMatchResultTy parseOptionalOperand(OperandVector &Operands);
836 
837   OperandMatchResultTy parseExpTgt(OperandVector &Operands);
838   OperandMatchResultTy parseSendMsgOp(OperandVector &Operands);
839   OperandMatchResultTy parseInterpSlot(OperandVector &Operands);
840   OperandMatchResultTy parseInterpAttr(OperandVector &Operands);
841   OperandMatchResultTy parseSOppBrTarget(OperandVector &Operands);
842 
843   void cvtMubuf(MCInst &Inst, const OperandVector &Operands) { cvtMubufImpl(Inst, Operands, false, false); }
844   void cvtMubufAtomic(MCInst &Inst, const OperandVector &Operands) { cvtMubufImpl(Inst, Operands, true, false); }
845   void cvtMubufAtomicReturn(MCInst &Inst, const OperandVector &Operands) { cvtMubufImpl(Inst, Operands, true, true); }
846   AMDGPUOperand::Ptr defaultGLC() const;
847   AMDGPUOperand::Ptr defaultSLC() const;
848   AMDGPUOperand::Ptr defaultTFE() const;
849 
850   AMDGPUOperand::Ptr defaultDMask() const;
851   AMDGPUOperand::Ptr defaultUNorm() const;
852   AMDGPUOperand::Ptr defaultDA() const;
853   AMDGPUOperand::Ptr defaultR128() const;
854   AMDGPUOperand::Ptr defaultLWE() const;
855   AMDGPUOperand::Ptr defaultSMRDOffset8() const;
856   AMDGPUOperand::Ptr defaultSMRDOffset20() const;
857   AMDGPUOperand::Ptr defaultSMRDLiteralOffset() const;
858 
859   OperandMatchResultTy parseOModOperand(OperandVector &Operands);
860 
861   void cvtId(MCInst &Inst, const OperandVector &Operands);
862   void cvtVOP3_2_mod(MCInst &Inst, const OperandVector &Operands);
863   void cvtVOP3(MCInst &Inst, const OperandVector &Operands);
864 
865   void cvtMIMG(MCInst &Inst, const OperandVector &Operands);
866   void cvtMIMGAtomic(MCInst &Inst, const OperandVector &Operands);
867 
868   OperandMatchResultTy parseDPPCtrl(OperandVector &Operands);
869   AMDGPUOperand::Ptr defaultRowMask() const;
870   AMDGPUOperand::Ptr defaultBankMask() const;
871   AMDGPUOperand::Ptr defaultBoundCtrl() const;
872   void cvtDPP(MCInst &Inst, const OperandVector &Operands);
873 
874   OperandMatchResultTy parseSDWASel(OperandVector &Operands, StringRef Prefix,
875                                     AMDGPUOperand::ImmTy Type);
876   OperandMatchResultTy parseSDWADstUnused(OperandVector &Operands);
877   void cvtSdwaVOP1(MCInst &Inst, const OperandVector &Operands);
878   void cvtSdwaVOP2(MCInst &Inst, const OperandVector &Operands);
879   void cvtSdwaVOPC(MCInst &Inst, const OperandVector &Operands);
880   void cvtSDWA(MCInst &Inst, const OperandVector &Operands,
881                uint64_t BasicInstType);
882 };
883 
884 struct OptionalOperand {
885   const char *Name;
886   AMDGPUOperand::ImmTy Type;
887   bool IsBit;
888   bool (*ConvertResult)(int64_t&);
889 };
890 
891 } // end anonymous namespace
892 
893 // May be called with integer type with equivalent bitwidth.
894 static const fltSemantics *getFltSemantics(unsigned Size) {
895   switch (Size) {
896   case 4:
897     return &APFloat::IEEEsingle();
898   case 8:
899     return &APFloat::IEEEdouble();
900   case 2:
901     return &APFloat::IEEEhalf();
902   default:
903     llvm_unreachable("unsupported fp type");
904   }
905 }
906 
907 static const fltSemantics *getFltSemantics(MVT VT) {
908   return getFltSemantics(VT.getSizeInBits() / 8);
909 }
910 
911 //===----------------------------------------------------------------------===//
912 // Operand
913 //===----------------------------------------------------------------------===//
914 
915 static bool canLosslesslyConvertToFPType(APFloat &FPLiteral, MVT VT) {
916   bool Lost;
917 
918   // Convert literal to single precision
919   APFloat::opStatus Status = FPLiteral.convert(*getFltSemantics(VT),
920                                                APFloat::rmNearestTiesToEven,
921                                                &Lost);
922   // We allow precision lost but not overflow or underflow
923   if (Status != APFloat::opOK &&
924       Lost &&
925       ((Status & APFloat::opOverflow)  != 0 ||
926        (Status & APFloat::opUnderflow) != 0)) {
927     return false;
928   }
929 
930   return true;
931 }
932 
933 bool AMDGPUOperand::isInlinableImm(MVT type) const {
934   if (!isImmTy(ImmTyNone)) {
935     // Only plain immediates are inlinable (e.g. "clamp" attribute is not)
936     return false;
937   }
938   // TODO: We should avoid using host float here. It would be better to
939   // check the float bit values which is what a few other places do.
940   // We've had bot failures before due to weird NaN support on mips hosts.
941 
942   APInt Literal(64, Imm.Val);
943 
944   if (Imm.IsFPImm) { // We got fp literal token
945     if (type == MVT::f64 || type == MVT::i64) { // Expected 64-bit operand
946       return AMDGPU::isInlinableLiteral64(Imm.Val,
947                                           AsmParser->hasInv2PiInlineImm());
948     }
949 
950     APFloat FPLiteral(APFloat::IEEEdouble(), APInt(64, Imm.Val));
951     if (!canLosslesslyConvertToFPType(FPLiteral, type))
952       return false;
953 
954     // Check if single precision literal is inlinable
955     return AMDGPU::isInlinableLiteral32(
956       static_cast<int32_t>(FPLiteral.bitcastToAPInt().getZExtValue()),
957       AsmParser->hasInv2PiInlineImm());
958   }
959 
960 
961   // We got int literal token.
962   if (type == MVT::f64 || type == MVT::i64) { // Expected 64-bit operand
963     return AMDGPU::isInlinableLiteral64(Imm.Val,
964                                         AsmParser->hasInv2PiInlineImm());
965   }
966 
967   if (type.getScalarSizeInBits() == 16) {
968     return AMDGPU::isInlinableLiteral16(
969       static_cast<int16_t>(Literal.getLoBits(16).getSExtValue()),
970       AsmParser->hasInv2PiInlineImm());
971   }
972 
973   return AMDGPU::isInlinableLiteral32(
974     static_cast<int32_t>(Literal.getLoBits(32).getZExtValue()),
975     AsmParser->hasInv2PiInlineImm());
976 }
977 
978 bool AMDGPUOperand::isLiteralImm(MVT type) const {
979   // Check that this imediate can be added as literal
980   if (!isImmTy(ImmTyNone)) {
981     return false;
982   }
983 
984   if (!Imm.IsFPImm) {
985     // We got int literal token.
986 
987     unsigned Size = type.getSizeInBits();
988     if (Size == 64)
989       Size = 32;
990 
991     // FIXME: 64-bit operands can zero extend, sign extend, or pad zeroes for FP
992     // types.
993     return isUIntN(Size, Imm.Val) || isIntN(Size, Imm.Val);
994   }
995 
996   // We got fp literal token
997   if (type == MVT::f64) { // Expected 64-bit fp operand
998     // We would set low 64-bits of literal to zeroes but we accept this literals
999     return true;
1000   }
1001 
1002   if (type == MVT::i64) { // Expected 64-bit int operand
1003     // We don't allow fp literals in 64-bit integer instructions. It is
1004     // unclear how we should encode them.
1005     return false;
1006   }
1007 
1008   APFloat FPLiteral(APFloat::IEEEdouble(), APInt(64, Imm.Val));
1009   return canLosslesslyConvertToFPType(FPLiteral, type);
1010 }
1011 
1012 bool AMDGPUOperand::isRegClass(unsigned RCID) const {
1013   return isReg() && AsmParser->getMRI()->getRegClass(RCID).contains(getReg());
1014 }
1015 
1016 void AMDGPUOperand::addImmOperands(MCInst &Inst, unsigned N, bool ApplyModifiers) const {
1017   int64_t Val = Imm.Val;
1018   if (isImmTy(ImmTyNone) && ApplyModifiers && Imm.Mods.hasFPModifiers() && Imm.Mods.Neg) {
1019     // Apply modifiers to immediate value. Only negate can get here
1020     if (Imm.IsFPImm) {
1021       APFloat F(BitsToDouble(Val));
1022       F.changeSign();
1023       Val = F.bitcastToAPInt().getZExtValue();
1024     } else {
1025       Val = -Val;
1026     }
1027   }
1028 
1029   if (AMDGPU::isSISrcOperand(AsmParser->getMII()->get(Inst.getOpcode()),
1030                              Inst.getNumOperands())) {
1031     addLiteralImmOperand(Inst, Val);
1032   } else {
1033     Inst.addOperand(MCOperand::createImm(Val));
1034   }
1035 }
1036 
1037 void AMDGPUOperand::addLiteralImmOperand(MCInst &Inst, int64_t Val) const {
1038   const auto& InstDesc = AsmParser->getMII()->get(Inst.getOpcode());
1039   auto OpNum = Inst.getNumOperands();
1040   // Check that this operand accepts literals
1041   assert(AMDGPU::isSISrcOperand(InstDesc, OpNum));
1042 
1043   auto OpSize = AMDGPU::getOperandSize(InstDesc, OpNum); // expected operand size
1044 
1045   if (Imm.IsFPImm) { // We got fp literal token
1046     APInt Literal(64, Val);
1047 
1048     switch (OpSize) {
1049     case 8: {
1050       if (AMDGPU::isInlinableLiteral64(Literal.getZExtValue(),
1051                                        AsmParser->hasInv2PiInlineImm())) {
1052         Inst.addOperand(MCOperand::createImm(Literal.getZExtValue()));
1053         return;
1054       }
1055 
1056       // Non-inlineable
1057       if (AMDGPU::isSISrcFPOperand(InstDesc, OpNum)) { // Expected 64-bit fp operand
1058         // For fp operands we check if low 32 bits are zeros
1059         if (Literal.getLoBits(32) != 0) {
1060           const_cast<AMDGPUAsmParser *>(AsmParser)->Warning(Inst.getLoc(),
1061           "Can't encode literal as exact 64-bit floating-point operand. "
1062           "Low 32-bits will be set to zero");
1063         }
1064 
1065         Inst.addOperand(MCOperand::createImm(Literal.lshr(32).getZExtValue()));
1066         return;
1067       }
1068 
1069       // We don't allow fp literals in 64-bit integer instructions. It is
1070       // unclear how we should encode them. This case should be checked earlier
1071       // in predicate methods (isLiteralImm())
1072       llvm_unreachable("fp literal in 64-bit integer instruction.");
1073     }
1074     case 4:
1075     case 2: {
1076       bool lost;
1077       APFloat FPLiteral(APFloat::IEEEdouble(), Literal);
1078       // Convert literal to single precision
1079       FPLiteral.convert(*getFltSemantics(OpSize),
1080                         APFloat::rmNearestTiesToEven, &lost);
1081       // We allow precision lost but not overflow or underflow. This should be
1082       // checked earlier in isLiteralImm()
1083       Inst.addOperand(MCOperand::createImm(FPLiteral.bitcastToAPInt().getZExtValue()));
1084       return;
1085     }
1086     default:
1087       llvm_unreachable("invalid operand size");
1088     }
1089 
1090     return;
1091   }
1092 
1093    // We got int literal token.
1094   // Only sign extend inline immediates.
1095   // FIXME: No errors on truncation
1096   switch (OpSize) {
1097   case 4: {
1098     if (isInt<32>(Val) &&
1099         AMDGPU::isInlinableLiteral32(static_cast<int32_t>(Val),
1100                                      AsmParser->hasInv2PiInlineImm())) {
1101       Inst.addOperand(MCOperand::createImm(Val));
1102       return;
1103     }
1104 
1105     Inst.addOperand(MCOperand::createImm(Val & 0xffffffff));
1106     return;
1107   }
1108   case 8: {
1109     if (AMDGPU::isInlinableLiteral64(Val,
1110                                      AsmParser->hasInv2PiInlineImm())) {
1111       Inst.addOperand(MCOperand::createImm(Val));
1112       return;
1113     }
1114 
1115     Inst.addOperand(MCOperand::createImm(Lo_32(Val)));
1116     return;
1117   }
1118   case 2: {
1119     if (isInt<16>(Val) &&
1120         AMDGPU::isInlinableLiteral16(static_cast<int16_t>(Val),
1121                                      AsmParser->hasInv2PiInlineImm())) {
1122       Inst.addOperand(MCOperand::createImm(Val));
1123       return;
1124     }
1125 
1126     Inst.addOperand(MCOperand::createImm(Val & 0xffff));
1127     return;
1128   }
1129   default:
1130     llvm_unreachable("invalid operand size");
1131   }
1132 }
1133 
1134 template <unsigned Bitwidth>
1135 void AMDGPUOperand::addKImmFPOperands(MCInst &Inst, unsigned N) const {
1136   APInt Literal(64, Imm.Val);
1137 
1138   if (!Imm.IsFPImm) {
1139     // We got int literal token.
1140     Inst.addOperand(MCOperand::createImm(Literal.getLoBits(Bitwidth).getZExtValue()));
1141     return;
1142   }
1143 
1144   bool Lost;
1145   APFloat FPLiteral(APFloat::IEEEdouble(), Literal);
1146   FPLiteral.convert(*getFltSemantics(Bitwidth / 8),
1147                     APFloat::rmNearestTiesToEven, &Lost);
1148   Inst.addOperand(MCOperand::createImm(FPLiteral.bitcastToAPInt().getZExtValue()));
1149 }
1150 
1151 void AMDGPUOperand::addRegOperands(MCInst &Inst, unsigned N) const {
1152   Inst.addOperand(MCOperand::createReg(AMDGPU::getMCReg(getReg(), AsmParser->getSTI())));
1153 }
1154 
1155 //===----------------------------------------------------------------------===//
1156 // AsmParser
1157 //===----------------------------------------------------------------------===//
1158 
1159 static int getRegClass(RegisterKind Is, unsigned RegWidth) {
1160   if (Is == IS_VGPR) {
1161     switch (RegWidth) {
1162       default: return -1;
1163       case 1: return AMDGPU::VGPR_32RegClassID;
1164       case 2: return AMDGPU::VReg_64RegClassID;
1165       case 3: return AMDGPU::VReg_96RegClassID;
1166       case 4: return AMDGPU::VReg_128RegClassID;
1167       case 8: return AMDGPU::VReg_256RegClassID;
1168       case 16: return AMDGPU::VReg_512RegClassID;
1169     }
1170   } else if (Is == IS_TTMP) {
1171     switch (RegWidth) {
1172       default: return -1;
1173       case 1: return AMDGPU::TTMP_32RegClassID;
1174       case 2: return AMDGPU::TTMP_64RegClassID;
1175       case 4: return AMDGPU::TTMP_128RegClassID;
1176     }
1177   } else if (Is == IS_SGPR) {
1178     switch (RegWidth) {
1179       default: return -1;
1180       case 1: return AMDGPU::SGPR_32RegClassID;
1181       case 2: return AMDGPU::SGPR_64RegClassID;
1182       case 4: return AMDGPU::SGPR_128RegClassID;
1183       case 8: return AMDGPU::SReg_256RegClassID;
1184       case 16: return AMDGPU::SReg_512RegClassID;
1185     }
1186   }
1187   return -1;
1188 }
1189 
1190 static unsigned getSpecialRegForName(StringRef RegName) {
1191   return StringSwitch<unsigned>(RegName)
1192     .Case("exec", AMDGPU::EXEC)
1193     .Case("vcc", AMDGPU::VCC)
1194     .Case("flat_scratch", AMDGPU::FLAT_SCR)
1195     .Case("m0", AMDGPU::M0)
1196     .Case("scc", AMDGPU::SCC)
1197     .Case("tba", AMDGPU::TBA)
1198     .Case("tma", AMDGPU::TMA)
1199     .Case("flat_scratch_lo", AMDGPU::FLAT_SCR_LO)
1200     .Case("flat_scratch_hi", AMDGPU::FLAT_SCR_HI)
1201     .Case("vcc_lo", AMDGPU::VCC_LO)
1202     .Case("vcc_hi", AMDGPU::VCC_HI)
1203     .Case("exec_lo", AMDGPU::EXEC_LO)
1204     .Case("exec_hi", AMDGPU::EXEC_HI)
1205     .Case("tma_lo", AMDGPU::TMA_LO)
1206     .Case("tma_hi", AMDGPU::TMA_HI)
1207     .Case("tba_lo", AMDGPU::TBA_LO)
1208     .Case("tba_hi", AMDGPU::TBA_HI)
1209     .Default(0);
1210 }
1211 
1212 bool AMDGPUAsmParser::ParseRegister(unsigned &RegNo, SMLoc &StartLoc, SMLoc &EndLoc) {
1213   auto R = parseRegister();
1214   if (!R) return true;
1215   assert(R->isReg());
1216   RegNo = R->getReg();
1217   StartLoc = R->getStartLoc();
1218   EndLoc = R->getEndLoc();
1219   return false;
1220 }
1221 
1222 bool AMDGPUAsmParser::AddNextRegisterToList(unsigned& Reg, unsigned& RegWidth, RegisterKind RegKind, unsigned Reg1, unsigned RegNum)
1223 {
1224   switch (RegKind) {
1225   case IS_SPECIAL:
1226     if (Reg == AMDGPU::EXEC_LO && Reg1 == AMDGPU::EXEC_HI) { Reg = AMDGPU::EXEC; RegWidth = 2; return true; }
1227     if (Reg == AMDGPU::FLAT_SCR_LO && Reg1 == AMDGPU::FLAT_SCR_HI) { Reg = AMDGPU::FLAT_SCR; RegWidth = 2; return true; }
1228     if (Reg == AMDGPU::VCC_LO && Reg1 == AMDGPU::VCC_HI) { Reg = AMDGPU::VCC; RegWidth = 2; return true; }
1229     if (Reg == AMDGPU::TBA_LO && Reg1 == AMDGPU::TBA_HI) { Reg = AMDGPU::TBA; RegWidth = 2; return true; }
1230     if (Reg == AMDGPU::TMA_LO && Reg1 == AMDGPU::TMA_HI) { Reg = AMDGPU::TMA; RegWidth = 2; return true; }
1231     return false;
1232   case IS_VGPR:
1233   case IS_SGPR:
1234   case IS_TTMP:
1235     if (Reg1 != Reg + RegWidth) { return false; }
1236     RegWidth++;
1237     return true;
1238   default:
1239     llvm_unreachable("unexpected register kind");
1240   }
1241 }
1242 
1243 bool AMDGPUAsmParser::ParseAMDGPURegister(RegisterKind& RegKind, unsigned& Reg, unsigned& RegNum, unsigned& RegWidth)
1244 {
1245   const MCRegisterInfo *TRI = getContext().getRegisterInfo();
1246   if (getLexer().is(AsmToken::Identifier)) {
1247     StringRef RegName = Parser.getTok().getString();
1248     if ((Reg = getSpecialRegForName(RegName))) {
1249       Parser.Lex();
1250       RegKind = IS_SPECIAL;
1251     } else {
1252       unsigned RegNumIndex = 0;
1253       if (RegName[0] == 'v') {
1254         RegNumIndex = 1;
1255         RegKind = IS_VGPR;
1256       } else if (RegName[0] == 's') {
1257         RegNumIndex = 1;
1258         RegKind = IS_SGPR;
1259       } else if (RegName.startswith("ttmp")) {
1260         RegNumIndex = strlen("ttmp");
1261         RegKind = IS_TTMP;
1262       } else {
1263         return false;
1264       }
1265       if (RegName.size() > RegNumIndex) {
1266         // Single 32-bit register: vXX.
1267         if (RegName.substr(RegNumIndex).getAsInteger(10, RegNum))
1268           return false;
1269         Parser.Lex();
1270         RegWidth = 1;
1271       } else {
1272         // Range of registers: v[XX:YY]. ":YY" is optional.
1273         Parser.Lex();
1274         int64_t RegLo, RegHi;
1275         if (getLexer().isNot(AsmToken::LBrac))
1276           return false;
1277         Parser.Lex();
1278 
1279         if (getParser().parseAbsoluteExpression(RegLo))
1280           return false;
1281 
1282         const bool isRBrace = getLexer().is(AsmToken::RBrac);
1283         if (!isRBrace && getLexer().isNot(AsmToken::Colon))
1284           return false;
1285         Parser.Lex();
1286 
1287         if (isRBrace) {
1288           RegHi = RegLo;
1289         } else {
1290           if (getParser().parseAbsoluteExpression(RegHi))
1291             return false;
1292 
1293           if (getLexer().isNot(AsmToken::RBrac))
1294             return false;
1295           Parser.Lex();
1296         }
1297         RegNum = (unsigned) RegLo;
1298         RegWidth = (RegHi - RegLo) + 1;
1299       }
1300     }
1301   } else if (getLexer().is(AsmToken::LBrac)) {
1302     // List of consecutive registers: [s0,s1,s2,s3]
1303     Parser.Lex();
1304     if (!ParseAMDGPURegister(RegKind, Reg, RegNum, RegWidth))
1305       return false;
1306     if (RegWidth != 1)
1307       return false;
1308     RegisterKind RegKind1;
1309     unsigned Reg1, RegNum1, RegWidth1;
1310     do {
1311       if (getLexer().is(AsmToken::Comma)) {
1312         Parser.Lex();
1313       } else if (getLexer().is(AsmToken::RBrac)) {
1314         Parser.Lex();
1315         break;
1316       } else if (ParseAMDGPURegister(RegKind1, Reg1, RegNum1, RegWidth1)) {
1317         if (RegWidth1 != 1) {
1318           return false;
1319         }
1320         if (RegKind1 != RegKind) {
1321           return false;
1322         }
1323         if (!AddNextRegisterToList(Reg, RegWidth, RegKind1, Reg1, RegNum1)) {
1324           return false;
1325         }
1326       } else {
1327         return false;
1328       }
1329     } while (true);
1330   } else {
1331     return false;
1332   }
1333   switch (RegKind) {
1334   case IS_SPECIAL:
1335     RegNum = 0;
1336     RegWidth = 1;
1337     break;
1338   case IS_VGPR:
1339   case IS_SGPR:
1340   case IS_TTMP:
1341   {
1342     unsigned Size = 1;
1343     if (RegKind == IS_SGPR || RegKind == IS_TTMP) {
1344       // SGPR and TTMP registers must be are aligned. Max required alignment is 4 dwords.
1345       Size = std::min(RegWidth, 4u);
1346     }
1347     if (RegNum % Size != 0)
1348       return false;
1349     RegNum = RegNum / Size;
1350     int RCID = getRegClass(RegKind, RegWidth);
1351     if (RCID == -1)
1352       return false;
1353     const MCRegisterClass RC = TRI->getRegClass(RCID);
1354     if (RegNum >= RC.getNumRegs())
1355       return false;
1356     Reg = RC.getRegister(RegNum);
1357     break;
1358   }
1359 
1360   default:
1361     llvm_unreachable("unexpected register kind");
1362   }
1363 
1364   if (!subtargetHasRegister(*TRI, Reg))
1365     return false;
1366   return true;
1367 }
1368 
1369 std::unique_ptr<AMDGPUOperand> AMDGPUAsmParser::parseRegister() {
1370   const auto &Tok = Parser.getTok();
1371   SMLoc StartLoc = Tok.getLoc();
1372   SMLoc EndLoc = Tok.getEndLoc();
1373   RegisterKind RegKind;
1374   unsigned Reg, RegNum, RegWidth;
1375 
1376   if (!ParseAMDGPURegister(RegKind, Reg, RegNum, RegWidth)) {
1377     return nullptr;
1378   }
1379   return AMDGPUOperand::CreateReg(this, Reg, StartLoc, EndLoc, false);
1380 }
1381 
1382 OperandMatchResultTy
1383 AMDGPUAsmParser::parseImm(OperandVector &Operands) {
1384   // TODO: add syntactic sugar for 1/(2*PI)
1385   bool Minus = false;
1386   if (getLexer().getKind() == AsmToken::Minus) {
1387     Minus = true;
1388     Parser.Lex();
1389   }
1390 
1391   SMLoc S = Parser.getTok().getLoc();
1392   switch(getLexer().getKind()) {
1393   case AsmToken::Integer: {
1394     int64_t IntVal;
1395     if (getParser().parseAbsoluteExpression(IntVal))
1396       return MatchOperand_ParseFail;
1397     if (Minus)
1398       IntVal *= -1;
1399     Operands.push_back(AMDGPUOperand::CreateImm(this, IntVal, S));
1400     return MatchOperand_Success;
1401   }
1402   case AsmToken::Real: {
1403     int64_t IntVal;
1404     if (getParser().parseAbsoluteExpression(IntVal))
1405       return MatchOperand_ParseFail;
1406 
1407     APFloat F(BitsToDouble(IntVal));
1408     if (Minus)
1409       F.changeSign();
1410     Operands.push_back(
1411         AMDGPUOperand::CreateImm(this, F.bitcastToAPInt().getZExtValue(), S,
1412                                  AMDGPUOperand::ImmTyNone, true));
1413     return MatchOperand_Success;
1414   }
1415   default:
1416     return Minus ? MatchOperand_ParseFail : MatchOperand_NoMatch;
1417   }
1418 }
1419 
1420 OperandMatchResultTy
1421 AMDGPUAsmParser::parseRegOrImm(OperandVector &Operands) {
1422   auto res = parseImm(Operands);
1423   if (res != MatchOperand_NoMatch) {
1424     return res;
1425   }
1426 
1427   if (auto R = parseRegister()) {
1428     assert(R->isReg());
1429     R->Reg.IsForcedVOP3 = isForcedVOP3();
1430     Operands.push_back(std::move(R));
1431     return MatchOperand_Success;
1432   }
1433   return MatchOperand_ParseFail;
1434 }
1435 
1436 OperandMatchResultTy
1437 AMDGPUAsmParser::parseRegOrImmWithFPInputMods(OperandVector &Operands) {
1438   // XXX: During parsing we can't determine if minus sign means
1439   // negate-modifier or negative immediate value.
1440   // By default we suppose it is modifier.
1441   bool Negate = false, Abs = false, Abs2 = false;
1442 
1443   if (getLexer().getKind()== AsmToken::Minus) {
1444     Parser.Lex();
1445     Negate = true;
1446   }
1447 
1448   if (getLexer().getKind() == AsmToken::Identifier && Parser.getTok().getString() == "abs") {
1449     Parser.Lex();
1450     Abs2 = true;
1451     if (getLexer().isNot(AsmToken::LParen)) {
1452       Error(Parser.getTok().getLoc(), "expected left paren after abs");
1453       return MatchOperand_ParseFail;
1454     }
1455     Parser.Lex();
1456   }
1457 
1458   if (getLexer().getKind() == AsmToken::Pipe) {
1459     if (Abs2) {
1460       Error(Parser.getTok().getLoc(), "expected register or immediate");
1461       return MatchOperand_ParseFail;
1462     }
1463     Parser.Lex();
1464     Abs = true;
1465   }
1466 
1467   auto Res = parseRegOrImm(Operands);
1468   if (Res != MatchOperand_Success) {
1469     return Res;
1470   }
1471 
1472   AMDGPUOperand::Modifiers Mods;
1473   if (Negate) {
1474     Mods.Neg = true;
1475   }
1476   if (Abs) {
1477     if (getLexer().getKind() != AsmToken::Pipe) {
1478       Error(Parser.getTok().getLoc(), "expected vertical bar");
1479       return MatchOperand_ParseFail;
1480     }
1481     Parser.Lex();
1482     Mods.Abs = true;
1483   }
1484   if (Abs2) {
1485     if (getLexer().isNot(AsmToken::RParen)) {
1486       Error(Parser.getTok().getLoc(), "expected closing parentheses");
1487       return MatchOperand_ParseFail;
1488     }
1489     Parser.Lex();
1490     Mods.Abs = true;
1491   }
1492 
1493   if (Mods.hasFPModifiers()) {
1494     AMDGPUOperand &Op = static_cast<AMDGPUOperand &>(*Operands.back());
1495     Op.setModifiers(Mods);
1496   }
1497   return MatchOperand_Success;
1498 }
1499 
1500 OperandMatchResultTy
1501 AMDGPUAsmParser::parseRegOrImmWithIntInputMods(OperandVector &Operands) {
1502   bool Sext = false;
1503 
1504   if (getLexer().getKind() == AsmToken::Identifier && Parser.getTok().getString() == "sext") {
1505     Parser.Lex();
1506     Sext = true;
1507     if (getLexer().isNot(AsmToken::LParen)) {
1508       Error(Parser.getTok().getLoc(), "expected left paren after sext");
1509       return MatchOperand_ParseFail;
1510     }
1511     Parser.Lex();
1512   }
1513 
1514   auto Res = parseRegOrImm(Operands);
1515   if (Res != MatchOperand_Success) {
1516     return Res;
1517   }
1518 
1519   AMDGPUOperand::Modifiers Mods;
1520   if (Sext) {
1521     if (getLexer().isNot(AsmToken::RParen)) {
1522       Error(Parser.getTok().getLoc(), "expected closing parentheses");
1523       return MatchOperand_ParseFail;
1524     }
1525     Parser.Lex();
1526     Mods.Sext = true;
1527   }
1528 
1529   if (Mods.hasIntModifiers()) {
1530     AMDGPUOperand &Op = static_cast<AMDGPUOperand &>(*Operands.back());
1531     Op.setModifiers(Mods);
1532   }
1533 
1534   return MatchOperand_Success;
1535 }
1536 
1537 OperandMatchResultTy AMDGPUAsmParser::parseVReg32OrOff(OperandVector &Operands) {
1538   std::unique_ptr<AMDGPUOperand> Reg = parseRegister();
1539   if (Reg) {
1540     Operands.push_back(std::move(Reg));
1541     return MatchOperand_Success;
1542   }
1543 
1544   const AsmToken &Tok = Parser.getTok();
1545   if (Tok.getString() == "off") {
1546     Operands.push_back(AMDGPUOperand::CreateImm(this, 0, Tok.getLoc(),
1547                                                 AMDGPUOperand::ImmTyOff, false));
1548     Parser.Lex();
1549     return MatchOperand_Success;
1550   }
1551 
1552   return MatchOperand_NoMatch;
1553 }
1554 
1555 unsigned AMDGPUAsmParser::checkTargetMatchPredicate(MCInst &Inst) {
1556 
1557   uint64_t TSFlags = MII.get(Inst.getOpcode()).TSFlags;
1558 
1559   if ((getForcedEncodingSize() == 32 && (TSFlags & SIInstrFlags::VOP3)) ||
1560       (getForcedEncodingSize() == 64 && !(TSFlags & SIInstrFlags::VOP3)) ||
1561       (isForcedDPP() && !(TSFlags & SIInstrFlags::DPP)) ||
1562       (isForcedSDWA() && !(TSFlags & SIInstrFlags::SDWA)) )
1563     return Match_InvalidOperand;
1564 
1565   if ((TSFlags & SIInstrFlags::VOP3) &&
1566       (TSFlags & SIInstrFlags::VOPAsmPrefer32Bit) &&
1567       getForcedEncodingSize() != 64)
1568     return Match_PreferE32;
1569 
1570   if (Inst.getOpcode() == AMDGPU::V_MAC_F32_sdwa_vi ||
1571       Inst.getOpcode() == AMDGPU::V_MAC_F16_sdwa_vi) {
1572     // v_mac_f32/16 allow only dst_sel == DWORD;
1573     auto OpNum =
1574         AMDGPU::getNamedOperandIdx(Inst.getOpcode(), AMDGPU::OpName::dst_sel);
1575     const auto &Op = Inst.getOperand(OpNum);
1576     if (!Op.isImm() || Op.getImm() != AMDGPU::SDWA::SdwaSel::DWORD) {
1577       return Match_InvalidOperand;
1578     }
1579   }
1580 
1581   return Match_Success;
1582 }
1583 
1584 bool AMDGPUAsmParser::MatchAndEmitInstruction(SMLoc IDLoc, unsigned &Opcode,
1585                                               OperandVector &Operands,
1586                                               MCStreamer &Out,
1587                                               uint64_t &ErrorInfo,
1588                                               bool MatchingInlineAsm) {
1589   // What asm variants we should check
1590   std::vector<unsigned> MatchedVariants;
1591   if (getForcedEncodingSize() == 32) {
1592     MatchedVariants = {AMDGPUAsmVariants::DEFAULT};
1593   } else if (isForcedVOP3()) {
1594     MatchedVariants = {AMDGPUAsmVariants::VOP3};
1595   } else if (isForcedSDWA()) {
1596     MatchedVariants = {AMDGPUAsmVariants::SDWA};
1597   } else if (isForcedDPP()) {
1598     MatchedVariants = {AMDGPUAsmVariants::DPP};
1599   } else {
1600     MatchedVariants = {AMDGPUAsmVariants::DEFAULT,
1601                        AMDGPUAsmVariants::VOP3,
1602                        AMDGPUAsmVariants::SDWA,
1603                        AMDGPUAsmVariants::DPP};
1604   }
1605 
1606   MCInst Inst;
1607   unsigned Result = Match_Success;
1608   for (auto Variant : MatchedVariants) {
1609     uint64_t EI;
1610     auto R = MatchInstructionImpl(Operands, Inst, EI, MatchingInlineAsm,
1611                                   Variant);
1612     // We order match statuses from least to most specific. We use most specific
1613     // status as resulting
1614     // Match_MnemonicFail < Match_InvalidOperand < Match_MissingFeature < Match_PreferE32
1615     if ((R == Match_Success) ||
1616         (R == Match_PreferE32) ||
1617         (R == Match_MissingFeature && Result != Match_PreferE32) ||
1618         (R == Match_InvalidOperand && Result != Match_MissingFeature
1619                                    && Result != Match_PreferE32) ||
1620         (R == Match_MnemonicFail   && Result != Match_InvalidOperand
1621                                    && Result != Match_MissingFeature
1622                                    && Result != Match_PreferE32)) {
1623       Result = R;
1624       ErrorInfo = EI;
1625     }
1626     if (R == Match_Success)
1627       break;
1628   }
1629 
1630   switch (Result) {
1631   default: break;
1632   case Match_Success:
1633     Inst.setLoc(IDLoc);
1634     Out.EmitInstruction(Inst, getSTI());
1635     return false;
1636 
1637   case Match_MissingFeature:
1638     return Error(IDLoc, "instruction not supported on this GPU");
1639 
1640   case Match_MnemonicFail:
1641     return Error(IDLoc, "unrecognized instruction mnemonic");
1642 
1643   case Match_InvalidOperand: {
1644     SMLoc ErrorLoc = IDLoc;
1645     if (ErrorInfo != ~0ULL) {
1646       if (ErrorInfo >= Operands.size()) {
1647         return Error(IDLoc, "too few operands for instruction");
1648       }
1649       ErrorLoc = ((AMDGPUOperand &)*Operands[ErrorInfo]).getStartLoc();
1650       if (ErrorLoc == SMLoc())
1651         ErrorLoc = IDLoc;
1652     }
1653     return Error(ErrorLoc, "invalid operand for instruction");
1654   }
1655 
1656   case Match_PreferE32:
1657     return Error(IDLoc, "internal error: instruction without _e64 suffix "
1658                         "should be encoded as e32");
1659   }
1660   llvm_unreachable("Implement any new match types added!");
1661 }
1662 
1663 bool AMDGPUAsmParser::ParseDirectiveMajorMinor(uint32_t &Major,
1664                                                uint32_t &Minor) {
1665   if (getLexer().isNot(AsmToken::Integer))
1666     return TokError("invalid major version");
1667 
1668   Major = getLexer().getTok().getIntVal();
1669   Lex();
1670 
1671   if (getLexer().isNot(AsmToken::Comma))
1672     return TokError("minor version number required, comma expected");
1673   Lex();
1674 
1675   if (getLexer().isNot(AsmToken::Integer))
1676     return TokError("invalid minor version");
1677 
1678   Minor = getLexer().getTok().getIntVal();
1679   Lex();
1680 
1681   return false;
1682 }
1683 
1684 bool AMDGPUAsmParser::ParseDirectiveHSACodeObjectVersion() {
1685 
1686   uint32_t Major;
1687   uint32_t Minor;
1688 
1689   if (ParseDirectiveMajorMinor(Major, Minor))
1690     return true;
1691 
1692   getTargetStreamer().EmitDirectiveHSACodeObjectVersion(Major, Minor);
1693   return false;
1694 }
1695 
1696 bool AMDGPUAsmParser::ParseDirectiveHSACodeObjectISA() {
1697   uint32_t Major;
1698   uint32_t Minor;
1699   uint32_t Stepping;
1700   StringRef VendorName;
1701   StringRef ArchName;
1702 
1703   // If this directive has no arguments, then use the ISA version for the
1704   // targeted GPU.
1705   if (getLexer().is(AsmToken::EndOfStatement)) {
1706     AMDGPU::IsaVersion Isa = AMDGPU::getIsaVersion(getSTI().getFeatureBits());
1707     getTargetStreamer().EmitDirectiveHSACodeObjectISA(Isa.Major, Isa.Minor,
1708                                                       Isa.Stepping,
1709                                                       "AMD", "AMDGPU");
1710     return false;
1711   }
1712 
1713   if (ParseDirectiveMajorMinor(Major, Minor))
1714     return true;
1715 
1716   if (getLexer().isNot(AsmToken::Comma))
1717     return TokError("stepping version number required, comma expected");
1718   Lex();
1719 
1720   if (getLexer().isNot(AsmToken::Integer))
1721     return TokError("invalid stepping version");
1722 
1723   Stepping = getLexer().getTok().getIntVal();
1724   Lex();
1725 
1726   if (getLexer().isNot(AsmToken::Comma))
1727     return TokError("vendor name required, comma expected");
1728   Lex();
1729 
1730   if (getLexer().isNot(AsmToken::String))
1731     return TokError("invalid vendor name");
1732 
1733   VendorName = getLexer().getTok().getStringContents();
1734   Lex();
1735 
1736   if (getLexer().isNot(AsmToken::Comma))
1737     return TokError("arch name required, comma expected");
1738   Lex();
1739 
1740   if (getLexer().isNot(AsmToken::String))
1741     return TokError("invalid arch name");
1742 
1743   ArchName = getLexer().getTok().getStringContents();
1744   Lex();
1745 
1746   getTargetStreamer().EmitDirectiveHSACodeObjectISA(Major, Minor, Stepping,
1747                                                     VendorName, ArchName);
1748   return false;
1749 }
1750 
1751 bool AMDGPUAsmParser::ParseDirectiveRuntimeMetadata() {
1752   std::string Metadata;
1753   raw_string_ostream MS(Metadata);
1754 
1755   getLexer().setSkipSpace(false);
1756 
1757   bool FoundEnd = false;
1758   while (!getLexer().is(AsmToken::Eof)) {
1759     while (getLexer().is(AsmToken::Space)) {
1760       MS << ' ';
1761       Lex();
1762     }
1763 
1764     if (getLexer().is(AsmToken::Identifier)) {
1765       StringRef ID = getLexer().getTok().getIdentifier();
1766       if (ID == ".end_amdgpu_runtime_metadata") {
1767         Lex();
1768         FoundEnd = true;
1769         break;
1770       }
1771     }
1772 
1773     MS << Parser.parseStringToEndOfStatement()
1774        << getContext().getAsmInfo()->getSeparatorString();
1775 
1776     Parser.eatToEndOfStatement();
1777   }
1778 
1779   getLexer().setSkipSpace(true);
1780 
1781   if (getLexer().is(AsmToken::Eof) && !FoundEnd)
1782     return TokError("expected directive .end_amdgpu_runtime_metadata not found");
1783 
1784   MS.flush();
1785 
1786   getTargetStreamer().EmitRuntimeMetadata(Metadata);
1787 
1788   return false;
1789 }
1790 
1791 bool AMDGPUAsmParser::ParseAMDKernelCodeTValue(StringRef ID,
1792                                                amd_kernel_code_t &Header) {
1793   SmallString<40> ErrStr;
1794   raw_svector_ostream Err(ErrStr);
1795   if (!parseAmdKernelCodeField(ID, getParser(), Header, Err)) {
1796     return TokError(Err.str());
1797   }
1798   Lex();
1799   return false;
1800 }
1801 
1802 bool AMDGPUAsmParser::ParseDirectiveAMDKernelCodeT() {
1803   amd_kernel_code_t Header;
1804   AMDGPU::initDefaultAMDKernelCodeT(Header, getSTI().getFeatureBits());
1805 
1806   while (true) {
1807     // Lex EndOfStatement.  This is in a while loop, because lexing a comment
1808     // will set the current token to EndOfStatement.
1809     while(getLexer().is(AsmToken::EndOfStatement))
1810       Lex();
1811 
1812     if (getLexer().isNot(AsmToken::Identifier))
1813       return TokError("expected value identifier or .end_amd_kernel_code_t");
1814 
1815     StringRef ID = getLexer().getTok().getIdentifier();
1816     Lex();
1817 
1818     if (ID == ".end_amd_kernel_code_t")
1819       break;
1820 
1821     if (ParseAMDKernelCodeTValue(ID, Header))
1822       return true;
1823   }
1824 
1825   getTargetStreamer().EmitAMDKernelCodeT(Header);
1826 
1827   return false;
1828 }
1829 
1830 bool AMDGPUAsmParser::ParseSectionDirectiveHSAText() {
1831   getParser().getStreamer().SwitchSection(
1832       AMDGPU::getHSATextSection(getContext()));
1833   return false;
1834 }
1835 
1836 bool AMDGPUAsmParser::ParseDirectiveAMDGPUHsaKernel() {
1837   if (getLexer().isNot(AsmToken::Identifier))
1838     return TokError("expected symbol name");
1839 
1840   StringRef KernelName = Parser.getTok().getString();
1841 
1842   getTargetStreamer().EmitAMDGPUSymbolType(KernelName,
1843                                            ELF::STT_AMDGPU_HSA_KERNEL);
1844   Lex();
1845   return false;
1846 }
1847 
1848 bool AMDGPUAsmParser::ParseDirectiveAMDGPUHsaModuleGlobal() {
1849   if (getLexer().isNot(AsmToken::Identifier))
1850     return TokError("expected symbol name");
1851 
1852   StringRef GlobalName = Parser.getTok().getIdentifier();
1853 
1854   getTargetStreamer().EmitAMDGPUHsaModuleScopeGlobal(GlobalName);
1855   Lex();
1856   return false;
1857 }
1858 
1859 bool AMDGPUAsmParser::ParseDirectiveAMDGPUHsaProgramGlobal() {
1860   if (getLexer().isNot(AsmToken::Identifier))
1861     return TokError("expected symbol name");
1862 
1863   StringRef GlobalName = Parser.getTok().getIdentifier();
1864 
1865   getTargetStreamer().EmitAMDGPUHsaProgramScopeGlobal(GlobalName);
1866   Lex();
1867   return false;
1868 }
1869 
1870 bool AMDGPUAsmParser::ParseSectionDirectiveHSADataGlobalAgent() {
1871   getParser().getStreamer().SwitchSection(
1872       AMDGPU::getHSADataGlobalAgentSection(getContext()));
1873   return false;
1874 }
1875 
1876 bool AMDGPUAsmParser::ParseSectionDirectiveHSADataGlobalProgram() {
1877   getParser().getStreamer().SwitchSection(
1878       AMDGPU::getHSADataGlobalProgramSection(getContext()));
1879   return false;
1880 }
1881 
1882 bool AMDGPUAsmParser::ParseSectionDirectiveHSARodataReadonlyAgent() {
1883   getParser().getStreamer().SwitchSection(
1884       AMDGPU::getHSARodataReadonlyAgentSection(getContext()));
1885   return false;
1886 }
1887 
1888 bool AMDGPUAsmParser::ParseDirective(AsmToken DirectiveID) {
1889   StringRef IDVal = DirectiveID.getString();
1890 
1891   if (IDVal == ".hsa_code_object_version")
1892     return ParseDirectiveHSACodeObjectVersion();
1893 
1894   if (IDVal == ".hsa_code_object_isa")
1895     return ParseDirectiveHSACodeObjectISA();
1896 
1897   if (IDVal == ".amdgpu_runtime_metadata")
1898     return ParseDirectiveRuntimeMetadata();
1899 
1900   if (IDVal == ".amd_kernel_code_t")
1901     return ParseDirectiveAMDKernelCodeT();
1902 
1903   if (IDVal == ".hsatext")
1904     return ParseSectionDirectiveHSAText();
1905 
1906   if (IDVal == ".amdgpu_hsa_kernel")
1907     return ParseDirectiveAMDGPUHsaKernel();
1908 
1909   if (IDVal == ".amdgpu_hsa_module_global")
1910     return ParseDirectiveAMDGPUHsaModuleGlobal();
1911 
1912   if (IDVal == ".amdgpu_hsa_program_global")
1913     return ParseDirectiveAMDGPUHsaProgramGlobal();
1914 
1915   if (IDVal == ".hsadata_global_agent")
1916     return ParseSectionDirectiveHSADataGlobalAgent();
1917 
1918   if (IDVal == ".hsadata_global_program")
1919     return ParseSectionDirectiveHSADataGlobalProgram();
1920 
1921   if (IDVal == ".hsarodata_readonly_agent")
1922     return ParseSectionDirectiveHSARodataReadonlyAgent();
1923 
1924   return true;
1925 }
1926 
1927 bool AMDGPUAsmParser::subtargetHasRegister(const MCRegisterInfo &MRI,
1928                                            unsigned RegNo) const {
1929   if (isCI())
1930     return true;
1931 
1932   if (isSI()) {
1933     // No flat_scr
1934     switch (RegNo) {
1935     case AMDGPU::FLAT_SCR:
1936     case AMDGPU::FLAT_SCR_LO:
1937     case AMDGPU::FLAT_SCR_HI:
1938       return false;
1939     default:
1940       return true;
1941     }
1942   }
1943 
1944   // VI only has 102 SGPRs, so make sure we aren't trying to use the 2 more that
1945   // SI/CI have.
1946   for (MCRegAliasIterator R(AMDGPU::SGPR102_SGPR103, &MRI, true);
1947        R.isValid(); ++R) {
1948     if (*R == RegNo)
1949       return false;
1950   }
1951 
1952   return true;
1953 }
1954 
1955 OperandMatchResultTy
1956 AMDGPUAsmParser::parseOperand(OperandVector &Operands, StringRef Mnemonic) {
1957 
1958   // Try to parse with a custom parser
1959   OperandMatchResultTy ResTy = MatchOperandParserImpl(Operands, Mnemonic);
1960 
1961   // If we successfully parsed the operand or if there as an error parsing,
1962   // we are done.
1963   //
1964   // If we are parsing after we reach EndOfStatement then this means we
1965   // are appending default values to the Operands list.  This is only done
1966   // by custom parser, so we shouldn't continue on to the generic parsing.
1967   if (ResTy == MatchOperand_Success || ResTy == MatchOperand_ParseFail ||
1968       getLexer().is(AsmToken::EndOfStatement))
1969     return ResTy;
1970 
1971   ResTy = parseRegOrImm(Operands);
1972 
1973   if (ResTy == MatchOperand_Success)
1974     return ResTy;
1975 
1976   if (getLexer().getKind() == AsmToken::Identifier) {
1977     // If this identifier is a symbol, we want to create an expression for it.
1978     // It is a little difficult to distinguish between a symbol name, and
1979     // an instruction flag like 'gds'.  In order to do this, we parse
1980     // all tokens as expressions and then treate the symbol name as the token
1981     // string when we want to interpret the operand as a token.
1982     const auto &Tok = Parser.getTok();
1983     SMLoc S = Tok.getLoc();
1984     const MCExpr *Expr = nullptr;
1985     if (!Parser.parseExpression(Expr)) {
1986       Operands.push_back(AMDGPUOperand::CreateExpr(this, Expr, S));
1987       return MatchOperand_Success;
1988     }
1989 
1990     Operands.push_back(AMDGPUOperand::CreateToken(this, Tok.getString(), Tok.getLoc()));
1991     Parser.Lex();
1992     return MatchOperand_Success;
1993   }
1994   return MatchOperand_NoMatch;
1995 }
1996 
1997 StringRef AMDGPUAsmParser::parseMnemonicSuffix(StringRef Name) {
1998   // Clear any forced encodings from the previous instruction.
1999   setForcedEncodingSize(0);
2000   setForcedDPP(false);
2001   setForcedSDWA(false);
2002 
2003   if (Name.endswith("_e64")) {
2004     setForcedEncodingSize(64);
2005     return Name.substr(0, Name.size() - 4);
2006   } else if (Name.endswith("_e32")) {
2007     setForcedEncodingSize(32);
2008     return Name.substr(0, Name.size() - 4);
2009   } else if (Name.endswith("_dpp")) {
2010     setForcedDPP(true);
2011     return Name.substr(0, Name.size() - 4);
2012   } else if (Name.endswith("_sdwa")) {
2013     setForcedSDWA(true);
2014     return Name.substr(0, Name.size() - 5);
2015   }
2016   return Name;
2017 }
2018 
2019 bool AMDGPUAsmParser::ParseInstruction(ParseInstructionInfo &Info,
2020                                        StringRef Name,
2021                                        SMLoc NameLoc, OperandVector &Operands) {
2022   // Add the instruction mnemonic
2023   Name = parseMnemonicSuffix(Name);
2024   Operands.push_back(AMDGPUOperand::CreateToken(this, Name, NameLoc));
2025 
2026   while (!getLexer().is(AsmToken::EndOfStatement)) {
2027     OperandMatchResultTy Res = parseOperand(Operands, Name);
2028 
2029     // Eat the comma or space if there is one.
2030     if (getLexer().is(AsmToken::Comma))
2031       Parser.Lex();
2032 
2033     switch (Res) {
2034       case MatchOperand_Success: break;
2035       case MatchOperand_ParseFail:
2036         Error(getLexer().getLoc(), "failed parsing operand.");
2037         while (!getLexer().is(AsmToken::EndOfStatement)) {
2038           Parser.Lex();
2039         }
2040         return true;
2041       case MatchOperand_NoMatch:
2042         Error(getLexer().getLoc(), "not a valid operand.");
2043         while (!getLexer().is(AsmToken::EndOfStatement)) {
2044           Parser.Lex();
2045         }
2046         return true;
2047     }
2048   }
2049 
2050   return false;
2051 }
2052 
2053 //===----------------------------------------------------------------------===//
2054 // Utility functions
2055 //===----------------------------------------------------------------------===//
2056 
2057 OperandMatchResultTy
2058 AMDGPUAsmParser::parseIntWithPrefix(const char *Prefix, int64_t &Int) {
2059   switch(getLexer().getKind()) {
2060     default: return MatchOperand_NoMatch;
2061     case AsmToken::Identifier: {
2062       StringRef Name = Parser.getTok().getString();
2063       if (!Name.equals(Prefix)) {
2064         return MatchOperand_NoMatch;
2065       }
2066 
2067       Parser.Lex();
2068       if (getLexer().isNot(AsmToken::Colon))
2069         return MatchOperand_ParseFail;
2070 
2071       Parser.Lex();
2072       if (getLexer().isNot(AsmToken::Integer))
2073         return MatchOperand_ParseFail;
2074 
2075       if (getParser().parseAbsoluteExpression(Int))
2076         return MatchOperand_ParseFail;
2077       break;
2078     }
2079   }
2080   return MatchOperand_Success;
2081 }
2082 
2083 OperandMatchResultTy
2084 AMDGPUAsmParser::parseIntWithPrefix(const char *Prefix, OperandVector &Operands,
2085                                     enum AMDGPUOperand::ImmTy ImmTy,
2086                                     bool (*ConvertResult)(int64_t&)) {
2087   SMLoc S = Parser.getTok().getLoc();
2088   int64_t Value = 0;
2089 
2090   OperandMatchResultTy Res = parseIntWithPrefix(Prefix, Value);
2091   if (Res != MatchOperand_Success)
2092     return Res;
2093 
2094   if (ConvertResult && !ConvertResult(Value)) {
2095     return MatchOperand_ParseFail;
2096   }
2097 
2098   Operands.push_back(AMDGPUOperand::CreateImm(this, Value, S, ImmTy));
2099   return MatchOperand_Success;
2100 }
2101 
2102 OperandMatchResultTy
2103 AMDGPUAsmParser::parseNamedBit(const char *Name, OperandVector &Operands,
2104                                enum AMDGPUOperand::ImmTy ImmTy) {
2105   int64_t Bit = 0;
2106   SMLoc S = Parser.getTok().getLoc();
2107 
2108   // We are at the end of the statement, and this is a default argument, so
2109   // use a default value.
2110   if (getLexer().isNot(AsmToken::EndOfStatement)) {
2111     switch(getLexer().getKind()) {
2112       case AsmToken::Identifier: {
2113         StringRef Tok = Parser.getTok().getString();
2114         if (Tok == Name) {
2115           Bit = 1;
2116           Parser.Lex();
2117         } else if (Tok.startswith("no") && Tok.endswith(Name)) {
2118           Bit = 0;
2119           Parser.Lex();
2120         } else {
2121           return MatchOperand_NoMatch;
2122         }
2123         break;
2124       }
2125       default:
2126         return MatchOperand_NoMatch;
2127     }
2128   }
2129 
2130   Operands.push_back(AMDGPUOperand::CreateImm(this, Bit, S, ImmTy));
2131   return MatchOperand_Success;
2132 }
2133 
2134 typedef std::map<enum AMDGPUOperand::ImmTy, unsigned> OptionalImmIndexMap;
2135 
2136 void addOptionalImmOperand(MCInst& Inst, const OperandVector& Operands,
2137                            OptionalImmIndexMap& OptionalIdx,
2138                            enum AMDGPUOperand::ImmTy ImmT, int64_t Default = 0) {
2139   auto i = OptionalIdx.find(ImmT);
2140   if (i != OptionalIdx.end()) {
2141     unsigned Idx = i->second;
2142     ((AMDGPUOperand &)*Operands[Idx]).addImmOperands(Inst, 1);
2143   } else {
2144     Inst.addOperand(MCOperand::createImm(Default));
2145   }
2146 }
2147 
2148 OperandMatchResultTy
2149 AMDGPUAsmParser::parseStringWithPrefix(StringRef Prefix, StringRef &Value) {
2150   if (getLexer().isNot(AsmToken::Identifier)) {
2151     return MatchOperand_NoMatch;
2152   }
2153   StringRef Tok = Parser.getTok().getString();
2154   if (Tok != Prefix) {
2155     return MatchOperand_NoMatch;
2156   }
2157 
2158   Parser.Lex();
2159   if (getLexer().isNot(AsmToken::Colon)) {
2160     return MatchOperand_ParseFail;
2161   }
2162 
2163   Parser.Lex();
2164   if (getLexer().isNot(AsmToken::Identifier)) {
2165     return MatchOperand_ParseFail;
2166   }
2167 
2168   Value = Parser.getTok().getString();
2169   return MatchOperand_Success;
2170 }
2171 
2172 //===----------------------------------------------------------------------===//
2173 // ds
2174 //===----------------------------------------------------------------------===//
2175 
2176 void AMDGPUAsmParser::cvtDSOffset01(MCInst &Inst,
2177                                     const OperandVector &Operands) {
2178   OptionalImmIndexMap OptionalIdx;
2179 
2180   for (unsigned i = 1, e = Operands.size(); i != e; ++i) {
2181     AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[i]);
2182 
2183     // Add the register arguments
2184     if (Op.isReg()) {
2185       Op.addRegOperands(Inst, 1);
2186       continue;
2187     }
2188 
2189     // Handle optional arguments
2190     OptionalIdx[Op.getImmTy()] = i;
2191   }
2192 
2193   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyOffset0);
2194   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyOffset1);
2195   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyGDS);
2196 
2197   Inst.addOperand(MCOperand::createReg(AMDGPU::M0)); // m0
2198 }
2199 
2200 void AMDGPUAsmParser::cvtDS(MCInst &Inst, const OperandVector &Operands) {
2201   std::map<enum AMDGPUOperand::ImmTy, unsigned> OptionalIdx;
2202   bool GDSOnly = false;
2203 
2204   for (unsigned i = 1, e = Operands.size(); i != e; ++i) {
2205     AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[i]);
2206 
2207     // Add the register arguments
2208     if (Op.isReg()) {
2209       Op.addRegOperands(Inst, 1);
2210       continue;
2211     }
2212 
2213     if (Op.isToken() && Op.getToken() == "gds") {
2214       GDSOnly = true;
2215       continue;
2216     }
2217 
2218     // Handle optional arguments
2219     OptionalIdx[Op.getImmTy()] = i;
2220   }
2221 
2222   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyOffset);
2223   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyGDS);
2224 
2225   if (!GDSOnly) {
2226     addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyGDS);
2227   }
2228   Inst.addOperand(MCOperand::createReg(AMDGPU::M0)); // m0
2229 }
2230 
2231 void AMDGPUAsmParser::cvtExp(MCInst &Inst, const OperandVector &Operands) {
2232   OptionalImmIndexMap OptionalIdx;
2233 
2234   unsigned EnMask = 0;
2235   int SrcIdx = 0;
2236 
2237   for (unsigned i = 1, e = Operands.size(); i != e; ++i) {
2238     AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[i]);
2239 
2240     // Add the register arguments
2241     if (Op.isReg()) {
2242       EnMask |= (1 << SrcIdx);
2243       Op.addRegOperands(Inst, 1);
2244       ++SrcIdx;
2245       continue;
2246     }
2247 
2248     if (Op.isOff()) {
2249       ++SrcIdx;
2250       Inst.addOperand(MCOperand::createReg(AMDGPU::NoRegister));
2251       continue;
2252     }
2253 
2254     if (Op.isImm() && Op.getImmTy() == AMDGPUOperand::ImmTyExpTgt) {
2255       Op.addImmOperands(Inst, 1);
2256       continue;
2257     }
2258 
2259     if (Op.isToken() && Op.getToken() == "done")
2260       continue;
2261 
2262     // Handle optional arguments
2263     OptionalIdx[Op.getImmTy()] = i;
2264   }
2265 
2266   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyExpVM);
2267   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyExpCompr);
2268 
2269   Inst.addOperand(MCOperand::createImm(EnMask));
2270 }
2271 
2272 //===----------------------------------------------------------------------===//
2273 // s_waitcnt
2274 //===----------------------------------------------------------------------===//
2275 
2276 bool AMDGPUAsmParser::parseCnt(int64_t &IntVal) {
2277   StringRef CntName = Parser.getTok().getString();
2278   int64_t CntVal;
2279 
2280   Parser.Lex();
2281   if (getLexer().isNot(AsmToken::LParen))
2282     return true;
2283 
2284   Parser.Lex();
2285   if (getLexer().isNot(AsmToken::Integer))
2286     return true;
2287 
2288   if (getParser().parseAbsoluteExpression(CntVal))
2289     return true;
2290 
2291   if (getLexer().isNot(AsmToken::RParen))
2292     return true;
2293 
2294   Parser.Lex();
2295   if (getLexer().is(AsmToken::Amp) || getLexer().is(AsmToken::Comma))
2296     Parser.Lex();
2297 
2298   IsaVersion IV = getIsaVersion(getSTI().getFeatureBits());
2299   if (CntName == "vmcnt")
2300     IntVal = encodeVmcnt(IV, IntVal, CntVal);
2301   else if (CntName == "expcnt")
2302     IntVal = encodeExpcnt(IV, IntVal, CntVal);
2303   else if (CntName == "lgkmcnt")
2304     IntVal = encodeLgkmcnt(IV, IntVal, CntVal);
2305   else
2306     return true;
2307 
2308   return false;
2309 }
2310 
2311 OperandMatchResultTy
2312 AMDGPUAsmParser::parseSWaitCntOps(OperandVector &Operands) {
2313   IsaVersion IV = getIsaVersion(getSTI().getFeatureBits());
2314   int64_t Waitcnt = getWaitcntBitMask(IV);
2315   SMLoc S = Parser.getTok().getLoc();
2316 
2317   switch(getLexer().getKind()) {
2318     default: return MatchOperand_ParseFail;
2319     case AsmToken::Integer:
2320       // The operand can be an integer value.
2321       if (getParser().parseAbsoluteExpression(Waitcnt))
2322         return MatchOperand_ParseFail;
2323       break;
2324 
2325     case AsmToken::Identifier:
2326       do {
2327         if (parseCnt(Waitcnt))
2328           return MatchOperand_ParseFail;
2329       } while(getLexer().isNot(AsmToken::EndOfStatement));
2330       break;
2331   }
2332   Operands.push_back(AMDGPUOperand::CreateImm(this, Waitcnt, S));
2333   return MatchOperand_Success;
2334 }
2335 
2336 bool AMDGPUAsmParser::parseHwregConstruct(OperandInfoTy &HwReg, int64_t &Offset, int64_t &Width) {
2337   using namespace llvm::AMDGPU::Hwreg;
2338 
2339   if (Parser.getTok().getString() != "hwreg")
2340     return true;
2341   Parser.Lex();
2342 
2343   if (getLexer().isNot(AsmToken::LParen))
2344     return true;
2345   Parser.Lex();
2346 
2347   if (getLexer().is(AsmToken::Identifier)) {
2348     HwReg.IsSymbolic = true;
2349     HwReg.Id = ID_UNKNOWN_;
2350     const StringRef tok = Parser.getTok().getString();
2351     for (int i = ID_SYMBOLIC_FIRST_; i < ID_SYMBOLIC_LAST_; ++i) {
2352       if (tok == IdSymbolic[i]) {
2353         HwReg.Id = i;
2354         break;
2355       }
2356     }
2357     Parser.Lex();
2358   } else {
2359     HwReg.IsSymbolic = false;
2360     if (getLexer().isNot(AsmToken::Integer))
2361       return true;
2362     if (getParser().parseAbsoluteExpression(HwReg.Id))
2363       return true;
2364   }
2365 
2366   if (getLexer().is(AsmToken::RParen)) {
2367     Parser.Lex();
2368     return false;
2369   }
2370 
2371   // optional params
2372   if (getLexer().isNot(AsmToken::Comma))
2373     return true;
2374   Parser.Lex();
2375 
2376   if (getLexer().isNot(AsmToken::Integer))
2377     return true;
2378   if (getParser().parseAbsoluteExpression(Offset))
2379     return true;
2380 
2381   if (getLexer().isNot(AsmToken::Comma))
2382     return true;
2383   Parser.Lex();
2384 
2385   if (getLexer().isNot(AsmToken::Integer))
2386     return true;
2387   if (getParser().parseAbsoluteExpression(Width))
2388     return true;
2389 
2390   if (getLexer().isNot(AsmToken::RParen))
2391     return true;
2392   Parser.Lex();
2393 
2394   return false;
2395 }
2396 
2397 OperandMatchResultTy
2398 AMDGPUAsmParser::parseHwreg(OperandVector &Operands) {
2399   using namespace llvm::AMDGPU::Hwreg;
2400 
2401   int64_t Imm16Val = 0;
2402   SMLoc S = Parser.getTok().getLoc();
2403 
2404   switch(getLexer().getKind()) {
2405     default: return MatchOperand_NoMatch;
2406     case AsmToken::Integer:
2407       // The operand can be an integer value.
2408       if (getParser().parseAbsoluteExpression(Imm16Val))
2409         return MatchOperand_NoMatch;
2410       if (Imm16Val < 0 || !isUInt<16>(Imm16Val)) {
2411         Error(S, "invalid immediate: only 16-bit values are legal");
2412         // Do not return error code, but create an imm operand anyway and proceed
2413         // to the next operand, if any. That avoids unneccessary error messages.
2414       }
2415       break;
2416 
2417     case AsmToken::Identifier: {
2418         OperandInfoTy HwReg(ID_UNKNOWN_);
2419         int64_t Offset = OFFSET_DEFAULT_;
2420         int64_t Width = WIDTH_M1_DEFAULT_ + 1;
2421         if (parseHwregConstruct(HwReg, Offset, Width))
2422           return MatchOperand_ParseFail;
2423         if (HwReg.Id < 0 || !isUInt<ID_WIDTH_>(HwReg.Id)) {
2424           if (HwReg.IsSymbolic)
2425             Error(S, "invalid symbolic name of hardware register");
2426           else
2427             Error(S, "invalid code of hardware register: only 6-bit values are legal");
2428         }
2429         if (Offset < 0 || !isUInt<OFFSET_WIDTH_>(Offset))
2430           Error(S, "invalid bit offset: only 5-bit values are legal");
2431         if ((Width-1) < 0 || !isUInt<WIDTH_M1_WIDTH_>(Width-1))
2432           Error(S, "invalid bitfield width: only values from 1 to 32 are legal");
2433         Imm16Val = (HwReg.Id << ID_SHIFT_) | (Offset << OFFSET_SHIFT_) | ((Width-1) << WIDTH_M1_SHIFT_);
2434       }
2435       break;
2436   }
2437   Operands.push_back(AMDGPUOperand::CreateImm(this, Imm16Val, S, AMDGPUOperand::ImmTyHwreg));
2438   return MatchOperand_Success;
2439 }
2440 
2441 bool AMDGPUOperand::isSWaitCnt() const {
2442   return isImm();
2443 }
2444 
2445 bool AMDGPUOperand::isHwreg() const {
2446   return isImmTy(ImmTyHwreg);
2447 }
2448 
2449 bool AMDGPUAsmParser::parseSendMsgConstruct(OperandInfoTy &Msg, OperandInfoTy &Operation, int64_t &StreamId) {
2450   using namespace llvm::AMDGPU::SendMsg;
2451 
2452   if (Parser.getTok().getString() != "sendmsg")
2453     return true;
2454   Parser.Lex();
2455 
2456   if (getLexer().isNot(AsmToken::LParen))
2457     return true;
2458   Parser.Lex();
2459 
2460   if (getLexer().is(AsmToken::Identifier)) {
2461     Msg.IsSymbolic = true;
2462     Msg.Id = ID_UNKNOWN_;
2463     const std::string tok = Parser.getTok().getString();
2464     for (int i = ID_GAPS_FIRST_; i < ID_GAPS_LAST_; ++i) {
2465       switch(i) {
2466         default: continue; // Omit gaps.
2467         case ID_INTERRUPT: case ID_GS: case ID_GS_DONE:  case ID_SYSMSG: break;
2468       }
2469       if (tok == IdSymbolic[i]) {
2470         Msg.Id = i;
2471         break;
2472       }
2473     }
2474     Parser.Lex();
2475   } else {
2476     Msg.IsSymbolic = false;
2477     if (getLexer().isNot(AsmToken::Integer))
2478       return true;
2479     if (getParser().parseAbsoluteExpression(Msg.Id))
2480       return true;
2481     if (getLexer().is(AsmToken::Integer))
2482       if (getParser().parseAbsoluteExpression(Msg.Id))
2483         Msg.Id = ID_UNKNOWN_;
2484   }
2485   if (Msg.Id == ID_UNKNOWN_) // Don't know how to parse the rest.
2486     return false;
2487 
2488   if (!(Msg.Id == ID_GS || Msg.Id == ID_GS_DONE || Msg.Id == ID_SYSMSG)) {
2489     if (getLexer().isNot(AsmToken::RParen))
2490       return true;
2491     Parser.Lex();
2492     return false;
2493   }
2494 
2495   if (getLexer().isNot(AsmToken::Comma))
2496     return true;
2497   Parser.Lex();
2498 
2499   assert(Msg.Id == ID_GS || Msg.Id == ID_GS_DONE || Msg.Id == ID_SYSMSG);
2500   Operation.Id = ID_UNKNOWN_;
2501   if (getLexer().is(AsmToken::Identifier)) {
2502     Operation.IsSymbolic = true;
2503     const char* const *S = (Msg.Id == ID_SYSMSG) ? OpSysSymbolic : OpGsSymbolic;
2504     const int F = (Msg.Id == ID_SYSMSG) ? OP_SYS_FIRST_ : OP_GS_FIRST_;
2505     const int L = (Msg.Id == ID_SYSMSG) ? OP_SYS_LAST_ : OP_GS_LAST_;
2506     const StringRef Tok = Parser.getTok().getString();
2507     for (int i = F; i < L; ++i) {
2508       if (Tok == S[i]) {
2509         Operation.Id = i;
2510         break;
2511       }
2512     }
2513     Parser.Lex();
2514   } else {
2515     Operation.IsSymbolic = false;
2516     if (getLexer().isNot(AsmToken::Integer))
2517       return true;
2518     if (getParser().parseAbsoluteExpression(Operation.Id))
2519       return true;
2520   }
2521 
2522   if ((Msg.Id == ID_GS || Msg.Id == ID_GS_DONE) && Operation.Id != OP_GS_NOP) {
2523     // Stream id is optional.
2524     if (getLexer().is(AsmToken::RParen)) {
2525       Parser.Lex();
2526       return false;
2527     }
2528 
2529     if (getLexer().isNot(AsmToken::Comma))
2530       return true;
2531     Parser.Lex();
2532 
2533     if (getLexer().isNot(AsmToken::Integer))
2534       return true;
2535     if (getParser().parseAbsoluteExpression(StreamId))
2536       return true;
2537   }
2538 
2539   if (getLexer().isNot(AsmToken::RParen))
2540     return true;
2541   Parser.Lex();
2542   return false;
2543 }
2544 
2545 OperandMatchResultTy AMDGPUAsmParser::parseInterpSlot(OperandVector &Operands) {
2546   if (getLexer().getKind() != AsmToken::Identifier)
2547     return MatchOperand_NoMatch;
2548 
2549   StringRef Str = Parser.getTok().getString();
2550   int Slot = StringSwitch<int>(Str)
2551     .Case("p10", 0)
2552     .Case("p20", 1)
2553     .Case("p0", 2)
2554     .Default(-1);
2555 
2556   SMLoc S = Parser.getTok().getLoc();
2557   if (Slot == -1)
2558     return MatchOperand_ParseFail;
2559 
2560   Parser.Lex();
2561   Operands.push_back(AMDGPUOperand::CreateImm(this, Slot, S,
2562                                               AMDGPUOperand::ImmTyInterpSlot));
2563   return MatchOperand_Success;
2564 }
2565 
2566 OperandMatchResultTy AMDGPUAsmParser::parseInterpAttr(OperandVector &Operands) {
2567   if (getLexer().getKind() != AsmToken::Identifier)
2568     return MatchOperand_NoMatch;
2569 
2570   StringRef Str = Parser.getTok().getString();
2571   if (!Str.startswith("attr"))
2572     return MatchOperand_NoMatch;
2573 
2574   StringRef Chan = Str.take_back(2);
2575   int AttrChan = StringSwitch<int>(Chan)
2576     .Case(".x", 0)
2577     .Case(".y", 1)
2578     .Case(".z", 2)
2579     .Case(".w", 3)
2580     .Default(-1);
2581   if (AttrChan == -1)
2582     return MatchOperand_ParseFail;
2583 
2584   Str = Str.drop_back(2).drop_front(4);
2585 
2586   uint8_t Attr;
2587   if (Str.getAsInteger(10, Attr))
2588     return MatchOperand_ParseFail;
2589 
2590   SMLoc S = Parser.getTok().getLoc();
2591   Parser.Lex();
2592   if (Attr > 63) {
2593     Error(S, "out of bounds attr");
2594     return MatchOperand_Success;
2595   }
2596 
2597   SMLoc SChan = SMLoc::getFromPointer(Chan.data());
2598 
2599   Operands.push_back(AMDGPUOperand::CreateImm(this, Attr, S,
2600                                               AMDGPUOperand::ImmTyInterpAttr));
2601   Operands.push_back(AMDGPUOperand::CreateImm(this, AttrChan, SChan,
2602                                               AMDGPUOperand::ImmTyAttrChan));
2603   return MatchOperand_Success;
2604 }
2605 
2606 void AMDGPUAsmParser::errorExpTgt() {
2607   Error(Parser.getTok().getLoc(), "invalid exp target");
2608 }
2609 
2610 OperandMatchResultTy AMDGPUAsmParser::parseExpTgtImpl(StringRef Str,
2611                                                       uint8_t &Val) {
2612   if (Str == "null") {
2613     Val = 9;
2614     return MatchOperand_Success;
2615   }
2616 
2617   if (Str.startswith("mrt")) {
2618     Str = Str.drop_front(3);
2619     if (Str == "z") { // == mrtz
2620       Val = 8;
2621       return MatchOperand_Success;
2622     }
2623 
2624     if (Str.getAsInteger(10, Val))
2625       return MatchOperand_ParseFail;
2626 
2627     if (Val > 7)
2628       errorExpTgt();
2629 
2630     return MatchOperand_Success;
2631   }
2632 
2633   if (Str.startswith("pos")) {
2634     Str = Str.drop_front(3);
2635     if (Str.getAsInteger(10, Val))
2636       return MatchOperand_ParseFail;
2637 
2638     if (Val > 3)
2639       errorExpTgt();
2640 
2641     Val += 12;
2642     return MatchOperand_Success;
2643   }
2644 
2645   if (Str.startswith("param")) {
2646     Str = Str.drop_front(5);
2647     if (Str.getAsInteger(10, Val))
2648       return MatchOperand_ParseFail;
2649 
2650     if (Val >= 32)
2651       errorExpTgt();
2652 
2653     Val += 32;
2654     return MatchOperand_Success;
2655   }
2656 
2657   if (Str.startswith("invalid_target_")) {
2658     Str = Str.drop_front(15);
2659     if (Str.getAsInteger(10, Val))
2660       return MatchOperand_ParseFail;
2661 
2662     errorExpTgt();
2663     return MatchOperand_Success;
2664   }
2665 
2666   return MatchOperand_NoMatch;
2667 }
2668 
2669 OperandMatchResultTy AMDGPUAsmParser::parseExpTgt(OperandVector &Operands) {
2670   uint8_t Val;
2671   StringRef Str = Parser.getTok().getString();
2672 
2673   auto Res = parseExpTgtImpl(Str, Val);
2674   if (Res != MatchOperand_Success)
2675     return Res;
2676 
2677   SMLoc S = Parser.getTok().getLoc();
2678   Parser.Lex();
2679 
2680   Operands.push_back(AMDGPUOperand::CreateImm(this, Val, S,
2681                                               AMDGPUOperand::ImmTyExpTgt));
2682   return MatchOperand_Success;
2683 }
2684 
2685 OperandMatchResultTy
2686 AMDGPUAsmParser::parseSendMsgOp(OperandVector &Operands) {
2687   using namespace llvm::AMDGPU::SendMsg;
2688 
2689   int64_t Imm16Val = 0;
2690   SMLoc S = Parser.getTok().getLoc();
2691 
2692   switch(getLexer().getKind()) {
2693   default:
2694     return MatchOperand_NoMatch;
2695   case AsmToken::Integer:
2696     // The operand can be an integer value.
2697     if (getParser().parseAbsoluteExpression(Imm16Val))
2698       return MatchOperand_NoMatch;
2699     if (Imm16Val < 0 || !isUInt<16>(Imm16Val)) {
2700       Error(S, "invalid immediate: only 16-bit values are legal");
2701       // Do not return error code, but create an imm operand anyway and proceed
2702       // to the next operand, if any. That avoids unneccessary error messages.
2703     }
2704     break;
2705   case AsmToken::Identifier: {
2706       OperandInfoTy Msg(ID_UNKNOWN_);
2707       OperandInfoTy Operation(OP_UNKNOWN_);
2708       int64_t StreamId = STREAM_ID_DEFAULT_;
2709       if (parseSendMsgConstruct(Msg, Operation, StreamId))
2710         return MatchOperand_ParseFail;
2711       do {
2712         // Validate and encode message ID.
2713         if (! ((ID_INTERRUPT <= Msg.Id && Msg.Id <= ID_GS_DONE)
2714                 || Msg.Id == ID_SYSMSG)) {
2715           if (Msg.IsSymbolic)
2716             Error(S, "invalid/unsupported symbolic name of message");
2717           else
2718             Error(S, "invalid/unsupported code of message");
2719           break;
2720         }
2721         Imm16Val = (Msg.Id << ID_SHIFT_);
2722         // Validate and encode operation ID.
2723         if (Msg.Id == ID_GS || Msg.Id == ID_GS_DONE) {
2724           if (! (OP_GS_FIRST_ <= Operation.Id && Operation.Id < OP_GS_LAST_)) {
2725             if (Operation.IsSymbolic)
2726               Error(S, "invalid symbolic name of GS_OP");
2727             else
2728               Error(S, "invalid code of GS_OP: only 2-bit values are legal");
2729             break;
2730           }
2731           if (Operation.Id == OP_GS_NOP
2732               && Msg.Id != ID_GS_DONE) {
2733             Error(S, "invalid GS_OP: NOP is for GS_DONE only");
2734             break;
2735           }
2736           Imm16Val |= (Operation.Id << OP_SHIFT_);
2737         }
2738         if (Msg.Id == ID_SYSMSG) {
2739           if (! (OP_SYS_FIRST_ <= Operation.Id && Operation.Id < OP_SYS_LAST_)) {
2740             if (Operation.IsSymbolic)
2741               Error(S, "invalid/unsupported symbolic name of SYSMSG_OP");
2742             else
2743               Error(S, "invalid/unsupported code of SYSMSG_OP");
2744             break;
2745           }
2746           Imm16Val |= (Operation.Id << OP_SHIFT_);
2747         }
2748         // Validate and encode stream ID.
2749         if ((Msg.Id == ID_GS || Msg.Id == ID_GS_DONE) && Operation.Id != OP_GS_NOP) {
2750           if (! (STREAM_ID_FIRST_ <= StreamId && StreamId < STREAM_ID_LAST_)) {
2751             Error(S, "invalid stream id: only 2-bit values are legal");
2752             break;
2753           }
2754           Imm16Val |= (StreamId << STREAM_ID_SHIFT_);
2755         }
2756       } while (false);
2757     }
2758     break;
2759   }
2760   Operands.push_back(AMDGPUOperand::CreateImm(this, Imm16Val, S, AMDGPUOperand::ImmTySendMsg));
2761   return MatchOperand_Success;
2762 }
2763 
2764 bool AMDGPUOperand::isSendMsg() const {
2765   return isImmTy(ImmTySendMsg);
2766 }
2767 
2768 //===----------------------------------------------------------------------===//
2769 // sopp branch targets
2770 //===----------------------------------------------------------------------===//
2771 
2772 OperandMatchResultTy
2773 AMDGPUAsmParser::parseSOppBrTarget(OperandVector &Operands) {
2774   SMLoc S = Parser.getTok().getLoc();
2775 
2776   switch (getLexer().getKind()) {
2777     default: return MatchOperand_ParseFail;
2778     case AsmToken::Integer: {
2779       int64_t Imm;
2780       if (getParser().parseAbsoluteExpression(Imm))
2781         return MatchOperand_ParseFail;
2782       Operands.push_back(AMDGPUOperand::CreateImm(this, Imm, S));
2783       return MatchOperand_Success;
2784     }
2785 
2786     case AsmToken::Identifier:
2787       Operands.push_back(AMDGPUOperand::CreateExpr(this,
2788           MCSymbolRefExpr::create(getContext().getOrCreateSymbol(
2789                                   Parser.getTok().getString()), getContext()), S));
2790       Parser.Lex();
2791       return MatchOperand_Success;
2792   }
2793 }
2794 
2795 //===----------------------------------------------------------------------===//
2796 // mubuf
2797 //===----------------------------------------------------------------------===//
2798 
2799 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultGLC() const {
2800   return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTyGLC);
2801 }
2802 
2803 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultSLC() const {
2804   return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTySLC);
2805 }
2806 
2807 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultTFE() const {
2808   return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTyTFE);
2809 }
2810 
2811 void AMDGPUAsmParser::cvtMubufImpl(MCInst &Inst,
2812                                const OperandVector &Operands,
2813                                bool IsAtomic, bool IsAtomicReturn) {
2814   OptionalImmIndexMap OptionalIdx;
2815   assert(IsAtomicReturn ? IsAtomic : true);
2816 
2817   for (unsigned i = 1, e = Operands.size(); i != e; ++i) {
2818     AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[i]);
2819 
2820     // Add the register arguments
2821     if (Op.isReg()) {
2822       Op.addRegOperands(Inst, 1);
2823       continue;
2824     }
2825 
2826     // Handle the case where soffset is an immediate
2827     if (Op.isImm() && Op.getImmTy() == AMDGPUOperand::ImmTyNone) {
2828       Op.addImmOperands(Inst, 1);
2829       continue;
2830     }
2831 
2832     // Handle tokens like 'offen' which are sometimes hard-coded into the
2833     // asm string.  There are no MCInst operands for these.
2834     if (Op.isToken()) {
2835       continue;
2836     }
2837     assert(Op.isImm());
2838 
2839     // Handle optional arguments
2840     OptionalIdx[Op.getImmTy()] = i;
2841   }
2842 
2843   // Copy $vdata_in operand and insert as $vdata for MUBUF_Atomic RTN insns.
2844   if (IsAtomicReturn) {
2845     MCInst::iterator I = Inst.begin(); // $vdata_in is always at the beginning.
2846     Inst.insert(I, *I);
2847   }
2848 
2849   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyOffset);
2850   if (!IsAtomic) { // glc is hard-coded.
2851     addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyGLC);
2852   }
2853   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySLC);
2854   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyTFE);
2855 }
2856 
2857 //===----------------------------------------------------------------------===//
2858 // mimg
2859 //===----------------------------------------------------------------------===//
2860 
2861 void AMDGPUAsmParser::cvtMIMG(MCInst &Inst, const OperandVector &Operands) {
2862   unsigned I = 1;
2863   const MCInstrDesc &Desc = MII.get(Inst.getOpcode());
2864   for (unsigned J = 0; J < Desc.getNumDefs(); ++J) {
2865     ((AMDGPUOperand &)*Operands[I++]).addRegOperands(Inst, 1);
2866   }
2867 
2868   OptionalImmIndexMap OptionalIdx;
2869 
2870   for (unsigned E = Operands.size(); I != E; ++I) {
2871     AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[I]);
2872 
2873     // Add the register arguments
2874     if (Op.isRegOrImm()) {
2875       Op.addRegOrImmOperands(Inst, 1);
2876       continue;
2877     } else if (Op.isImmModifier()) {
2878       OptionalIdx[Op.getImmTy()] = I;
2879     } else {
2880       llvm_unreachable("unexpected operand type");
2881     }
2882   }
2883 
2884   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyDMask);
2885   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyUNorm);
2886   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyGLC);
2887   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyDA);
2888   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyR128);
2889   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyTFE);
2890   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyLWE);
2891   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySLC);
2892 }
2893 
2894 void AMDGPUAsmParser::cvtMIMGAtomic(MCInst &Inst, const OperandVector &Operands) {
2895   unsigned I = 1;
2896   const MCInstrDesc &Desc = MII.get(Inst.getOpcode());
2897   for (unsigned J = 0; J < Desc.getNumDefs(); ++J) {
2898     ((AMDGPUOperand &)*Operands[I++]).addRegOperands(Inst, 1);
2899   }
2900 
2901   // Add src, same as dst
2902   ((AMDGPUOperand &)*Operands[I]).addRegOperands(Inst, 1);
2903 
2904   OptionalImmIndexMap OptionalIdx;
2905 
2906   for (unsigned E = Operands.size(); I != E; ++I) {
2907     AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[I]);
2908 
2909     // Add the register arguments
2910     if (Op.isRegOrImm()) {
2911       Op.addRegOrImmOperands(Inst, 1);
2912       continue;
2913     } else if (Op.isImmModifier()) {
2914       OptionalIdx[Op.getImmTy()] = I;
2915     } else {
2916       llvm_unreachable("unexpected operand type");
2917     }
2918   }
2919 
2920   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyDMask);
2921   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyUNorm);
2922   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyGLC);
2923   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyDA);
2924   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyR128);
2925   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyTFE);
2926   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyLWE);
2927   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySLC);
2928 }
2929 
2930 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultDMask() const {
2931   return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTyDMask);
2932 }
2933 
2934 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultUNorm() const {
2935   return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTyUNorm);
2936 }
2937 
2938 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultDA() const {
2939   return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTyDA);
2940 }
2941 
2942 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultR128() const {
2943   return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTyR128);
2944 }
2945 
2946 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultLWE() const {
2947   return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTyLWE);
2948 }
2949 
2950 //===----------------------------------------------------------------------===//
2951 // smrd
2952 //===----------------------------------------------------------------------===//
2953 
2954 bool AMDGPUOperand::isSMRDOffset8() const {
2955   return isImm() && isUInt<8>(getImm());
2956 }
2957 
2958 bool AMDGPUOperand::isSMRDOffset20() const {
2959   return isImm() && isUInt<20>(getImm());
2960 }
2961 
2962 bool AMDGPUOperand::isSMRDLiteralOffset() const {
2963   // 32-bit literals are only supported on CI and we only want to use them
2964   // when the offset is > 8-bits.
2965   return isImm() && !isUInt<8>(getImm()) && isUInt<32>(getImm());
2966 }
2967 
2968 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultSMRDOffset8() const {
2969   return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTyOffset);
2970 }
2971 
2972 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultSMRDOffset20() const {
2973   return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTyOffset);
2974 }
2975 
2976 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultSMRDLiteralOffset() const {
2977   return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTyOffset);
2978 }
2979 
2980 //===----------------------------------------------------------------------===//
2981 // vop3
2982 //===----------------------------------------------------------------------===//
2983 
2984 static bool ConvertOmodMul(int64_t &Mul) {
2985   if (Mul != 1 && Mul != 2 && Mul != 4)
2986     return false;
2987 
2988   Mul >>= 1;
2989   return true;
2990 }
2991 
2992 static bool ConvertOmodDiv(int64_t &Div) {
2993   if (Div == 1) {
2994     Div = 0;
2995     return true;
2996   }
2997 
2998   if (Div == 2) {
2999     Div = 3;
3000     return true;
3001   }
3002 
3003   return false;
3004 }
3005 
3006 static bool ConvertBoundCtrl(int64_t &BoundCtrl) {
3007   if (BoundCtrl == 0) {
3008     BoundCtrl = 1;
3009     return true;
3010   }
3011 
3012   if (BoundCtrl == -1) {
3013     BoundCtrl = 0;
3014     return true;
3015   }
3016 
3017   return false;
3018 }
3019 
3020 // Note: the order in this table matches the order of operands in AsmString.
3021 static const OptionalOperand AMDGPUOptionalOperandTable[] = {
3022   {"offen",   AMDGPUOperand::ImmTyOffen, true, nullptr},
3023   {"idxen",   AMDGPUOperand::ImmTyIdxen, true, nullptr},
3024   {"addr64",  AMDGPUOperand::ImmTyAddr64, true, nullptr},
3025   {"offset0", AMDGPUOperand::ImmTyOffset0, false, nullptr},
3026   {"offset1", AMDGPUOperand::ImmTyOffset1, false, nullptr},
3027   {"gds",     AMDGPUOperand::ImmTyGDS, true, nullptr},
3028   {"offset",  AMDGPUOperand::ImmTyOffset, false, nullptr},
3029   {"glc",     AMDGPUOperand::ImmTyGLC, true, nullptr},
3030   {"slc",     AMDGPUOperand::ImmTySLC, true, nullptr},
3031   {"tfe",     AMDGPUOperand::ImmTyTFE, true, nullptr},
3032   {"clamp",   AMDGPUOperand::ImmTyClampSI, true, nullptr},
3033   {"omod",    AMDGPUOperand::ImmTyOModSI, false, ConvertOmodMul},
3034   {"unorm",   AMDGPUOperand::ImmTyUNorm, true, nullptr},
3035   {"da",      AMDGPUOperand::ImmTyDA,    true, nullptr},
3036   {"r128",    AMDGPUOperand::ImmTyR128,  true, nullptr},
3037   {"lwe",     AMDGPUOperand::ImmTyLWE,   true, nullptr},
3038   {"dmask",   AMDGPUOperand::ImmTyDMask, false, nullptr},
3039   {"row_mask",   AMDGPUOperand::ImmTyDppRowMask, false, nullptr},
3040   {"bank_mask",  AMDGPUOperand::ImmTyDppBankMask, false, nullptr},
3041   {"bound_ctrl", AMDGPUOperand::ImmTyDppBoundCtrl, false, ConvertBoundCtrl},
3042   {"dst_sel",    AMDGPUOperand::ImmTySdwaDstSel, false, nullptr},
3043   {"src0_sel",   AMDGPUOperand::ImmTySdwaSrc0Sel, false, nullptr},
3044   {"src1_sel",   AMDGPUOperand::ImmTySdwaSrc1Sel, false, nullptr},
3045   {"dst_unused", AMDGPUOperand::ImmTySdwaDstUnused, false, nullptr},
3046   {"vm", AMDGPUOperand::ImmTyExpVM, true, nullptr},
3047 };
3048 
3049 OperandMatchResultTy AMDGPUAsmParser::parseOptionalOperand(OperandVector &Operands) {
3050   OperandMatchResultTy res;
3051   for (const OptionalOperand &Op : AMDGPUOptionalOperandTable) {
3052     // try to parse any optional operand here
3053     if (Op.IsBit) {
3054       res = parseNamedBit(Op.Name, Operands, Op.Type);
3055     } else if (Op.Type == AMDGPUOperand::ImmTyOModSI) {
3056       res = parseOModOperand(Operands);
3057     } else if (Op.Type == AMDGPUOperand::ImmTySdwaDstSel ||
3058                Op.Type == AMDGPUOperand::ImmTySdwaSrc0Sel ||
3059                Op.Type == AMDGPUOperand::ImmTySdwaSrc1Sel) {
3060       res = parseSDWASel(Operands, Op.Name, Op.Type);
3061     } else if (Op.Type == AMDGPUOperand::ImmTySdwaDstUnused) {
3062       res = parseSDWADstUnused(Operands);
3063     } else {
3064       res = parseIntWithPrefix(Op.Name, Operands, Op.Type, Op.ConvertResult);
3065     }
3066     if (res != MatchOperand_NoMatch) {
3067       return res;
3068     }
3069   }
3070   return MatchOperand_NoMatch;
3071 }
3072 
3073 OperandMatchResultTy AMDGPUAsmParser::parseOModOperand(OperandVector &Operands) {
3074   StringRef Name = Parser.getTok().getString();
3075   if (Name == "mul") {
3076     return parseIntWithPrefix("mul", Operands,
3077                               AMDGPUOperand::ImmTyOModSI, ConvertOmodMul);
3078   }
3079 
3080   if (Name == "div") {
3081     return parseIntWithPrefix("div", Operands,
3082                               AMDGPUOperand::ImmTyOModSI, ConvertOmodDiv);
3083   }
3084 
3085   return MatchOperand_NoMatch;
3086 }
3087 
3088 void AMDGPUAsmParser::cvtId(MCInst &Inst, const OperandVector &Operands) {
3089   unsigned I = 1;
3090   const MCInstrDesc &Desc = MII.get(Inst.getOpcode());
3091   for (unsigned J = 0; J < Desc.getNumDefs(); ++J) {
3092     ((AMDGPUOperand &)*Operands[I++]).addRegOperands(Inst, 1);
3093   }
3094   for (unsigned E = Operands.size(); I != E; ++I)
3095     ((AMDGPUOperand &)*Operands[I]).addRegOrImmOperands(Inst, 1);
3096 }
3097 
3098 void AMDGPUAsmParser::cvtVOP3_2_mod(MCInst &Inst, const OperandVector &Operands) {
3099   uint64_t TSFlags = MII.get(Inst.getOpcode()).TSFlags;
3100   if (TSFlags & SIInstrFlags::VOP3) {
3101     cvtVOP3(Inst, Operands);
3102   } else {
3103     cvtId(Inst, Operands);
3104   }
3105 }
3106 
3107 static bool isRegOrImmWithInputMods(const MCInstrDesc &Desc, unsigned OpNum) {
3108       // 1. This operand is input modifiers
3109   return Desc.OpInfo[OpNum].OperandType == AMDGPU::OPERAND_INPUT_MODS
3110       // 2. This is not last operand
3111       && Desc.NumOperands > (OpNum + 1)
3112       // 3. Next operand is register class
3113       && Desc.OpInfo[OpNum + 1].RegClass != -1
3114       // 4. Next register is not tied to any other operand
3115       && Desc.getOperandConstraint(OpNum + 1, MCOI::OperandConstraint::TIED_TO) == -1;
3116 }
3117 
3118 void AMDGPUAsmParser::cvtVOP3(MCInst &Inst, const OperandVector &Operands) {
3119   OptionalImmIndexMap OptionalIdx;
3120   unsigned I = 1;
3121   const MCInstrDesc &Desc = MII.get(Inst.getOpcode());
3122   for (unsigned J = 0; J < Desc.getNumDefs(); ++J) {
3123     ((AMDGPUOperand &)*Operands[I++]).addRegOperands(Inst, 1);
3124   }
3125 
3126   for (unsigned E = Operands.size(); I != E; ++I) {
3127     AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[I]);
3128     if (isRegOrImmWithInputMods(Desc, Inst.getNumOperands())) {
3129       Op.addRegOrImmWithFPInputModsOperands(Inst, 2);
3130     } else if (Op.isImm()) {
3131       OptionalIdx[Op.getImmTy()] = I;
3132     } else {
3133       llvm_unreachable("unhandled operand type");
3134     }
3135   }
3136 
3137   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyClampSI);
3138   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyOModSI);
3139 
3140   // special case v_mac_{f16, f32}:
3141   // it has src2 register operand that is tied to dst operand
3142   // we don't allow modifiers for this operand in assembler so src2_modifiers
3143   // should be 0
3144   if (Inst.getOpcode() == AMDGPU::V_MAC_F32_e64_si ||
3145       Inst.getOpcode() == AMDGPU::V_MAC_F32_e64_vi ||
3146       Inst.getOpcode() == AMDGPU::V_MAC_F16_e64_vi) {
3147     auto it = Inst.begin();
3148     std::advance(
3149       it,
3150       AMDGPU::getNamedOperandIdx(Inst.getOpcode() == AMDGPU::V_MAC_F16_e64_vi ?
3151                                      AMDGPU::V_MAC_F16_e64 :
3152                                      AMDGPU::V_MAC_F32_e64,
3153                                  AMDGPU::OpName::src2_modifiers));
3154     it = Inst.insert(it, MCOperand::createImm(0)); // no modifiers for src2
3155     ++it;
3156     Inst.insert(it, Inst.getOperand(0)); // src2 = dst
3157   }
3158 }
3159 
3160 //===----------------------------------------------------------------------===//
3161 // dpp
3162 //===----------------------------------------------------------------------===//
3163 
3164 bool AMDGPUOperand::isDPPCtrl() const {
3165   bool result = isImm() && getImmTy() == ImmTyDppCtrl && isUInt<9>(getImm());
3166   if (result) {
3167     int64_t Imm = getImm();
3168     return ((Imm >= 0x000) && (Imm <= 0x0ff)) ||
3169            ((Imm >= 0x101) && (Imm <= 0x10f)) ||
3170            ((Imm >= 0x111) && (Imm <= 0x11f)) ||
3171            ((Imm >= 0x121) && (Imm <= 0x12f)) ||
3172            (Imm == 0x130) ||
3173            (Imm == 0x134) ||
3174            (Imm == 0x138) ||
3175            (Imm == 0x13c) ||
3176            (Imm == 0x140) ||
3177            (Imm == 0x141) ||
3178            (Imm == 0x142) ||
3179            (Imm == 0x143);
3180   }
3181   return false;
3182 }
3183 
3184 bool AMDGPUOperand::isGPRIdxMode() const {
3185   return isImm() && isUInt<4>(getImm());
3186 }
3187 
3188 OperandMatchResultTy
3189 AMDGPUAsmParser::parseDPPCtrl(OperandVector &Operands) {
3190   SMLoc S = Parser.getTok().getLoc();
3191   StringRef Prefix;
3192   int64_t Int;
3193 
3194   if (getLexer().getKind() == AsmToken::Identifier) {
3195     Prefix = Parser.getTok().getString();
3196   } else {
3197     return MatchOperand_NoMatch;
3198   }
3199 
3200   if (Prefix == "row_mirror") {
3201     Int = 0x140;
3202     Parser.Lex();
3203   } else if (Prefix == "row_half_mirror") {
3204     Int = 0x141;
3205     Parser.Lex();
3206   } else {
3207     // Check to prevent parseDPPCtrlOps from eating invalid tokens
3208     if (Prefix != "quad_perm"
3209         && Prefix != "row_shl"
3210         && Prefix != "row_shr"
3211         && Prefix != "row_ror"
3212         && Prefix != "wave_shl"
3213         && Prefix != "wave_rol"
3214         && Prefix != "wave_shr"
3215         && Prefix != "wave_ror"
3216         && Prefix != "row_bcast") {
3217       return MatchOperand_NoMatch;
3218     }
3219 
3220     Parser.Lex();
3221     if (getLexer().isNot(AsmToken::Colon))
3222       return MatchOperand_ParseFail;
3223 
3224     if (Prefix == "quad_perm") {
3225       // quad_perm:[%d,%d,%d,%d]
3226       Parser.Lex();
3227       if (getLexer().isNot(AsmToken::LBrac))
3228         return MatchOperand_ParseFail;
3229       Parser.Lex();
3230 
3231       if (getParser().parseAbsoluteExpression(Int) || !(0 <= Int && Int <=3))
3232         return MatchOperand_ParseFail;
3233 
3234       for (int i = 0; i < 3; ++i) {
3235         if (getLexer().isNot(AsmToken::Comma))
3236           return MatchOperand_ParseFail;
3237         Parser.Lex();
3238 
3239         int64_t Temp;
3240         if (getParser().parseAbsoluteExpression(Temp) || !(0 <= Temp && Temp <=3))
3241           return MatchOperand_ParseFail;
3242         const int shift = i*2 + 2;
3243         Int += (Temp << shift);
3244       }
3245 
3246       if (getLexer().isNot(AsmToken::RBrac))
3247         return MatchOperand_ParseFail;
3248       Parser.Lex();
3249 
3250     } else {
3251       // sel:%d
3252       Parser.Lex();
3253       if (getParser().parseAbsoluteExpression(Int))
3254         return MatchOperand_ParseFail;
3255 
3256       if (Prefix == "row_shl" && 1 <= Int && Int <= 15) {
3257         Int |= 0x100;
3258       } else if (Prefix == "row_shr" && 1 <= Int && Int <= 15) {
3259         Int |= 0x110;
3260       } else if (Prefix == "row_ror" && 1 <= Int && Int <= 15) {
3261         Int |= 0x120;
3262       } else if (Prefix == "wave_shl" && 1 == Int) {
3263         Int = 0x130;
3264       } else if (Prefix == "wave_rol" && 1 == Int) {
3265         Int = 0x134;
3266       } else if (Prefix == "wave_shr" && 1 == Int) {
3267         Int = 0x138;
3268       } else if (Prefix == "wave_ror" && 1 == Int) {
3269         Int = 0x13C;
3270       } else if (Prefix == "row_bcast") {
3271         if (Int == 15) {
3272           Int = 0x142;
3273         } else if (Int == 31) {
3274           Int = 0x143;
3275         } else {
3276           return MatchOperand_ParseFail;
3277         }
3278       } else {
3279         return MatchOperand_ParseFail;
3280       }
3281     }
3282   }
3283 
3284   Operands.push_back(AMDGPUOperand::CreateImm(this, Int, S, AMDGPUOperand::ImmTyDppCtrl));
3285   return MatchOperand_Success;
3286 }
3287 
3288 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultRowMask() const {
3289   return AMDGPUOperand::CreateImm(this, 0xf, SMLoc(), AMDGPUOperand::ImmTyDppRowMask);
3290 }
3291 
3292 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultBankMask() const {
3293   return AMDGPUOperand::CreateImm(this, 0xf, SMLoc(), AMDGPUOperand::ImmTyDppBankMask);
3294 }
3295 
3296 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultBoundCtrl() const {
3297   return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTyDppBoundCtrl);
3298 }
3299 
3300 void AMDGPUAsmParser::cvtDPP(MCInst &Inst, const OperandVector &Operands) {
3301   OptionalImmIndexMap OptionalIdx;
3302 
3303   unsigned I = 1;
3304   const MCInstrDesc &Desc = MII.get(Inst.getOpcode());
3305   for (unsigned J = 0; J < Desc.getNumDefs(); ++J) {
3306     ((AMDGPUOperand &)*Operands[I++]).addRegOperands(Inst, 1);
3307   }
3308 
3309   for (unsigned E = Operands.size(); I != E; ++I) {
3310     AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[I]);
3311     // Add the register arguments
3312     if (isRegOrImmWithInputMods(Desc, Inst.getNumOperands())) {
3313       Op.addRegOrImmWithFPInputModsOperands(Inst, 2);
3314     } else if (Op.isDPPCtrl()) {
3315       Op.addImmOperands(Inst, 1);
3316     } else if (Op.isImm()) {
3317       // Handle optional arguments
3318       OptionalIdx[Op.getImmTy()] = I;
3319     } else {
3320       llvm_unreachable("Invalid operand type");
3321     }
3322   }
3323 
3324   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyDppRowMask, 0xf);
3325   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyDppBankMask, 0xf);
3326   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyDppBoundCtrl);
3327 
3328   // special case v_mac_{f16, f32}:
3329   // it has src2 register operand that is tied to dst operand
3330   if (Inst.getOpcode() == AMDGPU::V_MAC_F32_dpp ||
3331       Inst.getOpcode() == AMDGPU::V_MAC_F16_dpp) {
3332     auto it = Inst.begin();
3333     std::advance(
3334         it, AMDGPU::getNamedOperandIdx(Inst.getOpcode(), AMDGPU::OpName::src2));
3335     Inst.insert(it, Inst.getOperand(0)); // src2 = dst
3336   }
3337 }
3338 
3339 //===----------------------------------------------------------------------===//
3340 // sdwa
3341 //===----------------------------------------------------------------------===//
3342 
3343 OperandMatchResultTy
3344 AMDGPUAsmParser::parseSDWASel(OperandVector &Operands, StringRef Prefix,
3345                               AMDGPUOperand::ImmTy Type) {
3346   using namespace llvm::AMDGPU::SDWA;
3347 
3348   SMLoc S = Parser.getTok().getLoc();
3349   StringRef Value;
3350   OperandMatchResultTy res;
3351 
3352   res = parseStringWithPrefix(Prefix, Value);
3353   if (res != MatchOperand_Success) {
3354     return res;
3355   }
3356 
3357   int64_t Int;
3358   Int = StringSwitch<int64_t>(Value)
3359         .Case("BYTE_0", SdwaSel::BYTE_0)
3360         .Case("BYTE_1", SdwaSel::BYTE_1)
3361         .Case("BYTE_2", SdwaSel::BYTE_2)
3362         .Case("BYTE_3", SdwaSel::BYTE_3)
3363         .Case("WORD_0", SdwaSel::WORD_0)
3364         .Case("WORD_1", SdwaSel::WORD_1)
3365         .Case("DWORD", SdwaSel::DWORD)
3366         .Default(0xffffffff);
3367   Parser.Lex(); // eat last token
3368 
3369   if (Int == 0xffffffff) {
3370     return MatchOperand_ParseFail;
3371   }
3372 
3373   Operands.push_back(AMDGPUOperand::CreateImm(this, Int, S, Type));
3374   return MatchOperand_Success;
3375 }
3376 
3377 OperandMatchResultTy
3378 AMDGPUAsmParser::parseSDWADstUnused(OperandVector &Operands) {
3379   using namespace llvm::AMDGPU::SDWA;
3380 
3381   SMLoc S = Parser.getTok().getLoc();
3382   StringRef Value;
3383   OperandMatchResultTy res;
3384 
3385   res = parseStringWithPrefix("dst_unused", Value);
3386   if (res != MatchOperand_Success) {
3387     return res;
3388   }
3389 
3390   int64_t Int;
3391   Int = StringSwitch<int64_t>(Value)
3392         .Case("UNUSED_PAD", DstUnused::UNUSED_PAD)
3393         .Case("UNUSED_SEXT", DstUnused::UNUSED_SEXT)
3394         .Case("UNUSED_PRESERVE", DstUnused::UNUSED_PRESERVE)
3395         .Default(0xffffffff);
3396   Parser.Lex(); // eat last token
3397 
3398   if (Int == 0xffffffff) {
3399     return MatchOperand_ParseFail;
3400   }
3401 
3402   Operands.push_back(AMDGPUOperand::CreateImm(this, Int, S, AMDGPUOperand::ImmTySdwaDstUnused));
3403   return MatchOperand_Success;
3404 }
3405 
3406 void AMDGPUAsmParser::cvtSdwaVOP1(MCInst &Inst, const OperandVector &Operands) {
3407   cvtSDWA(Inst, Operands, SIInstrFlags::VOP1);
3408 }
3409 
3410 void AMDGPUAsmParser::cvtSdwaVOP2(MCInst &Inst, const OperandVector &Operands) {
3411   cvtSDWA(Inst, Operands, SIInstrFlags::VOP2);
3412 }
3413 
3414 void AMDGPUAsmParser::cvtSdwaVOPC(MCInst &Inst, const OperandVector &Operands) {
3415   cvtSDWA(Inst, Operands, SIInstrFlags::VOPC);
3416 }
3417 
3418 void AMDGPUAsmParser::cvtSDWA(MCInst &Inst, const OperandVector &Operands,
3419                               uint64_t BasicInstType) {
3420   OptionalImmIndexMap OptionalIdx;
3421 
3422   unsigned I = 1;
3423   const MCInstrDesc &Desc = MII.get(Inst.getOpcode());
3424   for (unsigned J = 0; J < Desc.getNumDefs(); ++J) {
3425     ((AMDGPUOperand &)*Operands[I++]).addRegOperands(Inst, 1);
3426   }
3427 
3428   for (unsigned E = Operands.size(); I != E; ++I) {
3429     AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[I]);
3430     // Add the register arguments
3431     if (BasicInstType == SIInstrFlags::VOPC &&
3432         Op.isReg() &&
3433         Op.Reg.RegNo == AMDGPU::VCC) {
3434       // VOPC sdwa use "vcc" token as dst. Skip it.
3435       continue;
3436     } else if (isRegOrImmWithInputMods(Desc, Inst.getNumOperands())) {
3437       Op.addRegOrImmWithInputModsOperands(Inst, 2);
3438     } else if (Op.isImm()) {
3439       // Handle optional arguments
3440       OptionalIdx[Op.getImmTy()] = I;
3441     } else {
3442       llvm_unreachable("Invalid operand type");
3443     }
3444   }
3445 
3446   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyClampSI, 0);
3447 
3448   if (Inst.getOpcode() != AMDGPU::V_NOP_sdwa_vi) {
3449     // V_NOP_sdwa_vi has no optional sdwa arguments
3450     switch (BasicInstType) {
3451     case SIInstrFlags::VOP1:
3452       addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySdwaDstSel, 6);
3453       addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySdwaDstUnused, 2);
3454       addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySdwaSrc0Sel, 6);
3455       break;
3456 
3457     case SIInstrFlags::VOP2:
3458       addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySdwaDstSel, 6);
3459       addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySdwaDstUnused, 2);
3460       addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySdwaSrc0Sel, 6);
3461       addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySdwaSrc1Sel, 6);
3462       break;
3463 
3464     case SIInstrFlags::VOPC:
3465       addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySdwaSrc0Sel, 6);
3466       addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySdwaSrc1Sel, 6);
3467       break;
3468 
3469     default:
3470       llvm_unreachable("Invalid instruction type. Only VOP1, VOP2 and VOPC allowed");
3471     }
3472   }
3473 
3474   // special case v_mac_{f16, f32}:
3475   // it has src2 register operand that is tied to dst operand
3476   if (Inst.getOpcode() == AMDGPU::V_MAC_F32_sdwa_vi ||
3477       Inst.getOpcode() == AMDGPU::V_MAC_F16_sdwa_vi)  {
3478     auto it = Inst.begin();
3479     std::advance(
3480         it, AMDGPU::getNamedOperandIdx(Inst.getOpcode(), AMDGPU::OpName::src2));
3481     Inst.insert(it, Inst.getOperand(0)); // src2 = dst
3482   }
3483 
3484 }
3485 
3486 /// Force static initialization.
3487 extern "C" void LLVMInitializeAMDGPUAsmParser() {
3488   RegisterMCAsmParser<AMDGPUAsmParser> A(getTheAMDGPUTarget());
3489   RegisterMCAsmParser<AMDGPUAsmParser> B(getTheGCNTarget());
3490 }
3491 
3492 #define GET_REGISTER_MATCHER
3493 #define GET_MATCHER_IMPLEMENTATION
3494 #include "AMDGPUGenAsmMatcher.inc"
3495 
3496 // This fuction should be defined after auto-generated include so that we have
3497 // MatchClassKind enum defined
3498 unsigned AMDGPUAsmParser::validateTargetOperandClass(MCParsedAsmOperand &Op,
3499                                                      unsigned Kind) {
3500   // Tokens like "glc" would be parsed as immediate operands in ParseOperand().
3501   // But MatchInstructionImpl() expects to meet token and fails to validate
3502   // operand. This method checks if we are given immediate operand but expect to
3503   // get corresponding token.
3504   AMDGPUOperand &Operand = (AMDGPUOperand&)Op;
3505   switch (Kind) {
3506   case MCK_addr64:
3507     return Operand.isAddr64() ? Match_Success : Match_InvalidOperand;
3508   case MCK_gds:
3509     return Operand.isGDS() ? Match_Success : Match_InvalidOperand;
3510   case MCK_glc:
3511     return Operand.isGLC() ? Match_Success : Match_InvalidOperand;
3512   case MCK_idxen:
3513     return Operand.isIdxen() ? Match_Success : Match_InvalidOperand;
3514   case MCK_offen:
3515     return Operand.isOffen() ? Match_Success : Match_InvalidOperand;
3516   case MCK_SSrcB32:
3517     // When operands have expression values, they will return true for isToken,
3518     // because it is not possible to distinguish between a token and an
3519     // expression at parse time. MatchInstructionImpl() will always try to
3520     // match an operand as a token, when isToken returns true, and when the
3521     // name of the expression is not a valid token, the match will fail,
3522     // so we need to handle it here.
3523     return Operand.isSSrcB32() ? Match_Success : Match_InvalidOperand;
3524   case MCK_SSrcF32:
3525     return Operand.isSSrcF32() ? Match_Success : Match_InvalidOperand;
3526   case MCK_SoppBrTarget:
3527     return Operand.isSoppBrTarget() ? Match_Success : Match_InvalidOperand;
3528   case MCK_VReg32OrOff:
3529     return Operand.isVReg32OrOff() ? Match_Success : Match_InvalidOperand;
3530   case MCK_InterpSlot:
3531     return Operand.isInterpSlot() ? Match_Success : Match_InvalidOperand;
3532   case MCK_Attr:
3533     return Operand.isInterpAttr() ? Match_Success : Match_InvalidOperand;
3534   case MCK_AttrChan:
3535     return Operand.isAttrChan() ? Match_Success : Match_InvalidOperand;
3536   default:
3537     return Match_InvalidOperand;
3538   }
3539 }
3540