1 //===-- AMDGPUAsmParser.cpp - Parse SI asm to MCInst instructions ---------===//
2 //
3 //                     The LLVM Compiler Infrastructure
4 //
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
7 //
8 //===----------------------------------------------------------------------===//
9 
10 #include "AMDKernelCodeT.h"
11 #include "MCTargetDesc/AMDGPUMCTargetDesc.h"
12 #include "MCTargetDesc/AMDGPUTargetStreamer.h"
13 #include "SIDefines.h"
14 #include "Utils/AMDGPUBaseInfo.h"
15 #include "Utils/AMDKernelCodeTUtils.h"
16 #include "Utils/AMDGPUAsmUtils.h"
17 #include "llvm/ADT/APFloat.h"
18 #include "llvm/ADT/APInt.h"
19 #include "llvm/ADT/SmallBitVector.h"
20 #include "llvm/ADT/SmallString.h"
21 #include "llvm/ADT/STLExtras.h"
22 #include "llvm/ADT/StringRef.h"
23 #include "llvm/ADT/StringSwitch.h"
24 #include "llvm/ADT/Twine.h"
25 #include "llvm/CodeGen/MachineValueType.h"
26 #include "llvm/MC/MCAsmInfo.h"
27 #include "llvm/MC/MCContext.h"
28 #include "llvm/MC/MCExpr.h"
29 #include "llvm/MC/MCInst.h"
30 #include "llvm/MC/MCInstrDesc.h"
31 #include "llvm/MC/MCInstrInfo.h"
32 #include "llvm/MC/MCParser/MCAsmLexer.h"
33 #include "llvm/MC/MCParser/MCAsmParser.h"
34 #include "llvm/MC/MCParser/MCAsmParserExtension.h"
35 #include "llvm/MC/MCParser/MCParsedAsmOperand.h"
36 #include "llvm/MC/MCParser/MCTargetAsmParser.h"
37 #include "llvm/MC/MCRegisterInfo.h"
38 #include "llvm/MC/MCStreamer.h"
39 #include "llvm/MC/MCSubtargetInfo.h"
40 #include "llvm/MC/MCSymbol.h"
41 #include "llvm/Support/Casting.h"
42 #include "llvm/Support/Debug.h"
43 #include "llvm/Support/ELF.h"
44 #include "llvm/Support/ErrorHandling.h"
45 #include "llvm/Support/MathExtras.h"
46 #include "llvm/Support/raw_ostream.h"
47 #include "llvm/Support/SMLoc.h"
48 #include "llvm/Support/TargetRegistry.h"
49 #include "llvm/Support/raw_ostream.h"
50 #include "llvm/Support/MathExtras.h"
51 #include <algorithm>
52 #include <cassert>
53 #include <cstdint>
54 #include <cstring>
55 #include <iterator>
56 #include <map>
57 #include <memory>
58 #include <string>
59 #include <vector>
60 
61 using namespace llvm;
62 using namespace llvm::AMDGPU;
63 
64 namespace {
65 
66 class AMDGPUAsmParser;
67 
68 enum RegisterKind { IS_UNKNOWN, IS_VGPR, IS_SGPR, IS_TTMP, IS_SPECIAL };
69 
70 //===----------------------------------------------------------------------===//
71 // Operand
72 //===----------------------------------------------------------------------===//
73 
74 class AMDGPUOperand : public MCParsedAsmOperand {
75   enum KindTy {
76     Token,
77     Immediate,
78     Register,
79     Expression
80   } Kind;
81 
82   SMLoc StartLoc, EndLoc;
83   const AMDGPUAsmParser *AsmParser;
84 
85 public:
86   AMDGPUOperand(enum KindTy Kind_, const AMDGPUAsmParser *AsmParser_)
87     : MCParsedAsmOperand(), Kind(Kind_), AsmParser(AsmParser_) {}
88 
89   typedef std::unique_ptr<AMDGPUOperand> Ptr;
90 
91   struct Modifiers {
92     bool Abs = false;
93     bool Neg = false;
94     bool Sext = false;
95 
96     bool hasFPModifiers() const { return Abs || Neg; }
97     bool hasIntModifiers() const { return Sext; }
98     bool hasModifiers() const { return hasFPModifiers() || hasIntModifiers(); }
99 
100     int64_t getFPModifiersOperand() const {
101       int64_t Operand = 0;
102       Operand |= Abs ? SISrcMods::ABS : 0;
103       Operand |= Neg ? SISrcMods::NEG : 0;
104       return Operand;
105     }
106 
107     int64_t getIntModifiersOperand() const {
108       int64_t Operand = 0;
109       Operand |= Sext ? SISrcMods::SEXT : 0;
110       return Operand;
111     }
112 
113     int64_t getModifiersOperand() const {
114       assert(!(hasFPModifiers() && hasIntModifiers())
115            && "fp and int modifiers should not be used simultaneously");
116       if (hasFPModifiers()) {
117         return getFPModifiersOperand();
118       } else if (hasIntModifiers()) {
119         return getIntModifiersOperand();
120       } else {
121         return 0;
122       }
123     }
124 
125     friend raw_ostream &operator <<(raw_ostream &OS, AMDGPUOperand::Modifiers Mods);
126   };
127 
128   enum ImmTy {
129     ImmTyNone,
130     ImmTyGDS,
131     ImmTyOffen,
132     ImmTyIdxen,
133     ImmTyAddr64,
134     ImmTyOffset,
135     ImmTyOffset0,
136     ImmTyOffset1,
137     ImmTyGLC,
138     ImmTySLC,
139     ImmTyTFE,
140     ImmTyClampSI,
141     ImmTyOModSI,
142     ImmTyDppCtrl,
143     ImmTyDppRowMask,
144     ImmTyDppBankMask,
145     ImmTyDppBoundCtrl,
146     ImmTySdwaDstSel,
147     ImmTySdwaSrc0Sel,
148     ImmTySdwaSrc1Sel,
149     ImmTySdwaDstUnused,
150     ImmTyDMask,
151     ImmTyUNorm,
152     ImmTyDA,
153     ImmTyR128,
154     ImmTyLWE,
155     ImmTyExpTgt,
156     ImmTyExpCompr,
157     ImmTyExpVM,
158     ImmTyHwreg,
159     ImmTyOff,
160     ImmTySendMsg,
161     ImmTyInterpSlot,
162     ImmTyInterpAttr,
163     ImmTyAttrChan
164   };
165 
166   struct TokOp {
167     const char *Data;
168     unsigned Length;
169   };
170 
171   struct ImmOp {
172     int64_t Val;
173     ImmTy Type;
174     bool IsFPImm;
175     Modifiers Mods;
176   };
177 
178   struct RegOp {
179     unsigned RegNo;
180     bool IsForcedVOP3;
181     Modifiers Mods;
182   };
183 
184   union {
185     TokOp Tok;
186     ImmOp Imm;
187     RegOp Reg;
188     const MCExpr *Expr;
189   };
190 
191   bool isToken() const override {
192     if (Kind == Token)
193       return true;
194 
195     if (Kind != Expression || !Expr)
196       return false;
197 
198     // When parsing operands, we can't always tell if something was meant to be
199     // a token, like 'gds', or an expression that references a global variable.
200     // In this case, we assume the string is an expression, and if we need to
201     // interpret is a token, then we treat the symbol name as the token.
202     return isa<MCSymbolRefExpr>(Expr);
203   }
204 
205   bool isImm() const override {
206     return Kind == Immediate;
207   }
208 
209   bool isInlinableImm(MVT type) const;
210   bool isLiteralImm(MVT type) const;
211 
212   bool isRegKind() const {
213     return Kind == Register;
214   }
215 
216   bool isReg() const override {
217     return isRegKind() && !hasModifiers();
218   }
219 
220   bool isRegOrImmWithInputMods(MVT type) const {
221     return isRegKind() || isInlinableImm(type);
222   }
223 
224   bool isRegOrImmWithInt16InputMods() const {
225     return isRegOrImmWithInputMods(MVT::i16);
226   }
227 
228   bool isRegOrImmWithInt32InputMods() const {
229     return isRegOrImmWithInputMods(MVT::i32);
230   }
231 
232   bool isRegOrImmWithInt64InputMods() const {
233     return isRegOrImmWithInputMods(MVT::i64);
234   }
235 
236   bool isRegOrImmWithFP16InputMods() const {
237     return isRegOrImmWithInputMods(MVT::f16);
238   }
239 
240   bool isRegOrImmWithFP32InputMods() const {
241     return isRegOrImmWithInputMods(MVT::f32);
242   }
243 
244   bool isRegOrImmWithFP64InputMods() const {
245     return isRegOrImmWithInputMods(MVT::f64);
246   }
247 
248   bool isVReg() const {
249     return isRegClass(AMDGPU::VGPR_32RegClassID) ||
250            isRegClass(AMDGPU::VReg_64RegClassID) ||
251            isRegClass(AMDGPU::VReg_96RegClassID) ||
252            isRegClass(AMDGPU::VReg_128RegClassID) ||
253            isRegClass(AMDGPU::VReg_256RegClassID) ||
254            isRegClass(AMDGPU::VReg_512RegClassID);
255   }
256 
257   bool isVReg32OrOff() const {
258     return isOff() || isRegClass(AMDGPU::VGPR_32RegClassID);
259   }
260 
261   bool isImmTy(ImmTy ImmT) const {
262     return isImm() && Imm.Type == ImmT;
263   }
264 
265   bool isImmModifier() const {
266     return isImm() && Imm.Type != ImmTyNone;
267   }
268 
269   bool isClampSI() const { return isImmTy(ImmTyClampSI); }
270   bool isOModSI() const { return isImmTy(ImmTyOModSI); }
271   bool isDMask() const { return isImmTy(ImmTyDMask); }
272   bool isUNorm() const { return isImmTy(ImmTyUNorm); }
273   bool isDA() const { return isImmTy(ImmTyDA); }
274   bool isR128() const { return isImmTy(ImmTyUNorm); }
275   bool isLWE() const { return isImmTy(ImmTyLWE); }
276   bool isOff() const { return isImmTy(ImmTyOff); }
277   bool isExpTgt() const { return isImmTy(ImmTyExpTgt); }
278   bool isExpVM() const { return isImmTy(ImmTyExpVM); }
279   bool isExpCompr() const { return isImmTy(ImmTyExpCompr); }
280   bool isOffen() const { return isImmTy(ImmTyOffen); }
281   bool isIdxen() const { return isImmTy(ImmTyIdxen); }
282   bool isAddr64() const { return isImmTy(ImmTyAddr64); }
283   bool isOffset() const { return isImmTy(ImmTyOffset) && isUInt<16>(getImm()); }
284   bool isOffset0() const { return isImmTy(ImmTyOffset0) && isUInt<16>(getImm()); }
285   bool isOffset1() const { return isImmTy(ImmTyOffset1) && isUInt<8>(getImm()); }
286   bool isGDS() const { return isImmTy(ImmTyGDS); }
287   bool isGLC() const { return isImmTy(ImmTyGLC); }
288   bool isSLC() const { return isImmTy(ImmTySLC); }
289   bool isTFE() const { return isImmTy(ImmTyTFE); }
290   bool isBankMask() const { return isImmTy(ImmTyDppBankMask); }
291   bool isRowMask() const { return isImmTy(ImmTyDppRowMask); }
292   bool isBoundCtrl() const { return isImmTy(ImmTyDppBoundCtrl); }
293   bool isSDWADstSel() const { return isImmTy(ImmTySdwaDstSel); }
294   bool isSDWASrc0Sel() const { return isImmTy(ImmTySdwaSrc0Sel); }
295   bool isSDWASrc1Sel() const { return isImmTy(ImmTySdwaSrc1Sel); }
296   bool isSDWADstUnused() const { return isImmTy(ImmTySdwaDstUnused); }
297   bool isInterpSlot() const { return isImmTy(ImmTyInterpSlot); }
298   bool isInterpAttr() const { return isImmTy(ImmTyInterpAttr); }
299   bool isAttrChan() const { return isImmTy(ImmTyAttrChan); }
300 
301   bool isMod() const {
302     return isClampSI() || isOModSI();
303   }
304 
305   bool isRegOrImm() const {
306     return isReg() || isImm();
307   }
308 
309   bool isRegClass(unsigned RCID) const;
310 
311   bool isRegOrInlineNoMods(unsigned RCID, MVT type) const {
312     return (isRegClass(RCID) || isInlinableImm(type)) && !hasModifiers();
313   }
314 
315   bool isSCSrcB16() const {
316     return isRegOrInlineNoMods(AMDGPU::SReg_32RegClassID, MVT::i16);
317   }
318 
319   bool isSCSrcB32() const {
320     return isRegOrInlineNoMods(AMDGPU::SReg_32RegClassID, MVT::i32);
321   }
322 
323   bool isSCSrcB64() const {
324     return isRegOrInlineNoMods(AMDGPU::SReg_64RegClassID, MVT::i64);
325   }
326 
327   bool isSCSrcF16() const {
328     return isRegOrInlineNoMods(AMDGPU::SReg_32RegClassID, MVT::f16);
329   }
330 
331   bool isSCSrcF32() const {
332     return isRegOrInlineNoMods(AMDGPU::SReg_32RegClassID, MVT::f32);
333   }
334 
335   bool isSCSrcF64() const {
336     return isRegOrInlineNoMods(AMDGPU::SReg_64RegClassID, MVT::f64);
337   }
338 
339   bool isSSrcB32() const {
340     return isSCSrcB32() || isLiteralImm(MVT::i32) || isExpr();
341   }
342 
343   bool isSSrcB16() const {
344     return isSCSrcB16() || isLiteralImm(MVT::i16);
345   }
346 
347   bool isSSrcB64() const {
348     // TODO: Find out how SALU supports extension of 32-bit literals to 64 bits.
349     // See isVSrc64().
350     return isSCSrcB64() || isLiteralImm(MVT::i64);
351   }
352 
353   bool isSSrcF32() const {
354     return isSCSrcB32() || isLiteralImm(MVT::f32) || isExpr();
355   }
356 
357   bool isSSrcF64() const {
358     return isSCSrcB64() || isLiteralImm(MVT::f64);
359   }
360 
361   bool isSSrcF16() const {
362     return isSCSrcB16() || isLiteralImm(MVT::f16);
363   }
364 
365   bool isVCSrcB32() const {
366     return isRegOrInlineNoMods(AMDGPU::VS_32RegClassID, MVT::i32);
367   }
368 
369   bool isVCSrcB64() const {
370     return isRegOrInlineNoMods(AMDGPU::VS_64RegClassID, MVT::i64);
371   }
372 
373   bool isVCSrcB16() const {
374     return isRegOrInlineNoMods(AMDGPU::VS_32RegClassID, MVT::i16);
375   }
376 
377   bool isVCSrcF32() const {
378     return isRegOrInlineNoMods(AMDGPU::VS_32RegClassID, MVT::f32);
379   }
380 
381   bool isVCSrcF64() const {
382     return isRegOrInlineNoMods(AMDGPU::VS_64RegClassID, MVT::f64);
383   }
384 
385   bool isVCSrcF16() const {
386     return isRegOrInlineNoMods(AMDGPU::VS_32RegClassID, MVT::f16);
387   }
388 
389   bool isVSrcB32() const {
390     return isVCSrcF32() || isLiteralImm(MVT::i32);
391   }
392 
393   bool isVSrcB64() const {
394     return isVCSrcF64() || isLiteralImm(MVT::i64);
395   }
396 
397   bool isVSrcB16() const {
398     return isVCSrcF16() || isLiteralImm(MVT::i16);
399   }
400 
401   bool isVSrcF32() const {
402     return isVCSrcF32() || isLiteralImm(MVT::f32);
403   }
404 
405   bool isVSrcF64() const {
406     return isVCSrcF64() || isLiteralImm(MVT::f64);
407   }
408 
409   bool isVSrcF16() const {
410     return isVCSrcF16() || isLiteralImm(MVT::f16);
411   }
412 
413   bool isKImmFP32() const {
414     return isLiteralImm(MVT::f32);
415   }
416 
417   bool isKImmFP16() const {
418     return isLiteralImm(MVT::f16);
419   }
420 
421   bool isMem() const override {
422     return false;
423   }
424 
425   bool isExpr() const {
426     return Kind == Expression;
427   }
428 
429   bool isSoppBrTarget() const {
430     return isExpr() || isImm();
431   }
432 
433   bool isSWaitCnt() const;
434   bool isHwreg() const;
435   bool isSendMsg() const;
436   bool isSMRDOffset8() const;
437   bool isSMRDOffset20() const;
438   bool isSMRDLiteralOffset() const;
439   bool isDPPCtrl() const;
440   bool isGPRIdxMode() const;
441 
442   StringRef getExpressionAsToken() const {
443     assert(isExpr());
444     const MCSymbolRefExpr *S = cast<MCSymbolRefExpr>(Expr);
445     return S->getSymbol().getName();
446   }
447 
448   StringRef getToken() const {
449     assert(isToken());
450 
451     if (Kind == Expression)
452       return getExpressionAsToken();
453 
454     return StringRef(Tok.Data, Tok.Length);
455   }
456 
457   int64_t getImm() const {
458     assert(isImm());
459     return Imm.Val;
460   }
461 
462   enum ImmTy getImmTy() const {
463     assert(isImm());
464     return Imm.Type;
465   }
466 
467   unsigned getReg() const override {
468     return Reg.RegNo;
469   }
470 
471   SMLoc getStartLoc() const override {
472     return StartLoc;
473   }
474 
475   SMLoc getEndLoc() const override {
476     return EndLoc;
477   }
478 
479   Modifiers getModifiers() const {
480     assert(isRegKind() || isImmTy(ImmTyNone));
481     return isRegKind() ? Reg.Mods : Imm.Mods;
482   }
483 
484   void setModifiers(Modifiers Mods) {
485     assert(isRegKind() || isImmTy(ImmTyNone));
486     if (isRegKind())
487       Reg.Mods = Mods;
488     else
489       Imm.Mods = Mods;
490   }
491 
492   bool hasModifiers() const {
493     return getModifiers().hasModifiers();
494   }
495 
496   bool hasFPModifiers() const {
497     return getModifiers().hasFPModifiers();
498   }
499 
500   bool hasIntModifiers() const {
501     return getModifiers().hasIntModifiers();
502   }
503 
504   void addImmOperands(MCInst &Inst, unsigned N, bool ApplyModifiers = true) const;
505 
506   void addLiteralImmOperand(MCInst &Inst, int64_t Val) const;
507 
508   template <unsigned Bitwidth>
509   void addKImmFPOperands(MCInst &Inst, unsigned N) const;
510 
511   void addKImmFP16Operands(MCInst &Inst, unsigned N) const {
512     addKImmFPOperands<16>(Inst, N);
513   }
514 
515   void addKImmFP32Operands(MCInst &Inst, unsigned N) const {
516     addKImmFPOperands<32>(Inst, N);
517   }
518 
519   void addRegOperands(MCInst &Inst, unsigned N) const;
520 
521   void addRegOrImmOperands(MCInst &Inst, unsigned N) const {
522     if (isRegKind())
523       addRegOperands(Inst, N);
524     else if (isExpr())
525       Inst.addOperand(MCOperand::createExpr(Expr));
526     else
527       addImmOperands(Inst, N);
528   }
529 
530   void addRegOrImmWithInputModsOperands(MCInst &Inst, unsigned N) const {
531     Modifiers Mods = getModifiers();
532     Inst.addOperand(MCOperand::createImm(Mods.getModifiersOperand()));
533     if (isRegKind()) {
534       addRegOperands(Inst, N);
535     } else {
536       addImmOperands(Inst, N, false);
537     }
538   }
539 
540   void addRegOrImmWithFPInputModsOperands(MCInst &Inst, unsigned N) const {
541     assert(!hasIntModifiers());
542     addRegOrImmWithInputModsOperands(Inst, N);
543   }
544 
545   void addRegOrImmWithIntInputModsOperands(MCInst &Inst, unsigned N) const {
546     assert(!hasFPModifiers());
547     addRegOrImmWithInputModsOperands(Inst, N);
548   }
549 
550   void addRegWithInputModsOperands(MCInst &Inst, unsigned N) const {
551     Modifiers Mods = getModifiers();
552     Inst.addOperand(MCOperand::createImm(Mods.getModifiersOperand()));
553     assert(isRegKind());
554     addRegOperands(Inst, N);
555   }
556 
557   void addRegWithFPInputModsOperands(MCInst &Inst, unsigned N) const {
558     assert(!hasIntModifiers());
559     addRegWithInputModsOperands(Inst, N);
560   }
561 
562   void addRegWithIntInputModsOperands(MCInst &Inst, unsigned N) const {
563     assert(!hasFPModifiers());
564     addRegWithInputModsOperands(Inst, N);
565   }
566 
567   void addSoppBrTargetOperands(MCInst &Inst, unsigned N) const {
568     if (isImm())
569       addImmOperands(Inst, N);
570     else {
571       assert(isExpr());
572       Inst.addOperand(MCOperand::createExpr(Expr));
573     }
574   }
575 
576   static void printImmTy(raw_ostream& OS, ImmTy Type) {
577     switch (Type) {
578     case ImmTyNone: OS << "None"; break;
579     case ImmTyGDS: OS << "GDS"; break;
580     case ImmTyOffen: OS << "Offen"; break;
581     case ImmTyIdxen: OS << "Idxen"; break;
582     case ImmTyAddr64: OS << "Addr64"; break;
583     case ImmTyOffset: OS << "Offset"; break;
584     case ImmTyOffset0: OS << "Offset0"; break;
585     case ImmTyOffset1: OS << "Offset1"; break;
586     case ImmTyGLC: OS << "GLC"; break;
587     case ImmTySLC: OS << "SLC"; break;
588     case ImmTyTFE: OS << "TFE"; break;
589     case ImmTyClampSI: OS << "ClampSI"; break;
590     case ImmTyOModSI: OS << "OModSI"; break;
591     case ImmTyDppCtrl: OS << "DppCtrl"; break;
592     case ImmTyDppRowMask: OS << "DppRowMask"; break;
593     case ImmTyDppBankMask: OS << "DppBankMask"; break;
594     case ImmTyDppBoundCtrl: OS << "DppBoundCtrl"; break;
595     case ImmTySdwaDstSel: OS << "SdwaDstSel"; break;
596     case ImmTySdwaSrc0Sel: OS << "SdwaSrc0Sel"; break;
597     case ImmTySdwaSrc1Sel: OS << "SdwaSrc1Sel"; break;
598     case ImmTySdwaDstUnused: OS << "SdwaDstUnused"; break;
599     case ImmTyDMask: OS << "DMask"; break;
600     case ImmTyUNorm: OS << "UNorm"; break;
601     case ImmTyDA: OS << "DA"; break;
602     case ImmTyR128: OS << "R128"; break;
603     case ImmTyLWE: OS << "LWE"; break;
604     case ImmTyOff: OS << "Off"; break;
605     case ImmTyExpTgt: OS << "ExpTgt"; break;
606     case ImmTyExpCompr: OS << "ExpCompr"; break;
607     case ImmTyExpVM: OS << "ExpVM"; break;
608     case ImmTyHwreg: OS << "Hwreg"; break;
609     case ImmTySendMsg: OS << "SendMsg"; break;
610     case ImmTyInterpSlot: OS << "InterpSlot"; break;
611     case ImmTyInterpAttr: OS << "InterpAttr"; break;
612     case ImmTyAttrChan: OS << "AttrChan"; break;
613     }
614   }
615 
616   void print(raw_ostream &OS) const override {
617     switch (Kind) {
618     case Register:
619       OS << "<register " << getReg() << " mods: " << Reg.Mods << '>';
620       break;
621     case Immediate:
622       OS << '<' << getImm();
623       if (getImmTy() != ImmTyNone) {
624         OS << " type: "; printImmTy(OS, getImmTy());
625       }
626       OS << " mods: " << Imm.Mods << '>';
627       break;
628     case Token:
629       OS << '\'' << getToken() << '\'';
630       break;
631     case Expression:
632       OS << "<expr " << *Expr << '>';
633       break;
634     }
635   }
636 
637   static AMDGPUOperand::Ptr CreateImm(const AMDGPUAsmParser *AsmParser,
638                                       int64_t Val, SMLoc Loc,
639                                       enum ImmTy Type = ImmTyNone,
640                                       bool IsFPImm = false) {
641     auto Op = llvm::make_unique<AMDGPUOperand>(Immediate, AsmParser);
642     Op->Imm.Val = Val;
643     Op->Imm.IsFPImm = IsFPImm;
644     Op->Imm.Type = Type;
645     Op->Imm.Mods = Modifiers();
646     Op->StartLoc = Loc;
647     Op->EndLoc = Loc;
648     return Op;
649   }
650 
651   static AMDGPUOperand::Ptr CreateToken(const AMDGPUAsmParser *AsmParser,
652                                         StringRef Str, SMLoc Loc,
653                                         bool HasExplicitEncodingSize = true) {
654     auto Res = llvm::make_unique<AMDGPUOperand>(Token, AsmParser);
655     Res->Tok.Data = Str.data();
656     Res->Tok.Length = Str.size();
657     Res->StartLoc = Loc;
658     Res->EndLoc = Loc;
659     return Res;
660   }
661 
662   static AMDGPUOperand::Ptr CreateReg(const AMDGPUAsmParser *AsmParser,
663                                       unsigned RegNo, SMLoc S,
664                                       SMLoc E,
665                                       bool ForceVOP3) {
666     auto Op = llvm::make_unique<AMDGPUOperand>(Register, AsmParser);
667     Op->Reg.RegNo = RegNo;
668     Op->Reg.Mods = Modifiers();
669     Op->Reg.IsForcedVOP3 = ForceVOP3;
670     Op->StartLoc = S;
671     Op->EndLoc = E;
672     return Op;
673   }
674 
675   static AMDGPUOperand::Ptr CreateExpr(const AMDGPUAsmParser *AsmParser,
676                                        const class MCExpr *Expr, SMLoc S) {
677     auto Op = llvm::make_unique<AMDGPUOperand>(Expression, AsmParser);
678     Op->Expr = Expr;
679     Op->StartLoc = S;
680     Op->EndLoc = S;
681     return Op;
682   }
683 };
684 
685 raw_ostream &operator <<(raw_ostream &OS, AMDGPUOperand::Modifiers Mods) {
686   OS << "abs:" << Mods.Abs << " neg: " << Mods.Neg << " sext:" << Mods.Sext;
687   return OS;
688 }
689 
690 //===----------------------------------------------------------------------===//
691 // AsmParser
692 //===----------------------------------------------------------------------===//
693 
694 // Holds info related to the current kernel, e.g. count of SGPRs used.
695 // Kernel scope begins at .amdgpu_hsa_kernel directive, ends at next
696 // .amdgpu_hsa_kernel or at EOF.
697 class KernelScopeInfo {
698   int SgprIndexUnusedMin;
699   int VgprIndexUnusedMin;
700   MCContext *Ctx;
701 
702   void usesSgprAt(int i) {
703     if (i >= SgprIndexUnusedMin) {
704       SgprIndexUnusedMin = ++i;
705       if (Ctx) {
706         MCSymbol * const Sym = Ctx->getOrCreateSymbol(Twine(".kernel.sgpr_count"));
707         Sym->setVariableValue(MCConstantExpr::create(SgprIndexUnusedMin, *Ctx));
708       }
709     }
710   }
711   void usesVgprAt(int i) {
712     if (i >= VgprIndexUnusedMin) {
713       VgprIndexUnusedMin = ++i;
714       if (Ctx) {
715         MCSymbol * const Sym = Ctx->getOrCreateSymbol(Twine(".kernel.vgpr_count"));
716         Sym->setVariableValue(MCConstantExpr::create(VgprIndexUnusedMin, *Ctx));
717       }
718     }
719   }
720 public:
721   KernelScopeInfo() : SgprIndexUnusedMin(-1), VgprIndexUnusedMin(-1), Ctx(nullptr)
722   {}
723   void initialize(MCContext &Context) {
724     Ctx = &Context;
725     usesSgprAt(SgprIndexUnusedMin = -1);
726     usesVgprAt(VgprIndexUnusedMin = -1);
727   }
728   void usesRegister(RegisterKind RegKind, unsigned DwordRegIndex, unsigned RegWidth) {
729     switch (RegKind) {
730       case IS_SGPR: usesSgprAt(DwordRegIndex + RegWidth - 1); break;
731       case IS_VGPR: usesVgprAt(DwordRegIndex + RegWidth - 1); break;
732       default: break;
733     }
734   }
735 };
736 
737 class AMDGPUAsmParser : public MCTargetAsmParser {
738   const MCInstrInfo &MII;
739   MCAsmParser &Parser;
740 
741   unsigned ForcedEncodingSize;
742   bool ForcedDPP;
743   bool ForcedSDWA;
744   KernelScopeInfo KernelScope;
745 
746   /// @name Auto-generated Match Functions
747   /// {
748 
749 #define GET_ASSEMBLER_HEADER
750 #include "AMDGPUGenAsmMatcher.inc"
751 
752   /// }
753 
754 private:
755   bool ParseAsAbsoluteExpression(uint32_t &Ret);
756   bool ParseDirectiveMajorMinor(uint32_t &Major, uint32_t &Minor);
757   bool ParseDirectiveHSACodeObjectVersion();
758   bool ParseDirectiveHSACodeObjectISA();
759   bool ParseDirectiveRuntimeMetadata();
760   bool ParseAMDKernelCodeTValue(StringRef ID, amd_kernel_code_t &Header);
761   bool ParseDirectiveAMDKernelCodeT();
762   bool ParseSectionDirectiveHSAText();
763   bool subtargetHasRegister(const MCRegisterInfo &MRI, unsigned RegNo) const;
764   bool ParseDirectiveAMDGPUHsaKernel();
765   bool ParseDirectiveAMDGPUHsaModuleGlobal();
766   bool ParseDirectiveAMDGPUHsaProgramGlobal();
767   bool ParseSectionDirectiveHSADataGlobalAgent();
768   bool ParseSectionDirectiveHSADataGlobalProgram();
769   bool ParseSectionDirectiveHSARodataReadonlyAgent();
770   bool AddNextRegisterToList(unsigned& Reg, unsigned& RegWidth, RegisterKind RegKind, unsigned Reg1, unsigned RegNum);
771   bool ParseAMDGPURegister(RegisterKind& RegKind, unsigned& Reg, unsigned& RegNum, unsigned& RegWidth, unsigned *DwordRegIndex);
772   void cvtMubufImpl(MCInst &Inst, const OperandVector &Operands, bool IsAtomic, bool IsAtomicReturn);
773 
774 public:
775   enum AMDGPUMatchResultTy {
776     Match_PreferE32 = FIRST_TARGET_MATCH_RESULT_TY
777   };
778 
779   AMDGPUAsmParser(const MCSubtargetInfo &STI, MCAsmParser &_Parser,
780                const MCInstrInfo &MII,
781                const MCTargetOptions &Options)
782       : MCTargetAsmParser(Options, STI), MII(MII), Parser(_Parser),
783         ForcedEncodingSize(0),
784         ForcedDPP(false),
785         ForcedSDWA(false) {
786     MCAsmParserExtension::Initialize(Parser);
787 
788     if (getSTI().getFeatureBits().none()) {
789       // Set default features.
790       copySTI().ToggleFeature("SOUTHERN_ISLANDS");
791     }
792 
793     setAvailableFeatures(ComputeAvailableFeatures(getSTI().getFeatureBits()));
794 
795     {
796       // TODO: make those pre-defined variables read-only.
797       // Currently there is none suitable machinery in the core llvm-mc for this.
798       // MCSymbol::isRedefinable is intended for another purpose, and
799       // AsmParser::parseDirectiveSet() cannot be specialized for specific target.
800       AMDGPU::IsaVersion Isa = AMDGPU::getIsaVersion(getSTI().getFeatureBits());
801       MCContext &Ctx = getContext();
802       MCSymbol *Sym = Ctx.getOrCreateSymbol(Twine(".option.machine_version_major"));
803       Sym->setVariableValue(MCConstantExpr::create(Isa.Major, Ctx));
804       Sym = Ctx.getOrCreateSymbol(Twine(".option.machine_version_minor"));
805       Sym->setVariableValue(MCConstantExpr::create(Isa.Minor, Ctx));
806       Sym = Ctx.getOrCreateSymbol(Twine(".option.machine_version_stepping"));
807       Sym->setVariableValue(MCConstantExpr::create(Isa.Stepping, Ctx));
808     }
809     KernelScope.initialize(getContext());
810   }
811 
812   bool isSI() const {
813     return AMDGPU::isSI(getSTI());
814   }
815 
816   bool isCI() const {
817     return AMDGPU::isCI(getSTI());
818   }
819 
820   bool isVI() const {
821     return AMDGPU::isVI(getSTI());
822   }
823 
824   bool hasInv2PiInlineImm() const {
825     return getSTI().getFeatureBits()[AMDGPU::FeatureInv2PiInlineImm];
826   }
827 
828   bool hasSGPR102_SGPR103() const {
829     return !isVI();
830   }
831 
832   AMDGPUTargetStreamer &getTargetStreamer() {
833     MCTargetStreamer &TS = *getParser().getStreamer().getTargetStreamer();
834     return static_cast<AMDGPUTargetStreamer &>(TS);
835   }
836 
837   const MCRegisterInfo *getMRI() const {
838     // We need this const_cast because for some reason getContext() is not const
839     // in MCAsmParser.
840     return const_cast<AMDGPUAsmParser*>(this)->getContext().getRegisterInfo();
841   }
842 
843   const MCInstrInfo *getMII() const {
844     return &MII;
845   }
846 
847   void setForcedEncodingSize(unsigned Size) { ForcedEncodingSize = Size; }
848   void setForcedDPP(bool ForceDPP_) { ForcedDPP = ForceDPP_; }
849   void setForcedSDWA(bool ForceSDWA_) { ForcedSDWA = ForceSDWA_; }
850 
851   unsigned getForcedEncodingSize() const { return ForcedEncodingSize; }
852   bool isForcedVOP3() const { return ForcedEncodingSize == 64; }
853   bool isForcedDPP() const { return ForcedDPP; }
854   bool isForcedSDWA() const { return ForcedSDWA; }
855   ArrayRef<unsigned> getMatchedVariants() const;
856 
857   std::unique_ptr<AMDGPUOperand> parseRegister();
858   bool ParseRegister(unsigned &RegNo, SMLoc &StartLoc, SMLoc &EndLoc) override;
859   unsigned checkTargetMatchPredicate(MCInst &Inst) override;
860   unsigned validateTargetOperandClass(MCParsedAsmOperand &Op,
861                                       unsigned Kind) override;
862   bool MatchAndEmitInstruction(SMLoc IDLoc, unsigned &Opcode,
863                                OperandVector &Operands, MCStreamer &Out,
864                                uint64_t &ErrorInfo,
865                                bool MatchingInlineAsm) override;
866   bool ParseDirective(AsmToken DirectiveID) override;
867   OperandMatchResultTy parseOperand(OperandVector &Operands, StringRef Mnemonic);
868   StringRef parseMnemonicSuffix(StringRef Name);
869   bool ParseInstruction(ParseInstructionInfo &Info, StringRef Name,
870                         SMLoc NameLoc, OperandVector &Operands) override;
871   //bool ProcessInstruction(MCInst &Inst);
872 
873   OperandMatchResultTy parseIntWithPrefix(const char *Prefix, int64_t &Int);
874   OperandMatchResultTy
875   parseIntWithPrefix(const char *Prefix, OperandVector &Operands,
876                      enum AMDGPUOperand::ImmTy ImmTy = AMDGPUOperand::ImmTyNone,
877                      bool (*ConvertResult)(int64_t &) = nullptr);
878   OperandMatchResultTy
879   parseNamedBit(const char *Name, OperandVector &Operands,
880                 enum AMDGPUOperand::ImmTy ImmTy = AMDGPUOperand::ImmTyNone);
881   OperandMatchResultTy parseStringWithPrefix(StringRef Prefix,
882                                              StringRef &Value);
883 
884   OperandMatchResultTy parseImm(OperandVector &Operands);
885   OperandMatchResultTy parseReg(OperandVector &Operands);
886   OperandMatchResultTy parseRegOrImm(OperandVector &Operands);
887   OperandMatchResultTy parseRegOrImmWithFPInputMods(OperandVector &Operands, bool AllowImm = true);
888   OperandMatchResultTy parseRegOrImmWithIntInputMods(OperandVector &Operands, bool AllowImm = true);
889   OperandMatchResultTy parseRegWithFPInputMods(OperandVector &Operands);
890   OperandMatchResultTy parseRegWithIntInputMods(OperandVector &Operands);
891   OperandMatchResultTy parseVReg32OrOff(OperandVector &Operands);
892 
893   void cvtDSOffset01(MCInst &Inst, const OperandVector &Operands);
894   void cvtDS(MCInst &Inst, const OperandVector &Operands);
895   void cvtExp(MCInst &Inst, const OperandVector &Operands);
896 
897   bool parseCnt(int64_t &IntVal);
898   OperandMatchResultTy parseSWaitCntOps(OperandVector &Operands);
899   OperandMatchResultTy parseHwreg(OperandVector &Operands);
900 
901 private:
902   struct OperandInfoTy {
903     int64_t Id;
904     bool IsSymbolic;
905     OperandInfoTy(int64_t Id_) : Id(Id_), IsSymbolic(false) { }
906   };
907 
908   bool parseSendMsgConstruct(OperandInfoTy &Msg, OperandInfoTy &Operation, int64_t &StreamId);
909   bool parseHwregConstruct(OperandInfoTy &HwReg, int64_t &Offset, int64_t &Width);
910 
911   void errorExpTgt();
912   OperandMatchResultTy parseExpTgtImpl(StringRef Str, uint8_t &Val);
913 
914 public:
915   OperandMatchResultTy parseOptionalOperand(OperandVector &Operands);
916 
917   OperandMatchResultTy parseExpTgt(OperandVector &Operands);
918   OperandMatchResultTy parseSendMsgOp(OperandVector &Operands);
919   OperandMatchResultTy parseInterpSlot(OperandVector &Operands);
920   OperandMatchResultTy parseInterpAttr(OperandVector &Operands);
921   OperandMatchResultTy parseSOppBrTarget(OperandVector &Operands);
922 
923   void cvtMubuf(MCInst &Inst, const OperandVector &Operands) { cvtMubufImpl(Inst, Operands, false, false); }
924   void cvtMubufAtomic(MCInst &Inst, const OperandVector &Operands) { cvtMubufImpl(Inst, Operands, true, false); }
925   void cvtMubufAtomicReturn(MCInst &Inst, const OperandVector &Operands) { cvtMubufImpl(Inst, Operands, true, true); }
926   AMDGPUOperand::Ptr defaultGLC() const;
927   AMDGPUOperand::Ptr defaultSLC() const;
928   AMDGPUOperand::Ptr defaultTFE() const;
929 
930   AMDGPUOperand::Ptr defaultDMask() const;
931   AMDGPUOperand::Ptr defaultUNorm() const;
932   AMDGPUOperand::Ptr defaultDA() const;
933   AMDGPUOperand::Ptr defaultR128() const;
934   AMDGPUOperand::Ptr defaultLWE() const;
935   AMDGPUOperand::Ptr defaultSMRDOffset8() const;
936   AMDGPUOperand::Ptr defaultSMRDOffset20() const;
937   AMDGPUOperand::Ptr defaultSMRDLiteralOffset() const;
938 
939   OperandMatchResultTy parseOModOperand(OperandVector &Operands);
940 
941   void cvtId(MCInst &Inst, const OperandVector &Operands);
942   void cvtVOP3_2_mod(MCInst &Inst, const OperandVector &Operands);
943   void cvtVOP3(MCInst &Inst, const OperandVector &Operands);
944 
945   void cvtMIMG(MCInst &Inst, const OperandVector &Operands);
946   void cvtMIMGAtomic(MCInst &Inst, const OperandVector &Operands);
947 
948   OperandMatchResultTy parseDPPCtrl(OperandVector &Operands);
949   AMDGPUOperand::Ptr defaultRowMask() const;
950   AMDGPUOperand::Ptr defaultBankMask() const;
951   AMDGPUOperand::Ptr defaultBoundCtrl() const;
952   void cvtDPP(MCInst &Inst, const OperandVector &Operands);
953 
954   OperandMatchResultTy parseSDWASel(OperandVector &Operands, StringRef Prefix,
955                                     AMDGPUOperand::ImmTy Type);
956   OperandMatchResultTy parseSDWADstUnused(OperandVector &Operands);
957   void cvtSdwaVOP1(MCInst &Inst, const OperandVector &Operands);
958   void cvtSdwaVOP2(MCInst &Inst, const OperandVector &Operands);
959   void cvtSdwaVOPC(MCInst &Inst, const OperandVector &Operands);
960   void cvtSDWA(MCInst &Inst, const OperandVector &Operands,
961                uint64_t BasicInstType);
962 };
963 
964 struct OptionalOperand {
965   const char *Name;
966   AMDGPUOperand::ImmTy Type;
967   bool IsBit;
968   bool (*ConvertResult)(int64_t&);
969 };
970 
971 } // end anonymous namespace
972 
973 // May be called with integer type with equivalent bitwidth.
974 static const fltSemantics *getFltSemantics(unsigned Size) {
975   switch (Size) {
976   case 4:
977     return &APFloat::IEEEsingle();
978   case 8:
979     return &APFloat::IEEEdouble();
980   case 2:
981     return &APFloat::IEEEhalf();
982   default:
983     llvm_unreachable("unsupported fp type");
984   }
985 }
986 
987 static const fltSemantics *getFltSemantics(MVT VT) {
988   return getFltSemantics(VT.getSizeInBits() / 8);
989 }
990 
991 //===----------------------------------------------------------------------===//
992 // Operand
993 //===----------------------------------------------------------------------===//
994 
995 static bool canLosslesslyConvertToFPType(APFloat &FPLiteral, MVT VT) {
996   bool Lost;
997 
998   // Convert literal to single precision
999   APFloat::opStatus Status = FPLiteral.convert(*getFltSemantics(VT),
1000                                                APFloat::rmNearestTiesToEven,
1001                                                &Lost);
1002   // We allow precision lost but not overflow or underflow
1003   if (Status != APFloat::opOK &&
1004       Lost &&
1005       ((Status & APFloat::opOverflow)  != 0 ||
1006        (Status & APFloat::opUnderflow) != 0)) {
1007     return false;
1008   }
1009 
1010   return true;
1011 }
1012 
1013 bool AMDGPUOperand::isInlinableImm(MVT type) const {
1014   if (!isImmTy(ImmTyNone)) {
1015     // Only plain immediates are inlinable (e.g. "clamp" attribute is not)
1016     return false;
1017   }
1018   // TODO: We should avoid using host float here. It would be better to
1019   // check the float bit values which is what a few other places do.
1020   // We've had bot failures before due to weird NaN support on mips hosts.
1021 
1022   APInt Literal(64, Imm.Val);
1023 
1024   if (Imm.IsFPImm) { // We got fp literal token
1025     if (type == MVT::f64 || type == MVT::i64) { // Expected 64-bit operand
1026       return AMDGPU::isInlinableLiteral64(Imm.Val,
1027                                           AsmParser->hasInv2PiInlineImm());
1028     }
1029 
1030     APFloat FPLiteral(APFloat::IEEEdouble(), APInt(64, Imm.Val));
1031     if (!canLosslesslyConvertToFPType(FPLiteral, type))
1032       return false;
1033 
1034     if (type.getScalarSizeInBits() == 16) {
1035       return AMDGPU::isInlinableLiteral16(
1036         static_cast<int32_t>(FPLiteral.bitcastToAPInt().getZExtValue()),
1037         AsmParser->hasInv2PiInlineImm());
1038     }
1039 
1040     // Check if single precision literal is inlinable
1041     return AMDGPU::isInlinableLiteral32(
1042       static_cast<int32_t>(FPLiteral.bitcastToAPInt().getZExtValue()),
1043       AsmParser->hasInv2PiInlineImm());
1044   }
1045 
1046 
1047   // We got int literal token.
1048   if (type == MVT::f64 || type == MVT::i64) { // Expected 64-bit operand
1049     return AMDGPU::isInlinableLiteral64(Imm.Val,
1050                                         AsmParser->hasInv2PiInlineImm());
1051   }
1052 
1053   if (type.getScalarSizeInBits() == 16) {
1054     return AMDGPU::isInlinableLiteral16(
1055       static_cast<int16_t>(Literal.getLoBits(16).getSExtValue()),
1056       AsmParser->hasInv2PiInlineImm());
1057   }
1058 
1059   return AMDGPU::isInlinableLiteral32(
1060     static_cast<int32_t>(Literal.getLoBits(32).getZExtValue()),
1061     AsmParser->hasInv2PiInlineImm());
1062 }
1063 
1064 bool AMDGPUOperand::isLiteralImm(MVT type) const {
1065   // Check that this imediate can be added as literal
1066   if (!isImmTy(ImmTyNone)) {
1067     return false;
1068   }
1069 
1070   if (!Imm.IsFPImm) {
1071     // We got int literal token.
1072 
1073     unsigned Size = type.getSizeInBits();
1074     if (Size == 64)
1075       Size = 32;
1076 
1077     // FIXME: 64-bit operands can zero extend, sign extend, or pad zeroes for FP
1078     // types.
1079     return isUIntN(Size, Imm.Val) || isIntN(Size, Imm.Val);
1080   }
1081 
1082   // We got fp literal token
1083   if (type == MVT::f64) { // Expected 64-bit fp operand
1084     // We would set low 64-bits of literal to zeroes but we accept this literals
1085     return true;
1086   }
1087 
1088   if (type == MVT::i64) { // Expected 64-bit int operand
1089     // We don't allow fp literals in 64-bit integer instructions. It is
1090     // unclear how we should encode them.
1091     return false;
1092   }
1093 
1094   APFloat FPLiteral(APFloat::IEEEdouble(), APInt(64, Imm.Val));
1095   return canLosslesslyConvertToFPType(FPLiteral, type);
1096 }
1097 
1098 bool AMDGPUOperand::isRegClass(unsigned RCID) const {
1099   return isRegKind() && AsmParser->getMRI()->getRegClass(RCID).contains(getReg());
1100 }
1101 
1102 void AMDGPUOperand::addImmOperands(MCInst &Inst, unsigned N, bool ApplyModifiers) const {
1103   int64_t Val = Imm.Val;
1104   if (isImmTy(ImmTyNone) && ApplyModifiers && Imm.Mods.hasFPModifiers() && Imm.Mods.Neg) {
1105     // Apply modifiers to immediate value. Only negate can get here
1106     if (Imm.IsFPImm) {
1107       APFloat F(BitsToDouble(Val));
1108       F.changeSign();
1109       Val = F.bitcastToAPInt().getZExtValue();
1110     } else {
1111       Val = -Val;
1112     }
1113   }
1114 
1115   if (AMDGPU::isSISrcOperand(AsmParser->getMII()->get(Inst.getOpcode()),
1116                              Inst.getNumOperands())) {
1117     addLiteralImmOperand(Inst, Val);
1118   } else {
1119     Inst.addOperand(MCOperand::createImm(Val));
1120   }
1121 }
1122 
1123 void AMDGPUOperand::addLiteralImmOperand(MCInst &Inst, int64_t Val) const {
1124   const auto& InstDesc = AsmParser->getMII()->get(Inst.getOpcode());
1125   auto OpNum = Inst.getNumOperands();
1126   // Check that this operand accepts literals
1127   assert(AMDGPU::isSISrcOperand(InstDesc, OpNum));
1128 
1129   auto OpSize = AMDGPU::getOperandSize(InstDesc, OpNum); // expected operand size
1130 
1131   if (Imm.IsFPImm) { // We got fp literal token
1132     APInt Literal(64, Val);
1133 
1134     switch (OpSize) {
1135     case 8: {
1136       if (AMDGPU::isInlinableLiteral64(Literal.getZExtValue(),
1137                                        AsmParser->hasInv2PiInlineImm())) {
1138         Inst.addOperand(MCOperand::createImm(Literal.getZExtValue()));
1139         return;
1140       }
1141 
1142       // Non-inlineable
1143       if (AMDGPU::isSISrcFPOperand(InstDesc, OpNum)) { // Expected 64-bit fp operand
1144         // For fp operands we check if low 32 bits are zeros
1145         if (Literal.getLoBits(32) != 0) {
1146           const_cast<AMDGPUAsmParser *>(AsmParser)->Warning(Inst.getLoc(),
1147           "Can't encode literal as exact 64-bit floating-point operand. "
1148           "Low 32-bits will be set to zero");
1149         }
1150 
1151         Inst.addOperand(MCOperand::createImm(Literal.lshr(32).getZExtValue()));
1152         return;
1153       }
1154 
1155       // We don't allow fp literals in 64-bit integer instructions. It is
1156       // unclear how we should encode them. This case should be checked earlier
1157       // in predicate methods (isLiteralImm())
1158       llvm_unreachable("fp literal in 64-bit integer instruction.");
1159     }
1160     case 4:
1161     case 2: {
1162       bool lost;
1163       APFloat FPLiteral(APFloat::IEEEdouble(), Literal);
1164       // Convert literal to single precision
1165       FPLiteral.convert(*getFltSemantics(OpSize),
1166                         APFloat::rmNearestTiesToEven, &lost);
1167       // We allow precision lost but not overflow or underflow. This should be
1168       // checked earlier in isLiteralImm()
1169       Inst.addOperand(MCOperand::createImm(FPLiteral.bitcastToAPInt().getZExtValue()));
1170       return;
1171     }
1172     default:
1173       llvm_unreachable("invalid operand size");
1174     }
1175 
1176     return;
1177   }
1178 
1179    // We got int literal token.
1180   // Only sign extend inline immediates.
1181   // FIXME: No errors on truncation
1182   switch (OpSize) {
1183   case 4: {
1184     if (isInt<32>(Val) &&
1185         AMDGPU::isInlinableLiteral32(static_cast<int32_t>(Val),
1186                                      AsmParser->hasInv2PiInlineImm())) {
1187       Inst.addOperand(MCOperand::createImm(Val));
1188       return;
1189     }
1190 
1191     Inst.addOperand(MCOperand::createImm(Val & 0xffffffff));
1192     return;
1193   }
1194   case 8: {
1195     if (AMDGPU::isInlinableLiteral64(Val,
1196                                      AsmParser->hasInv2PiInlineImm())) {
1197       Inst.addOperand(MCOperand::createImm(Val));
1198       return;
1199     }
1200 
1201     Inst.addOperand(MCOperand::createImm(Lo_32(Val)));
1202     return;
1203   }
1204   case 2: {
1205     if (isInt<16>(Val) &&
1206         AMDGPU::isInlinableLiteral16(static_cast<int16_t>(Val),
1207                                      AsmParser->hasInv2PiInlineImm())) {
1208       Inst.addOperand(MCOperand::createImm(Val));
1209       return;
1210     }
1211 
1212     Inst.addOperand(MCOperand::createImm(Val & 0xffff));
1213     return;
1214   }
1215   default:
1216     llvm_unreachable("invalid operand size");
1217   }
1218 }
1219 
1220 template <unsigned Bitwidth>
1221 void AMDGPUOperand::addKImmFPOperands(MCInst &Inst, unsigned N) const {
1222   APInt Literal(64, Imm.Val);
1223 
1224   if (!Imm.IsFPImm) {
1225     // We got int literal token.
1226     Inst.addOperand(MCOperand::createImm(Literal.getLoBits(Bitwidth).getZExtValue()));
1227     return;
1228   }
1229 
1230   bool Lost;
1231   APFloat FPLiteral(APFloat::IEEEdouble(), Literal);
1232   FPLiteral.convert(*getFltSemantics(Bitwidth / 8),
1233                     APFloat::rmNearestTiesToEven, &Lost);
1234   Inst.addOperand(MCOperand::createImm(FPLiteral.bitcastToAPInt().getZExtValue()));
1235 }
1236 
1237 void AMDGPUOperand::addRegOperands(MCInst &Inst, unsigned N) const {
1238   Inst.addOperand(MCOperand::createReg(AMDGPU::getMCReg(getReg(), AsmParser->getSTI())));
1239 }
1240 
1241 //===----------------------------------------------------------------------===//
1242 // AsmParser
1243 //===----------------------------------------------------------------------===//
1244 
1245 static int getRegClass(RegisterKind Is, unsigned RegWidth) {
1246   if (Is == IS_VGPR) {
1247     switch (RegWidth) {
1248       default: return -1;
1249       case 1: return AMDGPU::VGPR_32RegClassID;
1250       case 2: return AMDGPU::VReg_64RegClassID;
1251       case 3: return AMDGPU::VReg_96RegClassID;
1252       case 4: return AMDGPU::VReg_128RegClassID;
1253       case 8: return AMDGPU::VReg_256RegClassID;
1254       case 16: return AMDGPU::VReg_512RegClassID;
1255     }
1256   } else if (Is == IS_TTMP) {
1257     switch (RegWidth) {
1258       default: return -1;
1259       case 1: return AMDGPU::TTMP_32RegClassID;
1260       case 2: return AMDGPU::TTMP_64RegClassID;
1261       case 4: return AMDGPU::TTMP_128RegClassID;
1262     }
1263   } else if (Is == IS_SGPR) {
1264     switch (RegWidth) {
1265       default: return -1;
1266       case 1: return AMDGPU::SGPR_32RegClassID;
1267       case 2: return AMDGPU::SGPR_64RegClassID;
1268       case 4: return AMDGPU::SGPR_128RegClassID;
1269       case 8: return AMDGPU::SReg_256RegClassID;
1270       case 16: return AMDGPU::SReg_512RegClassID;
1271     }
1272   }
1273   return -1;
1274 }
1275 
1276 static unsigned getSpecialRegForName(StringRef RegName) {
1277   return StringSwitch<unsigned>(RegName)
1278     .Case("exec", AMDGPU::EXEC)
1279     .Case("vcc", AMDGPU::VCC)
1280     .Case("flat_scratch", AMDGPU::FLAT_SCR)
1281     .Case("m0", AMDGPU::M0)
1282     .Case("scc", AMDGPU::SCC)
1283     .Case("tba", AMDGPU::TBA)
1284     .Case("tma", AMDGPU::TMA)
1285     .Case("flat_scratch_lo", AMDGPU::FLAT_SCR_LO)
1286     .Case("flat_scratch_hi", AMDGPU::FLAT_SCR_HI)
1287     .Case("vcc_lo", AMDGPU::VCC_LO)
1288     .Case("vcc_hi", AMDGPU::VCC_HI)
1289     .Case("exec_lo", AMDGPU::EXEC_LO)
1290     .Case("exec_hi", AMDGPU::EXEC_HI)
1291     .Case("tma_lo", AMDGPU::TMA_LO)
1292     .Case("tma_hi", AMDGPU::TMA_HI)
1293     .Case("tba_lo", AMDGPU::TBA_LO)
1294     .Case("tba_hi", AMDGPU::TBA_HI)
1295     .Default(0);
1296 }
1297 
1298 bool AMDGPUAsmParser::ParseRegister(unsigned &RegNo, SMLoc &StartLoc, SMLoc &EndLoc) {
1299   auto R = parseRegister();
1300   if (!R) return true;
1301   assert(R->isReg());
1302   RegNo = R->getReg();
1303   StartLoc = R->getStartLoc();
1304   EndLoc = R->getEndLoc();
1305   return false;
1306 }
1307 
1308 bool AMDGPUAsmParser::AddNextRegisterToList(unsigned& Reg, unsigned& RegWidth, RegisterKind RegKind, unsigned Reg1, unsigned RegNum)
1309 {
1310   switch (RegKind) {
1311   case IS_SPECIAL:
1312     if (Reg == AMDGPU::EXEC_LO && Reg1 == AMDGPU::EXEC_HI) { Reg = AMDGPU::EXEC; RegWidth = 2; return true; }
1313     if (Reg == AMDGPU::FLAT_SCR_LO && Reg1 == AMDGPU::FLAT_SCR_HI) { Reg = AMDGPU::FLAT_SCR; RegWidth = 2; return true; }
1314     if (Reg == AMDGPU::VCC_LO && Reg1 == AMDGPU::VCC_HI) { Reg = AMDGPU::VCC; RegWidth = 2; return true; }
1315     if (Reg == AMDGPU::TBA_LO && Reg1 == AMDGPU::TBA_HI) { Reg = AMDGPU::TBA; RegWidth = 2; return true; }
1316     if (Reg == AMDGPU::TMA_LO && Reg1 == AMDGPU::TMA_HI) { Reg = AMDGPU::TMA; RegWidth = 2; return true; }
1317     return false;
1318   case IS_VGPR:
1319   case IS_SGPR:
1320   case IS_TTMP:
1321     if (Reg1 != Reg + RegWidth) { return false; }
1322     RegWidth++;
1323     return true;
1324   default:
1325     llvm_unreachable("unexpected register kind");
1326   }
1327 }
1328 
1329 bool AMDGPUAsmParser::ParseAMDGPURegister(RegisterKind& RegKind, unsigned& Reg, unsigned& RegNum, unsigned& RegWidth, unsigned *DwordRegIndex)
1330 {
1331   if (DwordRegIndex) { *DwordRegIndex = 0; }
1332   const MCRegisterInfo *TRI = getContext().getRegisterInfo();
1333   if (getLexer().is(AsmToken::Identifier)) {
1334     StringRef RegName = Parser.getTok().getString();
1335     if ((Reg = getSpecialRegForName(RegName))) {
1336       Parser.Lex();
1337       RegKind = IS_SPECIAL;
1338     } else {
1339       unsigned RegNumIndex = 0;
1340       if (RegName[0] == 'v') {
1341         RegNumIndex = 1;
1342         RegKind = IS_VGPR;
1343       } else if (RegName[0] == 's') {
1344         RegNumIndex = 1;
1345         RegKind = IS_SGPR;
1346       } else if (RegName.startswith("ttmp")) {
1347         RegNumIndex = strlen("ttmp");
1348         RegKind = IS_TTMP;
1349       } else {
1350         return false;
1351       }
1352       if (RegName.size() > RegNumIndex) {
1353         // Single 32-bit register: vXX.
1354         if (RegName.substr(RegNumIndex).getAsInteger(10, RegNum))
1355           return false;
1356         Parser.Lex();
1357         RegWidth = 1;
1358       } else {
1359         // Range of registers: v[XX:YY]. ":YY" is optional.
1360         Parser.Lex();
1361         int64_t RegLo, RegHi;
1362         if (getLexer().isNot(AsmToken::LBrac))
1363           return false;
1364         Parser.Lex();
1365 
1366         if (getParser().parseAbsoluteExpression(RegLo))
1367           return false;
1368 
1369         const bool isRBrace = getLexer().is(AsmToken::RBrac);
1370         if (!isRBrace && getLexer().isNot(AsmToken::Colon))
1371           return false;
1372         Parser.Lex();
1373 
1374         if (isRBrace) {
1375           RegHi = RegLo;
1376         } else {
1377           if (getParser().parseAbsoluteExpression(RegHi))
1378             return false;
1379 
1380           if (getLexer().isNot(AsmToken::RBrac))
1381             return false;
1382           Parser.Lex();
1383         }
1384         RegNum = (unsigned) RegLo;
1385         RegWidth = (RegHi - RegLo) + 1;
1386       }
1387     }
1388   } else if (getLexer().is(AsmToken::LBrac)) {
1389     // List of consecutive registers: [s0,s1,s2,s3]
1390     Parser.Lex();
1391     if (!ParseAMDGPURegister(RegKind, Reg, RegNum, RegWidth, nullptr))
1392       return false;
1393     if (RegWidth != 1)
1394       return false;
1395     RegisterKind RegKind1;
1396     unsigned Reg1, RegNum1, RegWidth1;
1397     do {
1398       if (getLexer().is(AsmToken::Comma)) {
1399         Parser.Lex();
1400       } else if (getLexer().is(AsmToken::RBrac)) {
1401         Parser.Lex();
1402         break;
1403       } else if (ParseAMDGPURegister(RegKind1, Reg1, RegNum1, RegWidth1, nullptr)) {
1404         if (RegWidth1 != 1) {
1405           return false;
1406         }
1407         if (RegKind1 != RegKind) {
1408           return false;
1409         }
1410         if (!AddNextRegisterToList(Reg, RegWidth, RegKind1, Reg1, RegNum1)) {
1411           return false;
1412         }
1413       } else {
1414         return false;
1415       }
1416     } while (true);
1417   } else {
1418     return false;
1419   }
1420   switch (RegKind) {
1421   case IS_SPECIAL:
1422     RegNum = 0;
1423     RegWidth = 1;
1424     break;
1425   case IS_VGPR:
1426   case IS_SGPR:
1427   case IS_TTMP:
1428   {
1429     unsigned Size = 1;
1430     if (RegKind == IS_SGPR || RegKind == IS_TTMP) {
1431       // SGPR and TTMP registers must be aligned. Max required alignment is 4 dwords.
1432       Size = std::min(RegWidth, 4u);
1433     }
1434     if (RegNum % Size != 0)
1435       return false;
1436     if (DwordRegIndex) { *DwordRegIndex = RegNum; }
1437     RegNum = RegNum / Size;
1438     int RCID = getRegClass(RegKind, RegWidth);
1439     if (RCID == -1)
1440       return false;
1441     const MCRegisterClass RC = TRI->getRegClass(RCID);
1442     if (RegNum >= RC.getNumRegs())
1443       return false;
1444     Reg = RC.getRegister(RegNum);
1445     break;
1446   }
1447 
1448   default:
1449     llvm_unreachable("unexpected register kind");
1450   }
1451 
1452   if (!subtargetHasRegister(*TRI, Reg))
1453     return false;
1454   return true;
1455 }
1456 
1457 std::unique_ptr<AMDGPUOperand> AMDGPUAsmParser::parseRegister() {
1458   const auto &Tok = Parser.getTok();
1459   SMLoc StartLoc = Tok.getLoc();
1460   SMLoc EndLoc = Tok.getEndLoc();
1461   RegisterKind RegKind;
1462   unsigned Reg, RegNum, RegWidth, DwordRegIndex;
1463 
1464   if (!ParseAMDGPURegister(RegKind, Reg, RegNum, RegWidth, &DwordRegIndex)) {
1465     return nullptr;
1466   }
1467   KernelScope.usesRegister(RegKind, DwordRegIndex, RegWidth);
1468   return AMDGPUOperand::CreateReg(this, Reg, StartLoc, EndLoc, false);
1469 }
1470 
1471 OperandMatchResultTy
1472 AMDGPUAsmParser::parseImm(OperandVector &Operands) {
1473   // TODO: add syntactic sugar for 1/(2*PI)
1474   bool Minus = false;
1475   if (getLexer().getKind() == AsmToken::Minus) {
1476     Minus = true;
1477     Parser.Lex();
1478   }
1479 
1480   SMLoc S = Parser.getTok().getLoc();
1481   switch(getLexer().getKind()) {
1482   case AsmToken::Integer: {
1483     int64_t IntVal;
1484     if (getParser().parseAbsoluteExpression(IntVal))
1485       return MatchOperand_ParseFail;
1486     if (Minus)
1487       IntVal *= -1;
1488     Operands.push_back(AMDGPUOperand::CreateImm(this, IntVal, S));
1489     return MatchOperand_Success;
1490   }
1491   case AsmToken::Real: {
1492     int64_t IntVal;
1493     if (getParser().parseAbsoluteExpression(IntVal))
1494       return MatchOperand_ParseFail;
1495 
1496     APFloat F(BitsToDouble(IntVal));
1497     if (Minus)
1498       F.changeSign();
1499     Operands.push_back(
1500         AMDGPUOperand::CreateImm(this, F.bitcastToAPInt().getZExtValue(), S,
1501                                  AMDGPUOperand::ImmTyNone, true));
1502     return MatchOperand_Success;
1503   }
1504   default:
1505     return Minus ? MatchOperand_ParseFail : MatchOperand_NoMatch;
1506   }
1507 }
1508 
1509 OperandMatchResultTy
1510 AMDGPUAsmParser::parseReg(OperandVector &Operands) {
1511   if (auto R = parseRegister()) {
1512     assert(R->isReg());
1513     R->Reg.IsForcedVOP3 = isForcedVOP3();
1514     Operands.push_back(std::move(R));
1515     return MatchOperand_Success;
1516   }
1517   return MatchOperand_NoMatch;
1518 }
1519 
1520 OperandMatchResultTy
1521 AMDGPUAsmParser::parseRegOrImm(OperandVector &Operands) {
1522   auto res = parseImm(Operands);
1523   if (res != MatchOperand_NoMatch) {
1524     return res;
1525   }
1526 
1527   return parseReg(Operands);
1528 }
1529 
1530 OperandMatchResultTy
1531 AMDGPUAsmParser::parseRegOrImmWithFPInputMods(OperandVector &Operands, bool AllowImm) {
1532   // XXX: During parsing we can't determine if minus sign means
1533   // negate-modifier or negative immediate value.
1534   // By default we suppose it is modifier.
1535   bool Negate = false, Abs = false, Abs2 = false;
1536 
1537   if (getLexer().getKind()== AsmToken::Minus) {
1538     Parser.Lex();
1539     Negate = true;
1540   }
1541 
1542   if (getLexer().getKind() == AsmToken::Identifier && Parser.getTok().getString() == "abs") {
1543     Parser.Lex();
1544     Abs2 = true;
1545     if (getLexer().isNot(AsmToken::LParen)) {
1546       Error(Parser.getTok().getLoc(), "expected left paren after abs");
1547       return MatchOperand_ParseFail;
1548     }
1549     Parser.Lex();
1550   }
1551 
1552   if (getLexer().getKind() == AsmToken::Pipe) {
1553     if (Abs2) {
1554       Error(Parser.getTok().getLoc(), "expected register or immediate");
1555       return MatchOperand_ParseFail;
1556     }
1557     Parser.Lex();
1558     Abs = true;
1559   }
1560 
1561   OperandMatchResultTy Res;
1562   if (AllowImm) {
1563     Res = parseRegOrImm(Operands);
1564   } else {
1565     Res = parseReg(Operands);
1566   }
1567   if (Res != MatchOperand_Success) {
1568     return Res;
1569   }
1570 
1571   AMDGPUOperand::Modifiers Mods;
1572   if (Negate) {
1573     Mods.Neg = true;
1574   }
1575   if (Abs) {
1576     if (getLexer().getKind() != AsmToken::Pipe) {
1577       Error(Parser.getTok().getLoc(), "expected vertical bar");
1578       return MatchOperand_ParseFail;
1579     }
1580     Parser.Lex();
1581     Mods.Abs = true;
1582   }
1583   if (Abs2) {
1584     if (getLexer().isNot(AsmToken::RParen)) {
1585       Error(Parser.getTok().getLoc(), "expected closing parentheses");
1586       return MatchOperand_ParseFail;
1587     }
1588     Parser.Lex();
1589     Mods.Abs = true;
1590   }
1591 
1592   if (Mods.hasFPModifiers()) {
1593     AMDGPUOperand &Op = static_cast<AMDGPUOperand &>(*Operands.back());
1594     Op.setModifiers(Mods);
1595   }
1596   return MatchOperand_Success;
1597 }
1598 
1599 OperandMatchResultTy
1600 AMDGPUAsmParser::parseRegOrImmWithIntInputMods(OperandVector &Operands, bool AllowImm) {
1601   bool Sext = false;
1602 
1603   if (getLexer().getKind() == AsmToken::Identifier && Parser.getTok().getString() == "sext") {
1604     Parser.Lex();
1605     Sext = true;
1606     if (getLexer().isNot(AsmToken::LParen)) {
1607       Error(Parser.getTok().getLoc(), "expected left paren after sext");
1608       return MatchOperand_ParseFail;
1609     }
1610     Parser.Lex();
1611   }
1612 
1613   OperandMatchResultTy Res;
1614   if (AllowImm) {
1615     Res = parseRegOrImm(Operands);
1616   } else {
1617     Res = parseReg(Operands);
1618   }
1619   if (Res != MatchOperand_Success) {
1620     return Res;
1621   }
1622 
1623   AMDGPUOperand::Modifiers Mods;
1624   if (Sext) {
1625     if (getLexer().isNot(AsmToken::RParen)) {
1626       Error(Parser.getTok().getLoc(), "expected closing parentheses");
1627       return MatchOperand_ParseFail;
1628     }
1629     Parser.Lex();
1630     Mods.Sext = true;
1631   }
1632 
1633   if (Mods.hasIntModifiers()) {
1634     AMDGPUOperand &Op = static_cast<AMDGPUOperand &>(*Operands.back());
1635     Op.setModifiers(Mods);
1636   }
1637 
1638   return MatchOperand_Success;
1639 }
1640 
1641 OperandMatchResultTy
1642 AMDGPUAsmParser::parseRegWithFPInputMods(OperandVector &Operands) {
1643   return parseRegOrImmWithFPInputMods(Operands, false);
1644 }
1645 
1646 OperandMatchResultTy
1647 AMDGPUAsmParser::parseRegWithIntInputMods(OperandVector &Operands) {
1648   return parseRegOrImmWithIntInputMods(Operands, false);
1649 }
1650 
1651 OperandMatchResultTy AMDGPUAsmParser::parseVReg32OrOff(OperandVector &Operands) {
1652   std::unique_ptr<AMDGPUOperand> Reg = parseRegister();
1653   if (Reg) {
1654     Operands.push_back(std::move(Reg));
1655     return MatchOperand_Success;
1656   }
1657 
1658   const AsmToken &Tok = Parser.getTok();
1659   if (Tok.getString() == "off") {
1660     Operands.push_back(AMDGPUOperand::CreateImm(this, 0, Tok.getLoc(),
1661                                                 AMDGPUOperand::ImmTyOff, false));
1662     Parser.Lex();
1663     return MatchOperand_Success;
1664   }
1665 
1666   return MatchOperand_NoMatch;
1667 }
1668 
1669 unsigned AMDGPUAsmParser::checkTargetMatchPredicate(MCInst &Inst) {
1670 
1671   uint64_t TSFlags = MII.get(Inst.getOpcode()).TSFlags;
1672 
1673   if ((getForcedEncodingSize() == 32 && (TSFlags & SIInstrFlags::VOP3)) ||
1674       (getForcedEncodingSize() == 64 && !(TSFlags & SIInstrFlags::VOP3)) ||
1675       (isForcedDPP() && !(TSFlags & SIInstrFlags::DPP)) ||
1676       (isForcedSDWA() && !(TSFlags & SIInstrFlags::SDWA)) )
1677     return Match_InvalidOperand;
1678 
1679   if ((TSFlags & SIInstrFlags::VOP3) &&
1680       (TSFlags & SIInstrFlags::VOPAsmPrefer32Bit) &&
1681       getForcedEncodingSize() != 64)
1682     return Match_PreferE32;
1683 
1684   if (Inst.getOpcode() == AMDGPU::V_MAC_F32_sdwa_vi ||
1685       Inst.getOpcode() == AMDGPU::V_MAC_F16_sdwa_vi) {
1686     // v_mac_f32/16 allow only dst_sel == DWORD;
1687     auto OpNum =
1688         AMDGPU::getNamedOperandIdx(Inst.getOpcode(), AMDGPU::OpName::dst_sel);
1689     const auto &Op = Inst.getOperand(OpNum);
1690     if (!Op.isImm() || Op.getImm() != AMDGPU::SDWA::SdwaSel::DWORD) {
1691       return Match_InvalidOperand;
1692     }
1693   }
1694 
1695   return Match_Success;
1696 }
1697 
1698 // What asm variants we should check
1699 ArrayRef<unsigned> AMDGPUAsmParser::getMatchedVariants() const {
1700   if (getForcedEncodingSize() == 32) {
1701     static const unsigned Variants[] = {AMDGPUAsmVariants::DEFAULT};
1702     return makeArrayRef(Variants);
1703   }
1704 
1705   if (isForcedVOP3()) {
1706     static const unsigned Variants[] = {AMDGPUAsmVariants::VOP3};
1707     return makeArrayRef(Variants);
1708   }
1709 
1710   if (isForcedSDWA()) {
1711     static const unsigned Variants[] = {AMDGPUAsmVariants::SDWA};
1712     return makeArrayRef(Variants);
1713   }
1714 
1715   if (isForcedDPP()) {
1716     static const unsigned Variants[] = {AMDGPUAsmVariants::DPP};
1717     return makeArrayRef(Variants);
1718   }
1719 
1720   static const unsigned Variants[] = {
1721     AMDGPUAsmVariants::DEFAULT, AMDGPUAsmVariants::VOP3,
1722     AMDGPUAsmVariants::SDWA, AMDGPUAsmVariants::DPP
1723   };
1724 
1725   return makeArrayRef(Variants);
1726 }
1727 
1728 bool AMDGPUAsmParser::MatchAndEmitInstruction(SMLoc IDLoc, unsigned &Opcode,
1729                                               OperandVector &Operands,
1730                                               MCStreamer &Out,
1731                                               uint64_t &ErrorInfo,
1732                                               bool MatchingInlineAsm) {
1733   MCInst Inst;
1734   unsigned Result = Match_Success;
1735   for (auto Variant : getMatchedVariants()) {
1736     uint64_t EI;
1737     auto R = MatchInstructionImpl(Operands, Inst, EI, MatchingInlineAsm,
1738                                   Variant);
1739     // We order match statuses from least to most specific. We use most specific
1740     // status as resulting
1741     // Match_MnemonicFail < Match_InvalidOperand < Match_MissingFeature < Match_PreferE32
1742     if ((R == Match_Success) ||
1743         (R == Match_PreferE32) ||
1744         (R == Match_MissingFeature && Result != Match_PreferE32) ||
1745         (R == Match_InvalidOperand && Result != Match_MissingFeature
1746                                    && Result != Match_PreferE32) ||
1747         (R == Match_MnemonicFail   && Result != Match_InvalidOperand
1748                                    && Result != Match_MissingFeature
1749                                    && Result != Match_PreferE32)) {
1750       Result = R;
1751       ErrorInfo = EI;
1752     }
1753     if (R == Match_Success)
1754       break;
1755   }
1756 
1757   switch (Result) {
1758   default: break;
1759   case Match_Success:
1760     Inst.setLoc(IDLoc);
1761     Out.EmitInstruction(Inst, getSTI());
1762     return false;
1763 
1764   case Match_MissingFeature:
1765     return Error(IDLoc, "instruction not supported on this GPU");
1766 
1767   case Match_MnemonicFail:
1768     return Error(IDLoc, "unrecognized instruction mnemonic");
1769 
1770   case Match_InvalidOperand: {
1771     SMLoc ErrorLoc = IDLoc;
1772     if (ErrorInfo != ~0ULL) {
1773       if (ErrorInfo >= Operands.size()) {
1774         return Error(IDLoc, "too few operands for instruction");
1775       }
1776       ErrorLoc = ((AMDGPUOperand &)*Operands[ErrorInfo]).getStartLoc();
1777       if (ErrorLoc == SMLoc())
1778         ErrorLoc = IDLoc;
1779     }
1780     return Error(ErrorLoc, "invalid operand for instruction");
1781   }
1782 
1783   case Match_PreferE32:
1784     return Error(IDLoc, "internal error: instruction without _e64 suffix "
1785                         "should be encoded as e32");
1786   }
1787   llvm_unreachable("Implement any new match types added!");
1788 }
1789 
1790 bool AMDGPUAsmParser::ParseAsAbsoluteExpression(uint32_t &Ret) {
1791   int64_t Tmp = -1;
1792   if (getLexer().isNot(AsmToken::Integer) && getLexer().isNot(AsmToken::Identifier)) {
1793     return true;
1794   }
1795   if (getParser().parseAbsoluteExpression(Tmp)) {
1796     return true;
1797   }
1798   Ret = static_cast<uint32_t>(Tmp);
1799   return false;
1800 }
1801 
1802 
1803 bool AMDGPUAsmParser::ParseDirectiveMajorMinor(uint32_t &Major,
1804                                                uint32_t &Minor) {
1805   if (ParseAsAbsoluteExpression(Major))
1806     return TokError("invalid major version");
1807 
1808   if (getLexer().isNot(AsmToken::Comma))
1809     return TokError("minor version number required, comma expected");
1810   Lex();
1811 
1812   if (ParseAsAbsoluteExpression(Minor))
1813     return TokError("invalid minor version");
1814 
1815   return false;
1816 }
1817 
1818 bool AMDGPUAsmParser::ParseDirectiveHSACodeObjectVersion() {
1819 
1820   uint32_t Major;
1821   uint32_t Minor;
1822 
1823   if (ParseDirectiveMajorMinor(Major, Minor))
1824     return true;
1825 
1826   getTargetStreamer().EmitDirectiveHSACodeObjectVersion(Major, Minor);
1827   return false;
1828 }
1829 
1830 bool AMDGPUAsmParser::ParseDirectiveHSACodeObjectISA() {
1831   uint32_t Major;
1832   uint32_t Minor;
1833   uint32_t Stepping;
1834   StringRef VendorName;
1835   StringRef ArchName;
1836 
1837   // If this directive has no arguments, then use the ISA version for the
1838   // targeted GPU.
1839   if (getLexer().is(AsmToken::EndOfStatement)) {
1840     AMDGPU::IsaVersion Isa = AMDGPU::getIsaVersion(getSTI().getFeatureBits());
1841     getTargetStreamer().EmitDirectiveHSACodeObjectISA(Isa.Major, Isa.Minor,
1842                                                       Isa.Stepping,
1843                                                       "AMD", "AMDGPU");
1844     return false;
1845   }
1846 
1847   if (ParseDirectiveMajorMinor(Major, Minor))
1848     return true;
1849 
1850   if (getLexer().isNot(AsmToken::Comma))
1851     return TokError("stepping version number required, comma expected");
1852   Lex();
1853 
1854   if (ParseAsAbsoluteExpression(Stepping))
1855     return TokError("invalid stepping version");
1856 
1857   if (getLexer().isNot(AsmToken::Comma))
1858     return TokError("vendor name required, comma expected");
1859   Lex();
1860 
1861   if (getLexer().isNot(AsmToken::String))
1862     return TokError("invalid vendor name");
1863 
1864   VendorName = getLexer().getTok().getStringContents();
1865   Lex();
1866 
1867   if (getLexer().isNot(AsmToken::Comma))
1868     return TokError("arch name required, comma expected");
1869   Lex();
1870 
1871   if (getLexer().isNot(AsmToken::String))
1872     return TokError("invalid arch name");
1873 
1874   ArchName = getLexer().getTok().getStringContents();
1875   Lex();
1876 
1877   getTargetStreamer().EmitDirectiveHSACodeObjectISA(Major, Minor, Stepping,
1878                                                     VendorName, ArchName);
1879   return false;
1880 }
1881 
1882 bool AMDGPUAsmParser::ParseDirectiveRuntimeMetadata() {
1883   std::string Metadata;
1884   raw_string_ostream MS(Metadata);
1885 
1886   getLexer().setSkipSpace(false);
1887 
1888   bool FoundEnd = false;
1889   while (!getLexer().is(AsmToken::Eof)) {
1890     while (getLexer().is(AsmToken::Space)) {
1891       MS << ' ';
1892       Lex();
1893     }
1894 
1895     if (getLexer().is(AsmToken::Identifier)) {
1896       StringRef ID = getLexer().getTok().getIdentifier();
1897       if (ID == ".end_amdgpu_runtime_metadata") {
1898         Lex();
1899         FoundEnd = true;
1900         break;
1901       }
1902     }
1903 
1904     MS << Parser.parseStringToEndOfStatement()
1905        << getContext().getAsmInfo()->getSeparatorString();
1906 
1907     Parser.eatToEndOfStatement();
1908   }
1909 
1910   getLexer().setSkipSpace(true);
1911 
1912   if (getLexer().is(AsmToken::Eof) && !FoundEnd)
1913     return TokError("expected directive .end_amdgpu_runtime_metadata not found");
1914 
1915   MS.flush();
1916 
1917   getTargetStreamer().EmitRuntimeMetadata(Metadata);
1918 
1919   return false;
1920 }
1921 
1922 bool AMDGPUAsmParser::ParseAMDKernelCodeTValue(StringRef ID,
1923                                                amd_kernel_code_t &Header) {
1924   SmallString<40> ErrStr;
1925   raw_svector_ostream Err(ErrStr);
1926   if (!parseAmdKernelCodeField(ID, getParser(), Header, Err)) {
1927     return TokError(Err.str());
1928   }
1929   Lex();
1930   return false;
1931 }
1932 
1933 bool AMDGPUAsmParser::ParseDirectiveAMDKernelCodeT() {
1934   amd_kernel_code_t Header;
1935   AMDGPU::initDefaultAMDKernelCodeT(Header, getSTI().getFeatureBits());
1936 
1937   while (true) {
1938     // Lex EndOfStatement.  This is in a while loop, because lexing a comment
1939     // will set the current token to EndOfStatement.
1940     while(getLexer().is(AsmToken::EndOfStatement))
1941       Lex();
1942 
1943     if (getLexer().isNot(AsmToken::Identifier))
1944       return TokError("expected value identifier or .end_amd_kernel_code_t");
1945 
1946     StringRef ID = getLexer().getTok().getIdentifier();
1947     Lex();
1948 
1949     if (ID == ".end_amd_kernel_code_t")
1950       break;
1951 
1952     if (ParseAMDKernelCodeTValue(ID, Header))
1953       return true;
1954   }
1955 
1956   getTargetStreamer().EmitAMDKernelCodeT(Header);
1957 
1958   return false;
1959 }
1960 
1961 bool AMDGPUAsmParser::ParseSectionDirectiveHSAText() {
1962   getParser().getStreamer().SwitchSection(
1963       AMDGPU::getHSATextSection(getContext()));
1964   return false;
1965 }
1966 
1967 bool AMDGPUAsmParser::ParseDirectiveAMDGPUHsaKernel() {
1968   if (getLexer().isNot(AsmToken::Identifier))
1969     return TokError("expected symbol name");
1970 
1971   StringRef KernelName = Parser.getTok().getString();
1972 
1973   getTargetStreamer().EmitAMDGPUSymbolType(KernelName,
1974                                            ELF::STT_AMDGPU_HSA_KERNEL);
1975   Lex();
1976   KernelScope.initialize(getContext());
1977   return false;
1978 }
1979 
1980 bool AMDGPUAsmParser::ParseDirectiveAMDGPUHsaModuleGlobal() {
1981   if (getLexer().isNot(AsmToken::Identifier))
1982     return TokError("expected symbol name");
1983 
1984   StringRef GlobalName = Parser.getTok().getIdentifier();
1985 
1986   getTargetStreamer().EmitAMDGPUHsaModuleScopeGlobal(GlobalName);
1987   Lex();
1988   return false;
1989 }
1990 
1991 bool AMDGPUAsmParser::ParseDirectiveAMDGPUHsaProgramGlobal() {
1992   if (getLexer().isNot(AsmToken::Identifier))
1993     return TokError("expected symbol name");
1994 
1995   StringRef GlobalName = Parser.getTok().getIdentifier();
1996 
1997   getTargetStreamer().EmitAMDGPUHsaProgramScopeGlobal(GlobalName);
1998   Lex();
1999   return false;
2000 }
2001 
2002 bool AMDGPUAsmParser::ParseSectionDirectiveHSADataGlobalAgent() {
2003   getParser().getStreamer().SwitchSection(
2004       AMDGPU::getHSADataGlobalAgentSection(getContext()));
2005   return false;
2006 }
2007 
2008 bool AMDGPUAsmParser::ParseSectionDirectiveHSADataGlobalProgram() {
2009   getParser().getStreamer().SwitchSection(
2010       AMDGPU::getHSADataGlobalProgramSection(getContext()));
2011   return false;
2012 }
2013 
2014 bool AMDGPUAsmParser::ParseSectionDirectiveHSARodataReadonlyAgent() {
2015   getParser().getStreamer().SwitchSection(
2016       AMDGPU::getHSARodataReadonlyAgentSection(getContext()));
2017   return false;
2018 }
2019 
2020 bool AMDGPUAsmParser::ParseDirective(AsmToken DirectiveID) {
2021   StringRef IDVal = DirectiveID.getString();
2022 
2023   if (IDVal == ".hsa_code_object_version")
2024     return ParseDirectiveHSACodeObjectVersion();
2025 
2026   if (IDVal == ".hsa_code_object_isa")
2027     return ParseDirectiveHSACodeObjectISA();
2028 
2029   if (IDVal == ".amdgpu_runtime_metadata")
2030     return ParseDirectiveRuntimeMetadata();
2031 
2032   if (IDVal == ".amd_kernel_code_t")
2033     return ParseDirectiveAMDKernelCodeT();
2034 
2035   if (IDVal == ".hsatext")
2036     return ParseSectionDirectiveHSAText();
2037 
2038   if (IDVal == ".amdgpu_hsa_kernel")
2039     return ParseDirectiveAMDGPUHsaKernel();
2040 
2041   if (IDVal == ".amdgpu_hsa_module_global")
2042     return ParseDirectiveAMDGPUHsaModuleGlobal();
2043 
2044   if (IDVal == ".amdgpu_hsa_program_global")
2045     return ParseDirectiveAMDGPUHsaProgramGlobal();
2046 
2047   if (IDVal == ".hsadata_global_agent")
2048     return ParseSectionDirectiveHSADataGlobalAgent();
2049 
2050   if (IDVal == ".hsadata_global_program")
2051     return ParseSectionDirectiveHSADataGlobalProgram();
2052 
2053   if (IDVal == ".hsarodata_readonly_agent")
2054     return ParseSectionDirectiveHSARodataReadonlyAgent();
2055 
2056   return true;
2057 }
2058 
2059 bool AMDGPUAsmParser::subtargetHasRegister(const MCRegisterInfo &MRI,
2060                                            unsigned RegNo) const {
2061   if (isCI())
2062     return true;
2063 
2064   if (isSI()) {
2065     // No flat_scr
2066     switch (RegNo) {
2067     case AMDGPU::FLAT_SCR:
2068     case AMDGPU::FLAT_SCR_LO:
2069     case AMDGPU::FLAT_SCR_HI:
2070       return false;
2071     default:
2072       return true;
2073     }
2074   }
2075 
2076   // VI only has 102 SGPRs, so make sure we aren't trying to use the 2 more that
2077   // SI/CI have.
2078   for (MCRegAliasIterator R(AMDGPU::SGPR102_SGPR103, &MRI, true);
2079        R.isValid(); ++R) {
2080     if (*R == RegNo)
2081       return false;
2082   }
2083 
2084   return true;
2085 }
2086 
2087 OperandMatchResultTy
2088 AMDGPUAsmParser::parseOperand(OperandVector &Operands, StringRef Mnemonic) {
2089 
2090   // Try to parse with a custom parser
2091   OperandMatchResultTy ResTy = MatchOperandParserImpl(Operands, Mnemonic);
2092 
2093   // If we successfully parsed the operand or if there as an error parsing,
2094   // we are done.
2095   //
2096   // If we are parsing after we reach EndOfStatement then this means we
2097   // are appending default values to the Operands list.  This is only done
2098   // by custom parser, so we shouldn't continue on to the generic parsing.
2099   if (ResTy == MatchOperand_Success || ResTy == MatchOperand_ParseFail ||
2100       getLexer().is(AsmToken::EndOfStatement))
2101     return ResTy;
2102 
2103   ResTy = parseRegOrImm(Operands);
2104 
2105   if (ResTy == MatchOperand_Success)
2106     return ResTy;
2107 
2108   if (getLexer().getKind() == AsmToken::Identifier) {
2109     // If this identifier is a symbol, we want to create an expression for it.
2110     // It is a little difficult to distinguish between a symbol name, and
2111     // an instruction flag like 'gds'.  In order to do this, we parse
2112     // all tokens as expressions and then treate the symbol name as the token
2113     // string when we want to interpret the operand as a token.
2114     const auto &Tok = Parser.getTok();
2115     SMLoc S = Tok.getLoc();
2116     const MCExpr *Expr = nullptr;
2117     if (!Parser.parseExpression(Expr)) {
2118       Operands.push_back(AMDGPUOperand::CreateExpr(this, Expr, S));
2119       return MatchOperand_Success;
2120     }
2121 
2122     Operands.push_back(AMDGPUOperand::CreateToken(this, Tok.getString(), Tok.getLoc()));
2123     Parser.Lex();
2124     return MatchOperand_Success;
2125   }
2126   return MatchOperand_NoMatch;
2127 }
2128 
2129 StringRef AMDGPUAsmParser::parseMnemonicSuffix(StringRef Name) {
2130   // Clear any forced encodings from the previous instruction.
2131   setForcedEncodingSize(0);
2132   setForcedDPP(false);
2133   setForcedSDWA(false);
2134 
2135   if (Name.endswith("_e64")) {
2136     setForcedEncodingSize(64);
2137     return Name.substr(0, Name.size() - 4);
2138   } else if (Name.endswith("_e32")) {
2139     setForcedEncodingSize(32);
2140     return Name.substr(0, Name.size() - 4);
2141   } else if (Name.endswith("_dpp")) {
2142     setForcedDPP(true);
2143     return Name.substr(0, Name.size() - 4);
2144   } else if (Name.endswith("_sdwa")) {
2145     setForcedSDWA(true);
2146     return Name.substr(0, Name.size() - 5);
2147   }
2148   return Name;
2149 }
2150 
2151 bool AMDGPUAsmParser::ParseInstruction(ParseInstructionInfo &Info,
2152                                        StringRef Name,
2153                                        SMLoc NameLoc, OperandVector &Operands) {
2154   // Add the instruction mnemonic
2155   Name = parseMnemonicSuffix(Name);
2156   Operands.push_back(AMDGPUOperand::CreateToken(this, Name, NameLoc));
2157 
2158   while (!getLexer().is(AsmToken::EndOfStatement)) {
2159     OperandMatchResultTy Res = parseOperand(Operands, Name);
2160 
2161     // Eat the comma or space if there is one.
2162     if (getLexer().is(AsmToken::Comma))
2163       Parser.Lex();
2164 
2165     switch (Res) {
2166       case MatchOperand_Success: break;
2167       case MatchOperand_ParseFail:
2168         Error(getLexer().getLoc(), "failed parsing operand.");
2169         while (!getLexer().is(AsmToken::EndOfStatement)) {
2170           Parser.Lex();
2171         }
2172         return true;
2173       case MatchOperand_NoMatch:
2174         Error(getLexer().getLoc(), "not a valid operand.");
2175         while (!getLexer().is(AsmToken::EndOfStatement)) {
2176           Parser.Lex();
2177         }
2178         return true;
2179     }
2180   }
2181 
2182   return false;
2183 }
2184 
2185 //===----------------------------------------------------------------------===//
2186 // Utility functions
2187 //===----------------------------------------------------------------------===//
2188 
2189 OperandMatchResultTy
2190 AMDGPUAsmParser::parseIntWithPrefix(const char *Prefix, int64_t &Int) {
2191   switch(getLexer().getKind()) {
2192     default: return MatchOperand_NoMatch;
2193     case AsmToken::Identifier: {
2194       StringRef Name = Parser.getTok().getString();
2195       if (!Name.equals(Prefix)) {
2196         return MatchOperand_NoMatch;
2197       }
2198 
2199       Parser.Lex();
2200       if (getLexer().isNot(AsmToken::Colon))
2201         return MatchOperand_ParseFail;
2202 
2203       Parser.Lex();
2204       if (getLexer().isNot(AsmToken::Integer))
2205         return MatchOperand_ParseFail;
2206 
2207       if (getParser().parseAbsoluteExpression(Int))
2208         return MatchOperand_ParseFail;
2209       break;
2210     }
2211   }
2212   return MatchOperand_Success;
2213 }
2214 
2215 OperandMatchResultTy
2216 AMDGPUAsmParser::parseIntWithPrefix(const char *Prefix, OperandVector &Operands,
2217                                     enum AMDGPUOperand::ImmTy ImmTy,
2218                                     bool (*ConvertResult)(int64_t&)) {
2219   SMLoc S = Parser.getTok().getLoc();
2220   int64_t Value = 0;
2221 
2222   OperandMatchResultTy Res = parseIntWithPrefix(Prefix, Value);
2223   if (Res != MatchOperand_Success)
2224     return Res;
2225 
2226   if (ConvertResult && !ConvertResult(Value)) {
2227     return MatchOperand_ParseFail;
2228   }
2229 
2230   Operands.push_back(AMDGPUOperand::CreateImm(this, Value, S, ImmTy));
2231   return MatchOperand_Success;
2232 }
2233 
2234 OperandMatchResultTy
2235 AMDGPUAsmParser::parseNamedBit(const char *Name, OperandVector &Operands,
2236                                enum AMDGPUOperand::ImmTy ImmTy) {
2237   int64_t Bit = 0;
2238   SMLoc S = Parser.getTok().getLoc();
2239 
2240   // We are at the end of the statement, and this is a default argument, so
2241   // use a default value.
2242   if (getLexer().isNot(AsmToken::EndOfStatement)) {
2243     switch(getLexer().getKind()) {
2244       case AsmToken::Identifier: {
2245         StringRef Tok = Parser.getTok().getString();
2246         if (Tok == Name) {
2247           Bit = 1;
2248           Parser.Lex();
2249         } else if (Tok.startswith("no") && Tok.endswith(Name)) {
2250           Bit = 0;
2251           Parser.Lex();
2252         } else {
2253           return MatchOperand_NoMatch;
2254         }
2255         break;
2256       }
2257       default:
2258         return MatchOperand_NoMatch;
2259     }
2260   }
2261 
2262   Operands.push_back(AMDGPUOperand::CreateImm(this, Bit, S, ImmTy));
2263   return MatchOperand_Success;
2264 }
2265 
2266 typedef std::map<enum AMDGPUOperand::ImmTy, unsigned> OptionalImmIndexMap;
2267 
2268 void addOptionalImmOperand(MCInst& Inst, const OperandVector& Operands,
2269                            OptionalImmIndexMap& OptionalIdx,
2270                            enum AMDGPUOperand::ImmTy ImmT, int64_t Default = 0) {
2271   auto i = OptionalIdx.find(ImmT);
2272   if (i != OptionalIdx.end()) {
2273     unsigned Idx = i->second;
2274     ((AMDGPUOperand &)*Operands[Idx]).addImmOperands(Inst, 1);
2275   } else {
2276     Inst.addOperand(MCOperand::createImm(Default));
2277   }
2278 }
2279 
2280 OperandMatchResultTy
2281 AMDGPUAsmParser::parseStringWithPrefix(StringRef Prefix, StringRef &Value) {
2282   if (getLexer().isNot(AsmToken::Identifier)) {
2283     return MatchOperand_NoMatch;
2284   }
2285   StringRef Tok = Parser.getTok().getString();
2286   if (Tok != Prefix) {
2287     return MatchOperand_NoMatch;
2288   }
2289 
2290   Parser.Lex();
2291   if (getLexer().isNot(AsmToken::Colon)) {
2292     return MatchOperand_ParseFail;
2293   }
2294 
2295   Parser.Lex();
2296   if (getLexer().isNot(AsmToken::Identifier)) {
2297     return MatchOperand_ParseFail;
2298   }
2299 
2300   Value = Parser.getTok().getString();
2301   return MatchOperand_Success;
2302 }
2303 
2304 //===----------------------------------------------------------------------===//
2305 // ds
2306 //===----------------------------------------------------------------------===//
2307 
2308 void AMDGPUAsmParser::cvtDSOffset01(MCInst &Inst,
2309                                     const OperandVector &Operands) {
2310   OptionalImmIndexMap OptionalIdx;
2311 
2312   for (unsigned i = 1, e = Operands.size(); i != e; ++i) {
2313     AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[i]);
2314 
2315     // Add the register arguments
2316     if (Op.isReg()) {
2317       Op.addRegOperands(Inst, 1);
2318       continue;
2319     }
2320 
2321     // Handle optional arguments
2322     OptionalIdx[Op.getImmTy()] = i;
2323   }
2324 
2325   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyOffset0);
2326   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyOffset1);
2327   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyGDS);
2328 
2329   Inst.addOperand(MCOperand::createReg(AMDGPU::M0)); // m0
2330 }
2331 
2332 void AMDGPUAsmParser::cvtDS(MCInst &Inst, const OperandVector &Operands) {
2333   std::map<enum AMDGPUOperand::ImmTy, unsigned> OptionalIdx;
2334   bool GDSOnly = false;
2335 
2336   for (unsigned i = 1, e = Operands.size(); i != e; ++i) {
2337     AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[i]);
2338 
2339     // Add the register arguments
2340     if (Op.isReg()) {
2341       Op.addRegOperands(Inst, 1);
2342       continue;
2343     }
2344 
2345     if (Op.isToken() && Op.getToken() == "gds") {
2346       GDSOnly = true;
2347       continue;
2348     }
2349 
2350     // Handle optional arguments
2351     OptionalIdx[Op.getImmTy()] = i;
2352   }
2353 
2354   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyOffset);
2355   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyGDS);
2356 
2357   if (!GDSOnly) {
2358     addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyGDS);
2359   }
2360   Inst.addOperand(MCOperand::createReg(AMDGPU::M0)); // m0
2361 }
2362 
2363 void AMDGPUAsmParser::cvtExp(MCInst &Inst, const OperandVector &Operands) {
2364   OptionalImmIndexMap OptionalIdx;
2365 
2366   unsigned EnMask = 0;
2367   int SrcIdx = 0;
2368 
2369   for (unsigned i = 1, e = Operands.size(); i != e; ++i) {
2370     AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[i]);
2371 
2372     // Add the register arguments
2373     if (Op.isReg()) {
2374       EnMask |= (1 << SrcIdx);
2375       Op.addRegOperands(Inst, 1);
2376       ++SrcIdx;
2377       continue;
2378     }
2379 
2380     if (Op.isOff()) {
2381       ++SrcIdx;
2382       Inst.addOperand(MCOperand::createReg(AMDGPU::NoRegister));
2383       continue;
2384     }
2385 
2386     if (Op.isImm() && Op.getImmTy() == AMDGPUOperand::ImmTyExpTgt) {
2387       Op.addImmOperands(Inst, 1);
2388       continue;
2389     }
2390 
2391     if (Op.isToken() && Op.getToken() == "done")
2392       continue;
2393 
2394     // Handle optional arguments
2395     OptionalIdx[Op.getImmTy()] = i;
2396   }
2397 
2398   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyExpVM);
2399   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyExpCompr);
2400 
2401   Inst.addOperand(MCOperand::createImm(EnMask));
2402 }
2403 
2404 //===----------------------------------------------------------------------===//
2405 // s_waitcnt
2406 //===----------------------------------------------------------------------===//
2407 
2408 bool AMDGPUAsmParser::parseCnt(int64_t &IntVal) {
2409   StringRef CntName = Parser.getTok().getString();
2410   int64_t CntVal;
2411 
2412   Parser.Lex();
2413   if (getLexer().isNot(AsmToken::LParen))
2414     return true;
2415 
2416   Parser.Lex();
2417   if (getLexer().isNot(AsmToken::Integer))
2418     return true;
2419 
2420   if (getParser().parseAbsoluteExpression(CntVal))
2421     return true;
2422 
2423   if (getLexer().isNot(AsmToken::RParen))
2424     return true;
2425 
2426   Parser.Lex();
2427   if (getLexer().is(AsmToken::Amp) || getLexer().is(AsmToken::Comma))
2428     Parser.Lex();
2429 
2430   IsaVersion IV = getIsaVersion(getSTI().getFeatureBits());
2431   if (CntName == "vmcnt")
2432     IntVal = encodeVmcnt(IV, IntVal, CntVal);
2433   else if (CntName == "expcnt")
2434     IntVal = encodeExpcnt(IV, IntVal, CntVal);
2435   else if (CntName == "lgkmcnt")
2436     IntVal = encodeLgkmcnt(IV, IntVal, CntVal);
2437   else
2438     return true;
2439 
2440   return false;
2441 }
2442 
2443 OperandMatchResultTy
2444 AMDGPUAsmParser::parseSWaitCntOps(OperandVector &Operands) {
2445   IsaVersion IV = getIsaVersion(getSTI().getFeatureBits());
2446   int64_t Waitcnt = getWaitcntBitMask(IV);
2447   SMLoc S = Parser.getTok().getLoc();
2448 
2449   switch(getLexer().getKind()) {
2450     default: return MatchOperand_ParseFail;
2451     case AsmToken::Integer:
2452       // The operand can be an integer value.
2453       if (getParser().parseAbsoluteExpression(Waitcnt))
2454         return MatchOperand_ParseFail;
2455       break;
2456 
2457     case AsmToken::Identifier:
2458       do {
2459         if (parseCnt(Waitcnt))
2460           return MatchOperand_ParseFail;
2461       } while(getLexer().isNot(AsmToken::EndOfStatement));
2462       break;
2463   }
2464   Operands.push_back(AMDGPUOperand::CreateImm(this, Waitcnt, S));
2465   return MatchOperand_Success;
2466 }
2467 
2468 bool AMDGPUAsmParser::parseHwregConstruct(OperandInfoTy &HwReg, int64_t &Offset, int64_t &Width) {
2469   using namespace llvm::AMDGPU::Hwreg;
2470 
2471   if (Parser.getTok().getString() != "hwreg")
2472     return true;
2473   Parser.Lex();
2474 
2475   if (getLexer().isNot(AsmToken::LParen))
2476     return true;
2477   Parser.Lex();
2478 
2479   if (getLexer().is(AsmToken::Identifier)) {
2480     HwReg.IsSymbolic = true;
2481     HwReg.Id = ID_UNKNOWN_;
2482     const StringRef tok = Parser.getTok().getString();
2483     for (int i = ID_SYMBOLIC_FIRST_; i < ID_SYMBOLIC_LAST_; ++i) {
2484       if (tok == IdSymbolic[i]) {
2485         HwReg.Id = i;
2486         break;
2487       }
2488     }
2489     Parser.Lex();
2490   } else {
2491     HwReg.IsSymbolic = false;
2492     if (getLexer().isNot(AsmToken::Integer))
2493       return true;
2494     if (getParser().parseAbsoluteExpression(HwReg.Id))
2495       return true;
2496   }
2497 
2498   if (getLexer().is(AsmToken::RParen)) {
2499     Parser.Lex();
2500     return false;
2501   }
2502 
2503   // optional params
2504   if (getLexer().isNot(AsmToken::Comma))
2505     return true;
2506   Parser.Lex();
2507 
2508   if (getLexer().isNot(AsmToken::Integer))
2509     return true;
2510   if (getParser().parseAbsoluteExpression(Offset))
2511     return true;
2512 
2513   if (getLexer().isNot(AsmToken::Comma))
2514     return true;
2515   Parser.Lex();
2516 
2517   if (getLexer().isNot(AsmToken::Integer))
2518     return true;
2519   if (getParser().parseAbsoluteExpression(Width))
2520     return true;
2521 
2522   if (getLexer().isNot(AsmToken::RParen))
2523     return true;
2524   Parser.Lex();
2525 
2526   return false;
2527 }
2528 
2529 OperandMatchResultTy
2530 AMDGPUAsmParser::parseHwreg(OperandVector &Operands) {
2531   using namespace llvm::AMDGPU::Hwreg;
2532 
2533   int64_t Imm16Val = 0;
2534   SMLoc S = Parser.getTok().getLoc();
2535 
2536   switch(getLexer().getKind()) {
2537     default: return MatchOperand_NoMatch;
2538     case AsmToken::Integer:
2539       // The operand can be an integer value.
2540       if (getParser().parseAbsoluteExpression(Imm16Val))
2541         return MatchOperand_NoMatch;
2542       if (Imm16Val < 0 || !isUInt<16>(Imm16Val)) {
2543         Error(S, "invalid immediate: only 16-bit values are legal");
2544         // Do not return error code, but create an imm operand anyway and proceed
2545         // to the next operand, if any. That avoids unneccessary error messages.
2546       }
2547       break;
2548 
2549     case AsmToken::Identifier: {
2550         OperandInfoTy HwReg(ID_UNKNOWN_);
2551         int64_t Offset = OFFSET_DEFAULT_;
2552         int64_t Width = WIDTH_M1_DEFAULT_ + 1;
2553         if (parseHwregConstruct(HwReg, Offset, Width))
2554           return MatchOperand_ParseFail;
2555         if (HwReg.Id < 0 || !isUInt<ID_WIDTH_>(HwReg.Id)) {
2556           if (HwReg.IsSymbolic)
2557             Error(S, "invalid symbolic name of hardware register");
2558           else
2559             Error(S, "invalid code of hardware register: only 6-bit values are legal");
2560         }
2561         if (Offset < 0 || !isUInt<OFFSET_WIDTH_>(Offset))
2562           Error(S, "invalid bit offset: only 5-bit values are legal");
2563         if ((Width-1) < 0 || !isUInt<WIDTH_M1_WIDTH_>(Width-1))
2564           Error(S, "invalid bitfield width: only values from 1 to 32 are legal");
2565         Imm16Val = (HwReg.Id << ID_SHIFT_) | (Offset << OFFSET_SHIFT_) | ((Width-1) << WIDTH_M1_SHIFT_);
2566       }
2567       break;
2568   }
2569   Operands.push_back(AMDGPUOperand::CreateImm(this, Imm16Val, S, AMDGPUOperand::ImmTyHwreg));
2570   return MatchOperand_Success;
2571 }
2572 
2573 bool AMDGPUOperand::isSWaitCnt() const {
2574   return isImm();
2575 }
2576 
2577 bool AMDGPUOperand::isHwreg() const {
2578   return isImmTy(ImmTyHwreg);
2579 }
2580 
2581 bool AMDGPUAsmParser::parseSendMsgConstruct(OperandInfoTy &Msg, OperandInfoTy &Operation, int64_t &StreamId) {
2582   using namespace llvm::AMDGPU::SendMsg;
2583 
2584   if (Parser.getTok().getString() != "sendmsg")
2585     return true;
2586   Parser.Lex();
2587 
2588   if (getLexer().isNot(AsmToken::LParen))
2589     return true;
2590   Parser.Lex();
2591 
2592   if (getLexer().is(AsmToken::Identifier)) {
2593     Msg.IsSymbolic = true;
2594     Msg.Id = ID_UNKNOWN_;
2595     const std::string tok = Parser.getTok().getString();
2596     for (int i = ID_GAPS_FIRST_; i < ID_GAPS_LAST_; ++i) {
2597       switch(i) {
2598         default: continue; // Omit gaps.
2599         case ID_INTERRUPT: case ID_GS: case ID_GS_DONE:  case ID_SYSMSG: break;
2600       }
2601       if (tok == IdSymbolic[i]) {
2602         Msg.Id = i;
2603         break;
2604       }
2605     }
2606     Parser.Lex();
2607   } else {
2608     Msg.IsSymbolic = false;
2609     if (getLexer().isNot(AsmToken::Integer))
2610       return true;
2611     if (getParser().parseAbsoluteExpression(Msg.Id))
2612       return true;
2613     if (getLexer().is(AsmToken::Integer))
2614       if (getParser().parseAbsoluteExpression(Msg.Id))
2615         Msg.Id = ID_UNKNOWN_;
2616   }
2617   if (Msg.Id == ID_UNKNOWN_) // Don't know how to parse the rest.
2618     return false;
2619 
2620   if (!(Msg.Id == ID_GS || Msg.Id == ID_GS_DONE || Msg.Id == ID_SYSMSG)) {
2621     if (getLexer().isNot(AsmToken::RParen))
2622       return true;
2623     Parser.Lex();
2624     return false;
2625   }
2626 
2627   if (getLexer().isNot(AsmToken::Comma))
2628     return true;
2629   Parser.Lex();
2630 
2631   assert(Msg.Id == ID_GS || Msg.Id == ID_GS_DONE || Msg.Id == ID_SYSMSG);
2632   Operation.Id = ID_UNKNOWN_;
2633   if (getLexer().is(AsmToken::Identifier)) {
2634     Operation.IsSymbolic = true;
2635     const char* const *S = (Msg.Id == ID_SYSMSG) ? OpSysSymbolic : OpGsSymbolic;
2636     const int F = (Msg.Id == ID_SYSMSG) ? OP_SYS_FIRST_ : OP_GS_FIRST_;
2637     const int L = (Msg.Id == ID_SYSMSG) ? OP_SYS_LAST_ : OP_GS_LAST_;
2638     const StringRef Tok = Parser.getTok().getString();
2639     for (int i = F; i < L; ++i) {
2640       if (Tok == S[i]) {
2641         Operation.Id = i;
2642         break;
2643       }
2644     }
2645     Parser.Lex();
2646   } else {
2647     Operation.IsSymbolic = false;
2648     if (getLexer().isNot(AsmToken::Integer))
2649       return true;
2650     if (getParser().parseAbsoluteExpression(Operation.Id))
2651       return true;
2652   }
2653 
2654   if ((Msg.Id == ID_GS || Msg.Id == ID_GS_DONE) && Operation.Id != OP_GS_NOP) {
2655     // Stream id is optional.
2656     if (getLexer().is(AsmToken::RParen)) {
2657       Parser.Lex();
2658       return false;
2659     }
2660 
2661     if (getLexer().isNot(AsmToken::Comma))
2662       return true;
2663     Parser.Lex();
2664 
2665     if (getLexer().isNot(AsmToken::Integer))
2666       return true;
2667     if (getParser().parseAbsoluteExpression(StreamId))
2668       return true;
2669   }
2670 
2671   if (getLexer().isNot(AsmToken::RParen))
2672     return true;
2673   Parser.Lex();
2674   return false;
2675 }
2676 
2677 OperandMatchResultTy AMDGPUAsmParser::parseInterpSlot(OperandVector &Operands) {
2678   if (getLexer().getKind() != AsmToken::Identifier)
2679     return MatchOperand_NoMatch;
2680 
2681   StringRef Str = Parser.getTok().getString();
2682   int Slot = StringSwitch<int>(Str)
2683     .Case("p10", 0)
2684     .Case("p20", 1)
2685     .Case("p0", 2)
2686     .Default(-1);
2687 
2688   SMLoc S = Parser.getTok().getLoc();
2689   if (Slot == -1)
2690     return MatchOperand_ParseFail;
2691 
2692   Parser.Lex();
2693   Operands.push_back(AMDGPUOperand::CreateImm(this, Slot, S,
2694                                               AMDGPUOperand::ImmTyInterpSlot));
2695   return MatchOperand_Success;
2696 }
2697 
2698 OperandMatchResultTy AMDGPUAsmParser::parseInterpAttr(OperandVector &Operands) {
2699   if (getLexer().getKind() != AsmToken::Identifier)
2700     return MatchOperand_NoMatch;
2701 
2702   StringRef Str = Parser.getTok().getString();
2703   if (!Str.startswith("attr"))
2704     return MatchOperand_NoMatch;
2705 
2706   StringRef Chan = Str.take_back(2);
2707   int AttrChan = StringSwitch<int>(Chan)
2708     .Case(".x", 0)
2709     .Case(".y", 1)
2710     .Case(".z", 2)
2711     .Case(".w", 3)
2712     .Default(-1);
2713   if (AttrChan == -1)
2714     return MatchOperand_ParseFail;
2715 
2716   Str = Str.drop_back(2).drop_front(4);
2717 
2718   uint8_t Attr;
2719   if (Str.getAsInteger(10, Attr))
2720     return MatchOperand_ParseFail;
2721 
2722   SMLoc S = Parser.getTok().getLoc();
2723   Parser.Lex();
2724   if (Attr > 63) {
2725     Error(S, "out of bounds attr");
2726     return MatchOperand_Success;
2727   }
2728 
2729   SMLoc SChan = SMLoc::getFromPointer(Chan.data());
2730 
2731   Operands.push_back(AMDGPUOperand::CreateImm(this, Attr, S,
2732                                               AMDGPUOperand::ImmTyInterpAttr));
2733   Operands.push_back(AMDGPUOperand::CreateImm(this, AttrChan, SChan,
2734                                               AMDGPUOperand::ImmTyAttrChan));
2735   return MatchOperand_Success;
2736 }
2737 
2738 void AMDGPUAsmParser::errorExpTgt() {
2739   Error(Parser.getTok().getLoc(), "invalid exp target");
2740 }
2741 
2742 OperandMatchResultTy AMDGPUAsmParser::parseExpTgtImpl(StringRef Str,
2743                                                       uint8_t &Val) {
2744   if (Str == "null") {
2745     Val = 9;
2746     return MatchOperand_Success;
2747   }
2748 
2749   if (Str.startswith("mrt")) {
2750     Str = Str.drop_front(3);
2751     if (Str == "z") { // == mrtz
2752       Val = 8;
2753       return MatchOperand_Success;
2754     }
2755 
2756     if (Str.getAsInteger(10, Val))
2757       return MatchOperand_ParseFail;
2758 
2759     if (Val > 7)
2760       errorExpTgt();
2761 
2762     return MatchOperand_Success;
2763   }
2764 
2765   if (Str.startswith("pos")) {
2766     Str = Str.drop_front(3);
2767     if (Str.getAsInteger(10, Val))
2768       return MatchOperand_ParseFail;
2769 
2770     if (Val > 3)
2771       errorExpTgt();
2772 
2773     Val += 12;
2774     return MatchOperand_Success;
2775   }
2776 
2777   if (Str.startswith("param")) {
2778     Str = Str.drop_front(5);
2779     if (Str.getAsInteger(10, Val))
2780       return MatchOperand_ParseFail;
2781 
2782     if (Val >= 32)
2783       errorExpTgt();
2784 
2785     Val += 32;
2786     return MatchOperand_Success;
2787   }
2788 
2789   if (Str.startswith("invalid_target_")) {
2790     Str = Str.drop_front(15);
2791     if (Str.getAsInteger(10, Val))
2792       return MatchOperand_ParseFail;
2793 
2794     errorExpTgt();
2795     return MatchOperand_Success;
2796   }
2797 
2798   return MatchOperand_NoMatch;
2799 }
2800 
2801 OperandMatchResultTy AMDGPUAsmParser::parseExpTgt(OperandVector &Operands) {
2802   uint8_t Val;
2803   StringRef Str = Parser.getTok().getString();
2804 
2805   auto Res = parseExpTgtImpl(Str, Val);
2806   if (Res != MatchOperand_Success)
2807     return Res;
2808 
2809   SMLoc S = Parser.getTok().getLoc();
2810   Parser.Lex();
2811 
2812   Operands.push_back(AMDGPUOperand::CreateImm(this, Val, S,
2813                                               AMDGPUOperand::ImmTyExpTgt));
2814   return MatchOperand_Success;
2815 }
2816 
2817 OperandMatchResultTy
2818 AMDGPUAsmParser::parseSendMsgOp(OperandVector &Operands) {
2819   using namespace llvm::AMDGPU::SendMsg;
2820 
2821   int64_t Imm16Val = 0;
2822   SMLoc S = Parser.getTok().getLoc();
2823 
2824   switch(getLexer().getKind()) {
2825   default:
2826     return MatchOperand_NoMatch;
2827   case AsmToken::Integer:
2828     // The operand can be an integer value.
2829     if (getParser().parseAbsoluteExpression(Imm16Val))
2830       return MatchOperand_NoMatch;
2831     if (Imm16Val < 0 || !isUInt<16>(Imm16Val)) {
2832       Error(S, "invalid immediate: only 16-bit values are legal");
2833       // Do not return error code, but create an imm operand anyway and proceed
2834       // to the next operand, if any. That avoids unneccessary error messages.
2835     }
2836     break;
2837   case AsmToken::Identifier: {
2838       OperandInfoTy Msg(ID_UNKNOWN_);
2839       OperandInfoTy Operation(OP_UNKNOWN_);
2840       int64_t StreamId = STREAM_ID_DEFAULT_;
2841       if (parseSendMsgConstruct(Msg, Operation, StreamId))
2842         return MatchOperand_ParseFail;
2843       do {
2844         // Validate and encode message ID.
2845         if (! ((ID_INTERRUPT <= Msg.Id && Msg.Id <= ID_GS_DONE)
2846                 || Msg.Id == ID_SYSMSG)) {
2847           if (Msg.IsSymbolic)
2848             Error(S, "invalid/unsupported symbolic name of message");
2849           else
2850             Error(S, "invalid/unsupported code of message");
2851           break;
2852         }
2853         Imm16Val = (Msg.Id << ID_SHIFT_);
2854         // Validate and encode operation ID.
2855         if (Msg.Id == ID_GS || Msg.Id == ID_GS_DONE) {
2856           if (! (OP_GS_FIRST_ <= Operation.Id && Operation.Id < OP_GS_LAST_)) {
2857             if (Operation.IsSymbolic)
2858               Error(S, "invalid symbolic name of GS_OP");
2859             else
2860               Error(S, "invalid code of GS_OP: only 2-bit values are legal");
2861             break;
2862           }
2863           if (Operation.Id == OP_GS_NOP
2864               && Msg.Id != ID_GS_DONE) {
2865             Error(S, "invalid GS_OP: NOP is for GS_DONE only");
2866             break;
2867           }
2868           Imm16Val |= (Operation.Id << OP_SHIFT_);
2869         }
2870         if (Msg.Id == ID_SYSMSG) {
2871           if (! (OP_SYS_FIRST_ <= Operation.Id && Operation.Id < OP_SYS_LAST_)) {
2872             if (Operation.IsSymbolic)
2873               Error(S, "invalid/unsupported symbolic name of SYSMSG_OP");
2874             else
2875               Error(S, "invalid/unsupported code of SYSMSG_OP");
2876             break;
2877           }
2878           Imm16Val |= (Operation.Id << OP_SHIFT_);
2879         }
2880         // Validate and encode stream ID.
2881         if ((Msg.Id == ID_GS || Msg.Id == ID_GS_DONE) && Operation.Id != OP_GS_NOP) {
2882           if (! (STREAM_ID_FIRST_ <= StreamId && StreamId < STREAM_ID_LAST_)) {
2883             Error(S, "invalid stream id: only 2-bit values are legal");
2884             break;
2885           }
2886           Imm16Val |= (StreamId << STREAM_ID_SHIFT_);
2887         }
2888       } while (false);
2889     }
2890     break;
2891   }
2892   Operands.push_back(AMDGPUOperand::CreateImm(this, Imm16Val, S, AMDGPUOperand::ImmTySendMsg));
2893   return MatchOperand_Success;
2894 }
2895 
2896 bool AMDGPUOperand::isSendMsg() const {
2897   return isImmTy(ImmTySendMsg);
2898 }
2899 
2900 //===----------------------------------------------------------------------===//
2901 // sopp branch targets
2902 //===----------------------------------------------------------------------===//
2903 
2904 OperandMatchResultTy
2905 AMDGPUAsmParser::parseSOppBrTarget(OperandVector &Operands) {
2906   SMLoc S = Parser.getTok().getLoc();
2907 
2908   switch (getLexer().getKind()) {
2909     default: return MatchOperand_ParseFail;
2910     case AsmToken::Integer: {
2911       int64_t Imm;
2912       if (getParser().parseAbsoluteExpression(Imm))
2913         return MatchOperand_ParseFail;
2914       Operands.push_back(AMDGPUOperand::CreateImm(this, Imm, S));
2915       return MatchOperand_Success;
2916     }
2917 
2918     case AsmToken::Identifier:
2919       Operands.push_back(AMDGPUOperand::CreateExpr(this,
2920           MCSymbolRefExpr::create(getContext().getOrCreateSymbol(
2921                                   Parser.getTok().getString()), getContext()), S));
2922       Parser.Lex();
2923       return MatchOperand_Success;
2924   }
2925 }
2926 
2927 //===----------------------------------------------------------------------===//
2928 // mubuf
2929 //===----------------------------------------------------------------------===//
2930 
2931 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultGLC() const {
2932   return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTyGLC);
2933 }
2934 
2935 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultSLC() const {
2936   return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTySLC);
2937 }
2938 
2939 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultTFE() const {
2940   return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTyTFE);
2941 }
2942 
2943 void AMDGPUAsmParser::cvtMubufImpl(MCInst &Inst,
2944                                const OperandVector &Operands,
2945                                bool IsAtomic, bool IsAtomicReturn) {
2946   OptionalImmIndexMap OptionalIdx;
2947   assert(IsAtomicReturn ? IsAtomic : true);
2948 
2949   for (unsigned i = 1, e = Operands.size(); i != e; ++i) {
2950     AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[i]);
2951 
2952     // Add the register arguments
2953     if (Op.isReg()) {
2954       Op.addRegOperands(Inst, 1);
2955       continue;
2956     }
2957 
2958     // Handle the case where soffset is an immediate
2959     if (Op.isImm() && Op.getImmTy() == AMDGPUOperand::ImmTyNone) {
2960       Op.addImmOperands(Inst, 1);
2961       continue;
2962     }
2963 
2964     // Handle tokens like 'offen' which are sometimes hard-coded into the
2965     // asm string.  There are no MCInst operands for these.
2966     if (Op.isToken()) {
2967       continue;
2968     }
2969     assert(Op.isImm());
2970 
2971     // Handle optional arguments
2972     OptionalIdx[Op.getImmTy()] = i;
2973   }
2974 
2975   // Copy $vdata_in operand and insert as $vdata for MUBUF_Atomic RTN insns.
2976   if (IsAtomicReturn) {
2977     MCInst::iterator I = Inst.begin(); // $vdata_in is always at the beginning.
2978     Inst.insert(I, *I);
2979   }
2980 
2981   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyOffset);
2982   if (!IsAtomic) { // glc is hard-coded.
2983     addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyGLC);
2984   }
2985   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySLC);
2986   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyTFE);
2987 }
2988 
2989 //===----------------------------------------------------------------------===//
2990 // mimg
2991 //===----------------------------------------------------------------------===//
2992 
2993 void AMDGPUAsmParser::cvtMIMG(MCInst &Inst, const OperandVector &Operands) {
2994   unsigned I = 1;
2995   const MCInstrDesc &Desc = MII.get(Inst.getOpcode());
2996   for (unsigned J = 0; J < Desc.getNumDefs(); ++J) {
2997     ((AMDGPUOperand &)*Operands[I++]).addRegOperands(Inst, 1);
2998   }
2999 
3000   OptionalImmIndexMap OptionalIdx;
3001 
3002   for (unsigned E = Operands.size(); I != E; ++I) {
3003     AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[I]);
3004 
3005     // Add the register arguments
3006     if (Op.isRegOrImm()) {
3007       Op.addRegOrImmOperands(Inst, 1);
3008       continue;
3009     } else if (Op.isImmModifier()) {
3010       OptionalIdx[Op.getImmTy()] = I;
3011     } else {
3012       llvm_unreachable("unexpected operand type");
3013     }
3014   }
3015 
3016   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyDMask);
3017   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyUNorm);
3018   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyGLC);
3019   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyDA);
3020   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyR128);
3021   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyTFE);
3022   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyLWE);
3023   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySLC);
3024 }
3025 
3026 void AMDGPUAsmParser::cvtMIMGAtomic(MCInst &Inst, const OperandVector &Operands) {
3027   unsigned I = 1;
3028   const MCInstrDesc &Desc = MII.get(Inst.getOpcode());
3029   for (unsigned J = 0; J < Desc.getNumDefs(); ++J) {
3030     ((AMDGPUOperand &)*Operands[I++]).addRegOperands(Inst, 1);
3031   }
3032 
3033   // Add src, same as dst
3034   ((AMDGPUOperand &)*Operands[I]).addRegOperands(Inst, 1);
3035 
3036   OptionalImmIndexMap OptionalIdx;
3037 
3038   for (unsigned E = Operands.size(); I != E; ++I) {
3039     AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[I]);
3040 
3041     // Add the register arguments
3042     if (Op.isRegOrImm()) {
3043       Op.addRegOrImmOperands(Inst, 1);
3044       continue;
3045     } else if (Op.isImmModifier()) {
3046       OptionalIdx[Op.getImmTy()] = I;
3047     } else {
3048       llvm_unreachable("unexpected operand type");
3049     }
3050   }
3051 
3052   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyDMask);
3053   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyUNorm);
3054   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyGLC);
3055   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyDA);
3056   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyR128);
3057   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyTFE);
3058   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyLWE);
3059   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySLC);
3060 }
3061 
3062 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultDMask() const {
3063   return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTyDMask);
3064 }
3065 
3066 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultUNorm() const {
3067   return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTyUNorm);
3068 }
3069 
3070 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultDA() const {
3071   return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTyDA);
3072 }
3073 
3074 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultR128() const {
3075   return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTyR128);
3076 }
3077 
3078 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultLWE() const {
3079   return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTyLWE);
3080 }
3081 
3082 //===----------------------------------------------------------------------===//
3083 // smrd
3084 //===----------------------------------------------------------------------===//
3085 
3086 bool AMDGPUOperand::isSMRDOffset8() const {
3087   return isImm() && isUInt<8>(getImm());
3088 }
3089 
3090 bool AMDGPUOperand::isSMRDOffset20() const {
3091   return isImm() && isUInt<20>(getImm());
3092 }
3093 
3094 bool AMDGPUOperand::isSMRDLiteralOffset() const {
3095   // 32-bit literals are only supported on CI and we only want to use them
3096   // when the offset is > 8-bits.
3097   return isImm() && !isUInt<8>(getImm()) && isUInt<32>(getImm());
3098 }
3099 
3100 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultSMRDOffset8() const {
3101   return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTyOffset);
3102 }
3103 
3104 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultSMRDOffset20() const {
3105   return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTyOffset);
3106 }
3107 
3108 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultSMRDLiteralOffset() const {
3109   return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTyOffset);
3110 }
3111 
3112 //===----------------------------------------------------------------------===//
3113 // vop3
3114 //===----------------------------------------------------------------------===//
3115 
3116 static bool ConvertOmodMul(int64_t &Mul) {
3117   if (Mul != 1 && Mul != 2 && Mul != 4)
3118     return false;
3119 
3120   Mul >>= 1;
3121   return true;
3122 }
3123 
3124 static bool ConvertOmodDiv(int64_t &Div) {
3125   if (Div == 1) {
3126     Div = 0;
3127     return true;
3128   }
3129 
3130   if (Div == 2) {
3131     Div = 3;
3132     return true;
3133   }
3134 
3135   return false;
3136 }
3137 
3138 static bool ConvertBoundCtrl(int64_t &BoundCtrl) {
3139   if (BoundCtrl == 0) {
3140     BoundCtrl = 1;
3141     return true;
3142   }
3143 
3144   if (BoundCtrl == -1) {
3145     BoundCtrl = 0;
3146     return true;
3147   }
3148 
3149   return false;
3150 }
3151 
3152 // Note: the order in this table matches the order of operands in AsmString.
3153 static const OptionalOperand AMDGPUOptionalOperandTable[] = {
3154   {"offen",   AMDGPUOperand::ImmTyOffen, true, nullptr},
3155   {"idxen",   AMDGPUOperand::ImmTyIdxen, true, nullptr},
3156   {"addr64",  AMDGPUOperand::ImmTyAddr64, true, nullptr},
3157   {"offset0", AMDGPUOperand::ImmTyOffset0, false, nullptr},
3158   {"offset1", AMDGPUOperand::ImmTyOffset1, false, nullptr},
3159   {"gds",     AMDGPUOperand::ImmTyGDS, true, nullptr},
3160   {"offset",  AMDGPUOperand::ImmTyOffset, false, nullptr},
3161   {"glc",     AMDGPUOperand::ImmTyGLC, true, nullptr},
3162   {"slc",     AMDGPUOperand::ImmTySLC, true, nullptr},
3163   {"tfe",     AMDGPUOperand::ImmTyTFE, true, nullptr},
3164   {"clamp",   AMDGPUOperand::ImmTyClampSI, true, nullptr},
3165   {"omod",    AMDGPUOperand::ImmTyOModSI, false, ConvertOmodMul},
3166   {"unorm",   AMDGPUOperand::ImmTyUNorm, true, nullptr},
3167   {"da",      AMDGPUOperand::ImmTyDA,    true, nullptr},
3168   {"r128",    AMDGPUOperand::ImmTyR128,  true, nullptr},
3169   {"lwe",     AMDGPUOperand::ImmTyLWE,   true, nullptr},
3170   {"dmask",   AMDGPUOperand::ImmTyDMask, false, nullptr},
3171   {"row_mask",   AMDGPUOperand::ImmTyDppRowMask, false, nullptr},
3172   {"bank_mask",  AMDGPUOperand::ImmTyDppBankMask, false, nullptr},
3173   {"bound_ctrl", AMDGPUOperand::ImmTyDppBoundCtrl, false, ConvertBoundCtrl},
3174   {"dst_sel",    AMDGPUOperand::ImmTySdwaDstSel, false, nullptr},
3175   {"src0_sel",   AMDGPUOperand::ImmTySdwaSrc0Sel, false, nullptr},
3176   {"src1_sel",   AMDGPUOperand::ImmTySdwaSrc1Sel, false, nullptr},
3177   {"dst_unused", AMDGPUOperand::ImmTySdwaDstUnused, false, nullptr},
3178   {"vm", AMDGPUOperand::ImmTyExpVM, true, nullptr},
3179 };
3180 
3181 OperandMatchResultTy AMDGPUAsmParser::parseOptionalOperand(OperandVector &Operands) {
3182   OperandMatchResultTy res;
3183   for (const OptionalOperand &Op : AMDGPUOptionalOperandTable) {
3184     // try to parse any optional operand here
3185     if (Op.IsBit) {
3186       res = parseNamedBit(Op.Name, Operands, Op.Type);
3187     } else if (Op.Type == AMDGPUOperand::ImmTyOModSI) {
3188       res = parseOModOperand(Operands);
3189     } else if (Op.Type == AMDGPUOperand::ImmTySdwaDstSel ||
3190                Op.Type == AMDGPUOperand::ImmTySdwaSrc0Sel ||
3191                Op.Type == AMDGPUOperand::ImmTySdwaSrc1Sel) {
3192       res = parseSDWASel(Operands, Op.Name, Op.Type);
3193     } else if (Op.Type == AMDGPUOperand::ImmTySdwaDstUnused) {
3194       res = parseSDWADstUnused(Operands);
3195     } else {
3196       res = parseIntWithPrefix(Op.Name, Operands, Op.Type, Op.ConvertResult);
3197     }
3198     if (res != MatchOperand_NoMatch) {
3199       return res;
3200     }
3201   }
3202   return MatchOperand_NoMatch;
3203 }
3204 
3205 OperandMatchResultTy AMDGPUAsmParser::parseOModOperand(OperandVector &Operands) {
3206   StringRef Name = Parser.getTok().getString();
3207   if (Name == "mul") {
3208     return parseIntWithPrefix("mul", Operands,
3209                               AMDGPUOperand::ImmTyOModSI, ConvertOmodMul);
3210   }
3211 
3212   if (Name == "div") {
3213     return parseIntWithPrefix("div", Operands,
3214                               AMDGPUOperand::ImmTyOModSI, ConvertOmodDiv);
3215   }
3216 
3217   return MatchOperand_NoMatch;
3218 }
3219 
3220 void AMDGPUAsmParser::cvtId(MCInst &Inst, const OperandVector &Operands) {
3221   unsigned I = 1;
3222   const MCInstrDesc &Desc = MII.get(Inst.getOpcode());
3223   for (unsigned J = 0; J < Desc.getNumDefs(); ++J) {
3224     ((AMDGPUOperand &)*Operands[I++]).addRegOperands(Inst, 1);
3225   }
3226   for (unsigned E = Operands.size(); I != E; ++I)
3227     ((AMDGPUOperand &)*Operands[I]).addRegOrImmOperands(Inst, 1);
3228 }
3229 
3230 void AMDGPUAsmParser::cvtVOP3_2_mod(MCInst &Inst, const OperandVector &Operands) {
3231   uint64_t TSFlags = MII.get(Inst.getOpcode()).TSFlags;
3232   if (TSFlags & SIInstrFlags::VOP3) {
3233     cvtVOP3(Inst, Operands);
3234   } else {
3235     cvtId(Inst, Operands);
3236   }
3237 }
3238 
3239 static bool isRegOrImmWithInputMods(const MCInstrDesc &Desc, unsigned OpNum) {
3240       // 1. This operand is input modifiers
3241   return Desc.OpInfo[OpNum].OperandType == AMDGPU::OPERAND_INPUT_MODS
3242       // 2. This is not last operand
3243       && Desc.NumOperands > (OpNum + 1)
3244       // 3. Next operand is register class
3245       && Desc.OpInfo[OpNum + 1].RegClass != -1
3246       // 4. Next register is not tied to any other operand
3247       && Desc.getOperandConstraint(OpNum + 1, MCOI::OperandConstraint::TIED_TO) == -1;
3248 }
3249 
3250 void AMDGPUAsmParser::cvtVOP3(MCInst &Inst, const OperandVector &Operands) {
3251   OptionalImmIndexMap OptionalIdx;
3252   unsigned I = 1;
3253   const MCInstrDesc &Desc = MII.get(Inst.getOpcode());
3254   for (unsigned J = 0; J < Desc.getNumDefs(); ++J) {
3255     ((AMDGPUOperand &)*Operands[I++]).addRegOperands(Inst, 1);
3256   }
3257 
3258   for (unsigned E = Operands.size(); I != E; ++I) {
3259     AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[I]);
3260     if (isRegOrImmWithInputMods(Desc, Inst.getNumOperands())) {
3261       Op.addRegOrImmWithFPInputModsOperands(Inst, 2);
3262     } else if (Op.isImm()) {
3263       OptionalIdx[Op.getImmTy()] = I;
3264     } else {
3265       llvm_unreachable("unhandled operand type");
3266     }
3267   }
3268 
3269   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyClampSI);
3270   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyOModSI);
3271 
3272   // special case v_mac_{f16, f32}:
3273   // it has src2 register operand that is tied to dst operand
3274   // we don't allow modifiers for this operand in assembler so src2_modifiers
3275   // should be 0
3276   if (Inst.getOpcode() == AMDGPU::V_MAC_F32_e64_si ||
3277       Inst.getOpcode() == AMDGPU::V_MAC_F32_e64_vi ||
3278       Inst.getOpcode() == AMDGPU::V_MAC_F16_e64_vi) {
3279     auto it = Inst.begin();
3280     std::advance(
3281       it,
3282       AMDGPU::getNamedOperandIdx(Inst.getOpcode() == AMDGPU::V_MAC_F16_e64_vi ?
3283                                      AMDGPU::V_MAC_F16_e64 :
3284                                      AMDGPU::V_MAC_F32_e64,
3285                                  AMDGPU::OpName::src2_modifiers));
3286     it = Inst.insert(it, MCOperand::createImm(0)); // no modifiers for src2
3287     ++it;
3288     Inst.insert(it, Inst.getOperand(0)); // src2 = dst
3289   }
3290 }
3291 
3292 //===----------------------------------------------------------------------===//
3293 // dpp
3294 //===----------------------------------------------------------------------===//
3295 
3296 bool AMDGPUOperand::isDPPCtrl() const {
3297   bool result = isImm() && getImmTy() == ImmTyDppCtrl && isUInt<9>(getImm());
3298   if (result) {
3299     int64_t Imm = getImm();
3300     return ((Imm >= 0x000) && (Imm <= 0x0ff)) ||
3301            ((Imm >= 0x101) && (Imm <= 0x10f)) ||
3302            ((Imm >= 0x111) && (Imm <= 0x11f)) ||
3303            ((Imm >= 0x121) && (Imm <= 0x12f)) ||
3304            (Imm == 0x130) ||
3305            (Imm == 0x134) ||
3306            (Imm == 0x138) ||
3307            (Imm == 0x13c) ||
3308            (Imm == 0x140) ||
3309            (Imm == 0x141) ||
3310            (Imm == 0x142) ||
3311            (Imm == 0x143);
3312   }
3313   return false;
3314 }
3315 
3316 bool AMDGPUOperand::isGPRIdxMode() const {
3317   return isImm() && isUInt<4>(getImm());
3318 }
3319 
3320 OperandMatchResultTy
3321 AMDGPUAsmParser::parseDPPCtrl(OperandVector &Operands) {
3322   SMLoc S = Parser.getTok().getLoc();
3323   StringRef Prefix;
3324   int64_t Int;
3325 
3326   if (getLexer().getKind() == AsmToken::Identifier) {
3327     Prefix = Parser.getTok().getString();
3328   } else {
3329     return MatchOperand_NoMatch;
3330   }
3331 
3332   if (Prefix == "row_mirror") {
3333     Int = 0x140;
3334     Parser.Lex();
3335   } else if (Prefix == "row_half_mirror") {
3336     Int = 0x141;
3337     Parser.Lex();
3338   } else {
3339     // Check to prevent parseDPPCtrlOps from eating invalid tokens
3340     if (Prefix != "quad_perm"
3341         && Prefix != "row_shl"
3342         && Prefix != "row_shr"
3343         && Prefix != "row_ror"
3344         && Prefix != "wave_shl"
3345         && Prefix != "wave_rol"
3346         && Prefix != "wave_shr"
3347         && Prefix != "wave_ror"
3348         && Prefix != "row_bcast") {
3349       return MatchOperand_NoMatch;
3350     }
3351 
3352     Parser.Lex();
3353     if (getLexer().isNot(AsmToken::Colon))
3354       return MatchOperand_ParseFail;
3355 
3356     if (Prefix == "quad_perm") {
3357       // quad_perm:[%d,%d,%d,%d]
3358       Parser.Lex();
3359       if (getLexer().isNot(AsmToken::LBrac))
3360         return MatchOperand_ParseFail;
3361       Parser.Lex();
3362 
3363       if (getParser().parseAbsoluteExpression(Int) || !(0 <= Int && Int <=3))
3364         return MatchOperand_ParseFail;
3365 
3366       for (int i = 0; i < 3; ++i) {
3367         if (getLexer().isNot(AsmToken::Comma))
3368           return MatchOperand_ParseFail;
3369         Parser.Lex();
3370 
3371         int64_t Temp;
3372         if (getParser().parseAbsoluteExpression(Temp) || !(0 <= Temp && Temp <=3))
3373           return MatchOperand_ParseFail;
3374         const int shift = i*2 + 2;
3375         Int += (Temp << shift);
3376       }
3377 
3378       if (getLexer().isNot(AsmToken::RBrac))
3379         return MatchOperand_ParseFail;
3380       Parser.Lex();
3381 
3382     } else {
3383       // sel:%d
3384       Parser.Lex();
3385       if (getParser().parseAbsoluteExpression(Int))
3386         return MatchOperand_ParseFail;
3387 
3388       if (Prefix == "row_shl" && 1 <= Int && Int <= 15) {
3389         Int |= 0x100;
3390       } else if (Prefix == "row_shr" && 1 <= Int && Int <= 15) {
3391         Int |= 0x110;
3392       } else if (Prefix == "row_ror" && 1 <= Int && Int <= 15) {
3393         Int |= 0x120;
3394       } else if (Prefix == "wave_shl" && 1 == Int) {
3395         Int = 0x130;
3396       } else if (Prefix == "wave_rol" && 1 == Int) {
3397         Int = 0x134;
3398       } else if (Prefix == "wave_shr" && 1 == Int) {
3399         Int = 0x138;
3400       } else if (Prefix == "wave_ror" && 1 == Int) {
3401         Int = 0x13C;
3402       } else if (Prefix == "row_bcast") {
3403         if (Int == 15) {
3404           Int = 0x142;
3405         } else if (Int == 31) {
3406           Int = 0x143;
3407         } else {
3408           return MatchOperand_ParseFail;
3409         }
3410       } else {
3411         return MatchOperand_ParseFail;
3412       }
3413     }
3414   }
3415 
3416   Operands.push_back(AMDGPUOperand::CreateImm(this, Int, S, AMDGPUOperand::ImmTyDppCtrl));
3417   return MatchOperand_Success;
3418 }
3419 
3420 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultRowMask() const {
3421   return AMDGPUOperand::CreateImm(this, 0xf, SMLoc(), AMDGPUOperand::ImmTyDppRowMask);
3422 }
3423 
3424 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultBankMask() const {
3425   return AMDGPUOperand::CreateImm(this, 0xf, SMLoc(), AMDGPUOperand::ImmTyDppBankMask);
3426 }
3427 
3428 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultBoundCtrl() const {
3429   return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTyDppBoundCtrl);
3430 }
3431 
3432 void AMDGPUAsmParser::cvtDPP(MCInst &Inst, const OperandVector &Operands) {
3433   OptionalImmIndexMap OptionalIdx;
3434 
3435   unsigned I = 1;
3436   const MCInstrDesc &Desc = MII.get(Inst.getOpcode());
3437   for (unsigned J = 0; J < Desc.getNumDefs(); ++J) {
3438     ((AMDGPUOperand &)*Operands[I++]).addRegOperands(Inst, 1);
3439   }
3440 
3441   for (unsigned E = Operands.size(); I != E; ++I) {
3442     AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[I]);
3443     // Add the register arguments
3444     if (Op.isReg() && Op.Reg.RegNo == AMDGPU::VCC) {
3445       // VOP2b (v_add_u32, v_sub_u32 ...) sdwa use "vcc" token.
3446       // Skip it.
3447       continue;
3448     } if (isRegOrImmWithInputMods(Desc, Inst.getNumOperands())) {
3449       Op.addRegWithFPInputModsOperands(Inst, 2);
3450     } else if (Op.isDPPCtrl()) {
3451       Op.addImmOperands(Inst, 1);
3452     } else if (Op.isImm()) {
3453       // Handle optional arguments
3454       OptionalIdx[Op.getImmTy()] = I;
3455     } else {
3456       llvm_unreachable("Invalid operand type");
3457     }
3458   }
3459 
3460   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyDppRowMask, 0xf);
3461   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyDppBankMask, 0xf);
3462   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyDppBoundCtrl);
3463 
3464   // special case v_mac_{f16, f32}:
3465   // it has src2 register operand that is tied to dst operand
3466   if (Inst.getOpcode() == AMDGPU::V_MAC_F32_dpp ||
3467       Inst.getOpcode() == AMDGPU::V_MAC_F16_dpp) {
3468     auto it = Inst.begin();
3469     std::advance(
3470         it, AMDGPU::getNamedOperandIdx(Inst.getOpcode(), AMDGPU::OpName::src2));
3471     Inst.insert(it, Inst.getOperand(0)); // src2 = dst
3472   }
3473 }
3474 
3475 //===----------------------------------------------------------------------===//
3476 // sdwa
3477 //===----------------------------------------------------------------------===//
3478 
3479 OperandMatchResultTy
3480 AMDGPUAsmParser::parseSDWASel(OperandVector &Operands, StringRef Prefix,
3481                               AMDGPUOperand::ImmTy Type) {
3482   using namespace llvm::AMDGPU::SDWA;
3483 
3484   SMLoc S = Parser.getTok().getLoc();
3485   StringRef Value;
3486   OperandMatchResultTy res;
3487 
3488   res = parseStringWithPrefix(Prefix, Value);
3489   if (res != MatchOperand_Success) {
3490     return res;
3491   }
3492 
3493   int64_t Int;
3494   Int = StringSwitch<int64_t>(Value)
3495         .Case("BYTE_0", SdwaSel::BYTE_0)
3496         .Case("BYTE_1", SdwaSel::BYTE_1)
3497         .Case("BYTE_2", SdwaSel::BYTE_2)
3498         .Case("BYTE_3", SdwaSel::BYTE_3)
3499         .Case("WORD_0", SdwaSel::WORD_0)
3500         .Case("WORD_1", SdwaSel::WORD_1)
3501         .Case("DWORD", SdwaSel::DWORD)
3502         .Default(0xffffffff);
3503   Parser.Lex(); // eat last token
3504 
3505   if (Int == 0xffffffff) {
3506     return MatchOperand_ParseFail;
3507   }
3508 
3509   Operands.push_back(AMDGPUOperand::CreateImm(this, Int, S, Type));
3510   return MatchOperand_Success;
3511 }
3512 
3513 OperandMatchResultTy
3514 AMDGPUAsmParser::parseSDWADstUnused(OperandVector &Operands) {
3515   using namespace llvm::AMDGPU::SDWA;
3516 
3517   SMLoc S = Parser.getTok().getLoc();
3518   StringRef Value;
3519   OperandMatchResultTy res;
3520 
3521   res = parseStringWithPrefix("dst_unused", Value);
3522   if (res != MatchOperand_Success) {
3523     return res;
3524   }
3525 
3526   int64_t Int;
3527   Int = StringSwitch<int64_t>(Value)
3528         .Case("UNUSED_PAD", DstUnused::UNUSED_PAD)
3529         .Case("UNUSED_SEXT", DstUnused::UNUSED_SEXT)
3530         .Case("UNUSED_PRESERVE", DstUnused::UNUSED_PRESERVE)
3531         .Default(0xffffffff);
3532   Parser.Lex(); // eat last token
3533 
3534   if (Int == 0xffffffff) {
3535     return MatchOperand_ParseFail;
3536   }
3537 
3538   Operands.push_back(AMDGPUOperand::CreateImm(this, Int, S, AMDGPUOperand::ImmTySdwaDstUnused));
3539   return MatchOperand_Success;
3540 }
3541 
3542 void AMDGPUAsmParser::cvtSdwaVOP1(MCInst &Inst, const OperandVector &Operands) {
3543   cvtSDWA(Inst, Operands, SIInstrFlags::VOP1);
3544 }
3545 
3546 void AMDGPUAsmParser::cvtSdwaVOP2(MCInst &Inst, const OperandVector &Operands) {
3547   cvtSDWA(Inst, Operands, SIInstrFlags::VOP2);
3548 }
3549 
3550 void AMDGPUAsmParser::cvtSdwaVOPC(MCInst &Inst, const OperandVector &Operands) {
3551   cvtSDWA(Inst, Operands, SIInstrFlags::VOPC);
3552 }
3553 
3554 void AMDGPUAsmParser::cvtSDWA(MCInst &Inst, const OperandVector &Operands,
3555                               uint64_t BasicInstType) {
3556   using namespace llvm::AMDGPU::SDWA;
3557   OptionalImmIndexMap OptionalIdx;
3558 
3559   unsigned I = 1;
3560   const MCInstrDesc &Desc = MII.get(Inst.getOpcode());
3561   for (unsigned J = 0; J < Desc.getNumDefs(); ++J) {
3562     ((AMDGPUOperand &)*Operands[I++]).addRegOperands(Inst, 1);
3563   }
3564 
3565   for (unsigned E = Operands.size(); I != E; ++I) {
3566     AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[I]);
3567     // Add the register arguments
3568     if ((BasicInstType == SIInstrFlags::VOPC ||
3569          BasicInstType == SIInstrFlags::VOP2)&&
3570         Op.isReg() &&
3571         Op.Reg.RegNo == AMDGPU::VCC) {
3572       // VOPC and VOP2b (v_add_u32, v_sub_u32 ...) sdwa use "vcc" token as dst.
3573       // Skip it.
3574       continue;
3575     } else if (isRegOrImmWithInputMods(Desc, Inst.getNumOperands())) {
3576       Op.addRegWithInputModsOperands(Inst, 2);
3577     } else if (Op.isImm()) {
3578       // Handle optional arguments
3579       OptionalIdx[Op.getImmTy()] = I;
3580     } else {
3581       llvm_unreachable("Invalid operand type");
3582     }
3583   }
3584 
3585   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyClampSI, 0);
3586 
3587   if (Inst.getOpcode() != AMDGPU::V_NOP_sdwa_vi) {
3588     // V_NOP_sdwa_vi has no optional sdwa arguments
3589     switch (BasicInstType) {
3590     case SIInstrFlags::VOP1:
3591       addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySdwaDstSel, SdwaSel::DWORD);
3592       addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySdwaDstUnused, DstUnused::UNUSED_PRESERVE);
3593       addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySdwaSrc0Sel, SdwaSel::DWORD);
3594       break;
3595 
3596     case SIInstrFlags::VOP2:
3597       addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySdwaDstSel, SdwaSel::DWORD);
3598       addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySdwaDstUnused, DstUnused::UNUSED_PRESERVE);
3599       addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySdwaSrc0Sel, SdwaSel::DWORD);
3600       addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySdwaSrc1Sel, SdwaSel::DWORD);
3601       break;
3602 
3603     case SIInstrFlags::VOPC:
3604       addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySdwaSrc0Sel, SdwaSel::DWORD);
3605       addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySdwaSrc1Sel, SdwaSel::DWORD);
3606       break;
3607 
3608     default:
3609       llvm_unreachable("Invalid instruction type. Only VOP1, VOP2 and VOPC allowed");
3610     }
3611   }
3612 
3613   // special case v_mac_{f16, f32}:
3614   // it has src2 register operand that is tied to dst operand
3615   if (Inst.getOpcode() == AMDGPU::V_MAC_F32_sdwa_vi ||
3616       Inst.getOpcode() == AMDGPU::V_MAC_F16_sdwa_vi)  {
3617     auto it = Inst.begin();
3618     std::advance(
3619         it, AMDGPU::getNamedOperandIdx(Inst.getOpcode(), AMDGPU::OpName::src2));
3620     Inst.insert(it, Inst.getOperand(0)); // src2 = dst
3621   }
3622 
3623 }
3624 
3625 /// Force static initialization.
3626 extern "C" void LLVMInitializeAMDGPUAsmParser() {
3627   RegisterMCAsmParser<AMDGPUAsmParser> A(getTheAMDGPUTarget());
3628   RegisterMCAsmParser<AMDGPUAsmParser> B(getTheGCNTarget());
3629 }
3630 
3631 #define GET_REGISTER_MATCHER
3632 #define GET_MATCHER_IMPLEMENTATION
3633 #include "AMDGPUGenAsmMatcher.inc"
3634 
3635 // This fuction should be defined after auto-generated include so that we have
3636 // MatchClassKind enum defined
3637 unsigned AMDGPUAsmParser::validateTargetOperandClass(MCParsedAsmOperand &Op,
3638                                                      unsigned Kind) {
3639   // Tokens like "glc" would be parsed as immediate operands in ParseOperand().
3640   // But MatchInstructionImpl() expects to meet token and fails to validate
3641   // operand. This method checks if we are given immediate operand but expect to
3642   // get corresponding token.
3643   AMDGPUOperand &Operand = (AMDGPUOperand&)Op;
3644   switch (Kind) {
3645   case MCK_addr64:
3646     return Operand.isAddr64() ? Match_Success : Match_InvalidOperand;
3647   case MCK_gds:
3648     return Operand.isGDS() ? Match_Success : Match_InvalidOperand;
3649   case MCK_glc:
3650     return Operand.isGLC() ? Match_Success : Match_InvalidOperand;
3651   case MCK_idxen:
3652     return Operand.isIdxen() ? Match_Success : Match_InvalidOperand;
3653   case MCK_offen:
3654     return Operand.isOffen() ? Match_Success : Match_InvalidOperand;
3655   case MCK_SSrcB32:
3656     // When operands have expression values, they will return true for isToken,
3657     // because it is not possible to distinguish between a token and an
3658     // expression at parse time. MatchInstructionImpl() will always try to
3659     // match an operand as a token, when isToken returns true, and when the
3660     // name of the expression is not a valid token, the match will fail,
3661     // so we need to handle it here.
3662     return Operand.isSSrcB32() ? Match_Success : Match_InvalidOperand;
3663   case MCK_SSrcF32:
3664     return Operand.isSSrcF32() ? Match_Success : Match_InvalidOperand;
3665   case MCK_SoppBrTarget:
3666     return Operand.isSoppBrTarget() ? Match_Success : Match_InvalidOperand;
3667   case MCK_VReg32OrOff:
3668     return Operand.isVReg32OrOff() ? Match_Success : Match_InvalidOperand;
3669   case MCK_InterpSlot:
3670     return Operand.isInterpSlot() ? Match_Success : Match_InvalidOperand;
3671   case MCK_Attr:
3672     return Operand.isInterpAttr() ? Match_Success : Match_InvalidOperand;
3673   case MCK_AttrChan:
3674     return Operand.isAttrChan() ? Match_Success : Match_InvalidOperand;
3675   default:
3676     return Match_InvalidOperand;
3677   }
3678 }
3679