1 //===-- AMDGPUAsmParser.cpp - Parse SI asm to MCInst instructions ---------===//
2 //
3 //                     The LLVM Compiler Infrastructure
4 //
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
7 //
8 //===----------------------------------------------------------------------===//
9 
10 #include "AMDKernelCodeT.h"
11 #include "MCTargetDesc/AMDGPUMCTargetDesc.h"
12 #include "MCTargetDesc/AMDGPUTargetStreamer.h"
13 #include "SIDefines.h"
14 #include "Utils/AMDGPUBaseInfo.h"
15 #include "Utils/AMDKernelCodeTUtils.h"
16 #include "Utils/AMDGPUAsmUtils.h"
17 #include "llvm/ADT/APFloat.h"
18 #include "llvm/ADT/APInt.h"
19 #include "llvm/ADT/SmallBitVector.h"
20 #include "llvm/ADT/SmallString.h"
21 #include "llvm/ADT/STLExtras.h"
22 #include "llvm/ADT/StringRef.h"
23 #include "llvm/ADT/StringSwitch.h"
24 #include "llvm/ADT/Twine.h"
25 #include "llvm/CodeGen/MachineValueType.h"
26 #include "llvm/MC/MCAsmInfo.h"
27 #include "llvm/MC/MCContext.h"
28 #include "llvm/MC/MCExpr.h"
29 #include "llvm/MC/MCInst.h"
30 #include "llvm/MC/MCInstrDesc.h"
31 #include "llvm/MC/MCInstrInfo.h"
32 #include "llvm/MC/MCParser/MCAsmLexer.h"
33 #include "llvm/MC/MCParser/MCAsmParser.h"
34 #include "llvm/MC/MCParser/MCAsmParserExtension.h"
35 #include "llvm/MC/MCParser/MCParsedAsmOperand.h"
36 #include "llvm/MC/MCParser/MCTargetAsmParser.h"
37 #include "llvm/MC/MCRegisterInfo.h"
38 #include "llvm/MC/MCStreamer.h"
39 #include "llvm/MC/MCSubtargetInfo.h"
40 #include "llvm/MC/MCSymbol.h"
41 #include "llvm/Support/Casting.h"
42 #include "llvm/Support/Debug.h"
43 #include "llvm/Support/ELF.h"
44 #include "llvm/Support/ErrorHandling.h"
45 #include "llvm/Support/MathExtras.h"
46 #include "llvm/Support/raw_ostream.h"
47 #include "llvm/Support/SMLoc.h"
48 #include "llvm/Support/TargetRegistry.h"
49 #include "llvm/Support/raw_ostream.h"
50 #include "llvm/Support/MathExtras.h"
51 #include <algorithm>
52 #include <cassert>
53 #include <cstdint>
54 #include <cstring>
55 #include <iterator>
56 #include <map>
57 #include <memory>
58 #include <string>
59 #include <vector>
60 
61 using namespace llvm;
62 using namespace llvm::AMDGPU;
63 
64 namespace {
65 
66 class AMDGPUAsmParser;
67 
68 enum RegisterKind { IS_UNKNOWN, IS_VGPR, IS_SGPR, IS_TTMP, IS_SPECIAL };
69 
70 //===----------------------------------------------------------------------===//
71 // Operand
72 //===----------------------------------------------------------------------===//
73 
74 class AMDGPUOperand : public MCParsedAsmOperand {
75   enum KindTy {
76     Token,
77     Immediate,
78     Register,
79     Expression
80   } Kind;
81 
82   SMLoc StartLoc, EndLoc;
83   const AMDGPUAsmParser *AsmParser;
84 
85 public:
86   AMDGPUOperand(enum KindTy Kind_, const AMDGPUAsmParser *AsmParser_)
87     : MCParsedAsmOperand(), Kind(Kind_), AsmParser(AsmParser_) {}
88 
89   typedef std::unique_ptr<AMDGPUOperand> Ptr;
90 
91   struct Modifiers {
92     bool Abs = false;
93     bool Neg = false;
94     bool Sext = false;
95 
96     bool hasFPModifiers() const { return Abs || Neg; }
97     bool hasIntModifiers() const { return Sext; }
98     bool hasModifiers() const { return hasFPModifiers() || hasIntModifiers(); }
99 
100     int64_t getFPModifiersOperand() const {
101       int64_t Operand = 0;
102       Operand |= Abs ? SISrcMods::ABS : 0;
103       Operand |= Neg ? SISrcMods::NEG : 0;
104       return Operand;
105     }
106 
107     int64_t getIntModifiersOperand() const {
108       int64_t Operand = 0;
109       Operand |= Sext ? SISrcMods::SEXT : 0;
110       return Operand;
111     }
112 
113     int64_t getModifiersOperand() const {
114       assert(!(hasFPModifiers() && hasIntModifiers())
115            && "fp and int modifiers should not be used simultaneously");
116       if (hasFPModifiers()) {
117         return getFPModifiersOperand();
118       } else if (hasIntModifiers()) {
119         return getIntModifiersOperand();
120       } else {
121         return 0;
122       }
123     }
124 
125     friend raw_ostream &operator <<(raw_ostream &OS, AMDGPUOperand::Modifiers Mods);
126   };
127 
128   enum ImmTy {
129     ImmTyNone,
130     ImmTyGDS,
131     ImmTyOffen,
132     ImmTyIdxen,
133     ImmTyAddr64,
134     ImmTyOffset,
135     ImmTyOffset0,
136     ImmTyOffset1,
137     ImmTyGLC,
138     ImmTySLC,
139     ImmTyTFE,
140     ImmTyClampSI,
141     ImmTyOModSI,
142     ImmTyDppCtrl,
143     ImmTyDppRowMask,
144     ImmTyDppBankMask,
145     ImmTyDppBoundCtrl,
146     ImmTySdwaDstSel,
147     ImmTySdwaSrc0Sel,
148     ImmTySdwaSrc1Sel,
149     ImmTySdwaDstUnused,
150     ImmTyDMask,
151     ImmTyUNorm,
152     ImmTyDA,
153     ImmTyR128,
154     ImmTyLWE,
155     ImmTyExpTgt,
156     ImmTyExpCompr,
157     ImmTyExpVM,
158     ImmTyHwreg,
159     ImmTyOff,
160     ImmTySendMsg,
161     ImmTyInterpSlot,
162     ImmTyInterpAttr,
163     ImmTyAttrChan
164   };
165 
166   struct TokOp {
167     const char *Data;
168     unsigned Length;
169   };
170 
171   struct ImmOp {
172     int64_t Val;
173     ImmTy Type;
174     bool IsFPImm;
175     Modifiers Mods;
176   };
177 
178   struct RegOp {
179     unsigned RegNo;
180     bool IsForcedVOP3;
181     Modifiers Mods;
182   };
183 
184   union {
185     TokOp Tok;
186     ImmOp Imm;
187     RegOp Reg;
188     const MCExpr *Expr;
189   };
190 
191   bool isToken() const override {
192     if (Kind == Token)
193       return true;
194 
195     if (Kind != Expression || !Expr)
196       return false;
197 
198     // When parsing operands, we can't always tell if something was meant to be
199     // a token, like 'gds', or an expression that references a global variable.
200     // In this case, we assume the string is an expression, and if we need to
201     // interpret is a token, then we treat the symbol name as the token.
202     return isa<MCSymbolRefExpr>(Expr);
203   }
204 
205   bool isImm() const override {
206     return Kind == Immediate;
207   }
208 
209   bool isInlinableImm(MVT type) const;
210   bool isLiteralImm(MVT type) const;
211 
212   bool isRegKind() const {
213     return Kind == Register;
214   }
215 
216   bool isReg() const override {
217     return isRegKind() && !hasModifiers();
218   }
219 
220   bool isRegOrImmWithInputMods(MVT type) const {
221     return isRegKind() || isInlinableImm(type);
222   }
223 
224   bool isRegOrImmWithInt16InputMods() const {
225     return isRegOrImmWithInputMods(MVT::i16);
226   }
227 
228   bool isRegOrImmWithInt32InputMods() const {
229     return isRegOrImmWithInputMods(MVT::i32);
230   }
231 
232   bool isRegOrImmWithInt64InputMods() const {
233     return isRegOrImmWithInputMods(MVT::i64);
234   }
235 
236   bool isRegOrImmWithFP16InputMods() const {
237     return isRegOrImmWithInputMods(MVT::f16);
238   }
239 
240   bool isRegOrImmWithFP32InputMods() const {
241     return isRegOrImmWithInputMods(MVT::f32);
242   }
243 
244   bool isRegOrImmWithFP64InputMods() const {
245     return isRegOrImmWithInputMods(MVT::f64);
246   }
247 
248   bool isVReg() const {
249     return isRegClass(AMDGPU::VGPR_32RegClassID) ||
250            isRegClass(AMDGPU::VReg_64RegClassID) ||
251            isRegClass(AMDGPU::VReg_96RegClassID) ||
252            isRegClass(AMDGPU::VReg_128RegClassID) ||
253            isRegClass(AMDGPU::VReg_256RegClassID) ||
254            isRegClass(AMDGPU::VReg_512RegClassID);
255   }
256 
257   bool isVReg32OrOff() const {
258     return isOff() || isRegClass(AMDGPU::VGPR_32RegClassID);
259   }
260 
261   bool isImmTy(ImmTy ImmT) const {
262     return isImm() && Imm.Type == ImmT;
263   }
264 
265   bool isImmModifier() const {
266     return isImm() && Imm.Type != ImmTyNone;
267   }
268 
269   bool isClampSI() const { return isImmTy(ImmTyClampSI); }
270   bool isOModSI() const { return isImmTy(ImmTyOModSI); }
271   bool isDMask() const { return isImmTy(ImmTyDMask); }
272   bool isUNorm() const { return isImmTy(ImmTyUNorm); }
273   bool isDA() const { return isImmTy(ImmTyDA); }
274   bool isR128() const { return isImmTy(ImmTyUNorm); }
275   bool isLWE() const { return isImmTy(ImmTyLWE); }
276   bool isOff() const { return isImmTy(ImmTyOff); }
277   bool isExpTgt() const { return isImmTy(ImmTyExpTgt); }
278   bool isExpVM() const { return isImmTy(ImmTyExpVM); }
279   bool isExpCompr() const { return isImmTy(ImmTyExpCompr); }
280   bool isOffen() const { return isImmTy(ImmTyOffen); }
281   bool isIdxen() const { return isImmTy(ImmTyIdxen); }
282   bool isAddr64() const { return isImmTy(ImmTyAddr64); }
283   bool isOffset() const { return isImmTy(ImmTyOffset) && isUInt<16>(getImm()); }
284   bool isOffset0() const { return isImmTy(ImmTyOffset0) && isUInt<16>(getImm()); }
285   bool isOffset1() const { return isImmTy(ImmTyOffset1) && isUInt<8>(getImm()); }
286   bool isGDS() const { return isImmTy(ImmTyGDS); }
287   bool isGLC() const { return isImmTy(ImmTyGLC); }
288   bool isSLC() const { return isImmTy(ImmTySLC); }
289   bool isTFE() const { return isImmTy(ImmTyTFE); }
290   bool isBankMask() const { return isImmTy(ImmTyDppBankMask); }
291   bool isRowMask() const { return isImmTy(ImmTyDppRowMask); }
292   bool isBoundCtrl() const { return isImmTy(ImmTyDppBoundCtrl); }
293   bool isSDWADstSel() const { return isImmTy(ImmTySdwaDstSel); }
294   bool isSDWASrc0Sel() const { return isImmTy(ImmTySdwaSrc0Sel); }
295   bool isSDWASrc1Sel() const { return isImmTy(ImmTySdwaSrc1Sel); }
296   bool isSDWADstUnused() const { return isImmTy(ImmTySdwaDstUnused); }
297   bool isInterpSlot() const { return isImmTy(ImmTyInterpSlot); }
298   bool isInterpAttr() const { return isImmTy(ImmTyInterpAttr); }
299   bool isAttrChan() const { return isImmTy(ImmTyAttrChan); }
300 
301   bool isMod() const {
302     return isClampSI() || isOModSI();
303   }
304 
305   bool isRegOrImm() const {
306     return isReg() || isImm();
307   }
308 
309   bool isRegClass(unsigned RCID) const;
310 
311   bool isRegOrInlineNoMods(unsigned RCID, MVT type) const {
312     return (isRegClass(RCID) || isInlinableImm(type)) && !hasModifiers();
313   }
314 
315   bool isSCSrcB16() const {
316     return isRegOrInlineNoMods(AMDGPU::SReg_32RegClassID, MVT::i16);
317   }
318 
319   bool isSCSrcB32() const {
320     return isRegOrInlineNoMods(AMDGPU::SReg_32RegClassID, MVT::i32);
321   }
322 
323   bool isSCSrcB64() const {
324     return isRegOrInlineNoMods(AMDGPU::SReg_64RegClassID, MVT::i64);
325   }
326 
327   bool isSCSrcF16() const {
328     return isRegOrInlineNoMods(AMDGPU::SReg_32RegClassID, MVT::f16);
329   }
330 
331   bool isSCSrcF32() const {
332     return isRegOrInlineNoMods(AMDGPU::SReg_32RegClassID, MVT::f32);
333   }
334 
335   bool isSCSrcF64() const {
336     return isRegOrInlineNoMods(AMDGPU::SReg_64RegClassID, MVT::f64);
337   }
338 
339   bool isSSrcB32() const {
340     return isSCSrcB32() || isLiteralImm(MVT::i32) || isExpr();
341   }
342 
343   bool isSSrcB16() const {
344     return isSCSrcB16() || isLiteralImm(MVT::i16);
345   }
346 
347   bool isSSrcB64() const {
348     // TODO: Find out how SALU supports extension of 32-bit literals to 64 bits.
349     // See isVSrc64().
350     return isSCSrcB64() || isLiteralImm(MVT::i64);
351   }
352 
353   bool isSSrcF32() const {
354     return isSCSrcB32() || isLiteralImm(MVT::f32) || isExpr();
355   }
356 
357   bool isSSrcF64() const {
358     return isSCSrcB64() || isLiteralImm(MVT::f64);
359   }
360 
361   bool isSSrcF16() const {
362     return isSCSrcB16() || isLiteralImm(MVT::f16);
363   }
364 
365   bool isVCSrcB32() const {
366     return isRegOrInlineNoMods(AMDGPU::VS_32RegClassID, MVT::i32);
367   }
368 
369   bool isVCSrcB64() const {
370     return isRegOrInlineNoMods(AMDGPU::VS_64RegClassID, MVT::i64);
371   }
372 
373   bool isVCSrcB16() const {
374     return isRegOrInlineNoMods(AMDGPU::VS_32RegClassID, MVT::i16);
375   }
376 
377   bool isVCSrcF32() const {
378     return isRegOrInlineNoMods(AMDGPU::VS_32RegClassID, MVT::f32);
379   }
380 
381   bool isVCSrcF64() const {
382     return isRegOrInlineNoMods(AMDGPU::VS_64RegClassID, MVT::f64);
383   }
384 
385   bool isVCSrcF16() const {
386     return isRegOrInlineNoMods(AMDGPU::VS_32RegClassID, MVT::f16);
387   }
388 
389   bool isVSrcB32() const {
390     return isVCSrcF32() || isLiteralImm(MVT::i32);
391   }
392 
393   bool isVSrcB64() const {
394     return isVCSrcF64() || isLiteralImm(MVT::i64);
395   }
396 
397   bool isVSrcB16() const {
398     return isVCSrcF16() || isLiteralImm(MVT::i16);
399   }
400 
401   bool isVSrcF32() const {
402     return isVCSrcF32() || isLiteralImm(MVT::f32);
403   }
404 
405   bool isVSrcF64() const {
406     return isVCSrcF64() || isLiteralImm(MVT::f64);
407   }
408 
409   bool isVSrcF16() const {
410     return isVCSrcF16() || isLiteralImm(MVT::f16);
411   }
412 
413   bool isKImmFP32() const {
414     return isLiteralImm(MVT::f32);
415   }
416 
417   bool isKImmFP16() const {
418     return isLiteralImm(MVT::f16);
419   }
420 
421   bool isMem() const override {
422     return false;
423   }
424 
425   bool isExpr() const {
426     return Kind == Expression;
427   }
428 
429   bool isSoppBrTarget() const {
430     return isExpr() || isImm();
431   }
432 
433   bool isSWaitCnt() const;
434   bool isHwreg() const;
435   bool isSendMsg() const;
436   bool isSMRDOffset8() const;
437   bool isSMRDOffset20() const;
438   bool isSMRDLiteralOffset() const;
439   bool isDPPCtrl() const;
440   bool isGPRIdxMode() const;
441 
442   StringRef getExpressionAsToken() const {
443     assert(isExpr());
444     const MCSymbolRefExpr *S = cast<MCSymbolRefExpr>(Expr);
445     return S->getSymbol().getName();
446   }
447 
448   StringRef getToken() const {
449     assert(isToken());
450 
451     if (Kind == Expression)
452       return getExpressionAsToken();
453 
454     return StringRef(Tok.Data, Tok.Length);
455   }
456 
457   int64_t getImm() const {
458     assert(isImm());
459     return Imm.Val;
460   }
461 
462   enum ImmTy getImmTy() const {
463     assert(isImm());
464     return Imm.Type;
465   }
466 
467   unsigned getReg() const override {
468     return Reg.RegNo;
469   }
470 
471   SMLoc getStartLoc() const override {
472     return StartLoc;
473   }
474 
475   SMLoc getEndLoc() const override {
476     return EndLoc;
477   }
478 
479   Modifiers getModifiers() const {
480     assert(isRegKind() || isImmTy(ImmTyNone));
481     return isRegKind() ? Reg.Mods : Imm.Mods;
482   }
483 
484   void setModifiers(Modifiers Mods) {
485     assert(isRegKind() || isImmTy(ImmTyNone));
486     if (isRegKind())
487       Reg.Mods = Mods;
488     else
489       Imm.Mods = Mods;
490   }
491 
492   bool hasModifiers() const {
493     return getModifiers().hasModifiers();
494   }
495 
496   bool hasFPModifiers() const {
497     return getModifiers().hasFPModifiers();
498   }
499 
500   bool hasIntModifiers() const {
501     return getModifiers().hasIntModifiers();
502   }
503 
504   void addImmOperands(MCInst &Inst, unsigned N, bool ApplyModifiers = true) const;
505 
506   void addLiteralImmOperand(MCInst &Inst, int64_t Val) const;
507 
508   template <unsigned Bitwidth>
509   void addKImmFPOperands(MCInst &Inst, unsigned N) const;
510 
511   void addKImmFP16Operands(MCInst &Inst, unsigned N) const {
512     addKImmFPOperands<16>(Inst, N);
513   }
514 
515   void addKImmFP32Operands(MCInst &Inst, unsigned N) const {
516     addKImmFPOperands<32>(Inst, N);
517   }
518 
519   void addRegOperands(MCInst &Inst, unsigned N) const;
520 
521   void addRegOrImmOperands(MCInst &Inst, unsigned N) const {
522     if (isRegKind())
523       addRegOperands(Inst, N);
524     else if (isExpr())
525       Inst.addOperand(MCOperand::createExpr(Expr));
526     else
527       addImmOperands(Inst, N);
528   }
529 
530   void addRegOrImmWithInputModsOperands(MCInst &Inst, unsigned N) const {
531     Modifiers Mods = getModifiers();
532     Inst.addOperand(MCOperand::createImm(Mods.getModifiersOperand()));
533     if (isRegKind()) {
534       addRegOperands(Inst, N);
535     } else {
536       addImmOperands(Inst, N, false);
537     }
538   }
539 
540   void addRegOrImmWithFPInputModsOperands(MCInst &Inst, unsigned N) const {
541     assert(!hasIntModifiers());
542     addRegOrImmWithInputModsOperands(Inst, N);
543   }
544 
545   void addRegOrImmWithIntInputModsOperands(MCInst &Inst, unsigned N) const {
546     assert(!hasFPModifiers());
547     addRegOrImmWithInputModsOperands(Inst, N);
548   }
549 
550   void addRegWithInputModsOperands(MCInst &Inst, unsigned N) const {
551     Modifiers Mods = getModifiers();
552     Inst.addOperand(MCOperand::createImm(Mods.getModifiersOperand()));
553     assert(isRegKind());
554     addRegOperands(Inst, N);
555   }
556 
557   void addRegWithFPInputModsOperands(MCInst &Inst, unsigned N) const {
558     assert(!hasIntModifiers());
559     addRegWithInputModsOperands(Inst, N);
560   }
561 
562   void addRegWithIntInputModsOperands(MCInst &Inst, unsigned N) const {
563     assert(!hasFPModifiers());
564     addRegWithInputModsOperands(Inst, N);
565   }
566 
567   void addSoppBrTargetOperands(MCInst &Inst, unsigned N) const {
568     if (isImm())
569       addImmOperands(Inst, N);
570     else {
571       assert(isExpr());
572       Inst.addOperand(MCOperand::createExpr(Expr));
573     }
574   }
575 
576   static void printImmTy(raw_ostream& OS, ImmTy Type) {
577     switch (Type) {
578     case ImmTyNone: OS << "None"; break;
579     case ImmTyGDS: OS << "GDS"; break;
580     case ImmTyOffen: OS << "Offen"; break;
581     case ImmTyIdxen: OS << "Idxen"; break;
582     case ImmTyAddr64: OS << "Addr64"; break;
583     case ImmTyOffset: OS << "Offset"; break;
584     case ImmTyOffset0: OS << "Offset0"; break;
585     case ImmTyOffset1: OS << "Offset1"; break;
586     case ImmTyGLC: OS << "GLC"; break;
587     case ImmTySLC: OS << "SLC"; break;
588     case ImmTyTFE: OS << "TFE"; break;
589     case ImmTyClampSI: OS << "ClampSI"; break;
590     case ImmTyOModSI: OS << "OModSI"; break;
591     case ImmTyDppCtrl: OS << "DppCtrl"; break;
592     case ImmTyDppRowMask: OS << "DppRowMask"; break;
593     case ImmTyDppBankMask: OS << "DppBankMask"; break;
594     case ImmTyDppBoundCtrl: OS << "DppBoundCtrl"; break;
595     case ImmTySdwaDstSel: OS << "SdwaDstSel"; break;
596     case ImmTySdwaSrc0Sel: OS << "SdwaSrc0Sel"; break;
597     case ImmTySdwaSrc1Sel: OS << "SdwaSrc1Sel"; break;
598     case ImmTySdwaDstUnused: OS << "SdwaDstUnused"; break;
599     case ImmTyDMask: OS << "DMask"; break;
600     case ImmTyUNorm: OS << "UNorm"; break;
601     case ImmTyDA: OS << "DA"; break;
602     case ImmTyR128: OS << "R128"; break;
603     case ImmTyLWE: OS << "LWE"; break;
604     case ImmTyOff: OS << "Off"; break;
605     case ImmTyExpTgt: OS << "ExpTgt"; break;
606     case ImmTyExpCompr: OS << "ExpCompr"; break;
607     case ImmTyExpVM: OS << "ExpVM"; break;
608     case ImmTyHwreg: OS << "Hwreg"; break;
609     case ImmTySendMsg: OS << "SendMsg"; break;
610     case ImmTyInterpSlot: OS << "InterpSlot"; break;
611     case ImmTyInterpAttr: OS << "InterpAttr"; break;
612     case ImmTyAttrChan: OS << "AttrChan"; break;
613     }
614   }
615 
616   void print(raw_ostream &OS) const override {
617     switch (Kind) {
618     case Register:
619       OS << "<register " << getReg() << " mods: " << Reg.Mods << '>';
620       break;
621     case Immediate:
622       OS << '<' << getImm();
623       if (getImmTy() != ImmTyNone) {
624         OS << " type: "; printImmTy(OS, getImmTy());
625       }
626       OS << " mods: " << Imm.Mods << '>';
627       break;
628     case Token:
629       OS << '\'' << getToken() << '\'';
630       break;
631     case Expression:
632       OS << "<expr " << *Expr << '>';
633       break;
634     }
635   }
636 
637   static AMDGPUOperand::Ptr CreateImm(const AMDGPUAsmParser *AsmParser,
638                                       int64_t Val, SMLoc Loc,
639                                       enum ImmTy Type = ImmTyNone,
640                                       bool IsFPImm = false) {
641     auto Op = llvm::make_unique<AMDGPUOperand>(Immediate, AsmParser);
642     Op->Imm.Val = Val;
643     Op->Imm.IsFPImm = IsFPImm;
644     Op->Imm.Type = Type;
645     Op->Imm.Mods = Modifiers();
646     Op->StartLoc = Loc;
647     Op->EndLoc = Loc;
648     return Op;
649   }
650 
651   static AMDGPUOperand::Ptr CreateToken(const AMDGPUAsmParser *AsmParser,
652                                         StringRef Str, SMLoc Loc,
653                                         bool HasExplicitEncodingSize = true) {
654     auto Res = llvm::make_unique<AMDGPUOperand>(Token, AsmParser);
655     Res->Tok.Data = Str.data();
656     Res->Tok.Length = Str.size();
657     Res->StartLoc = Loc;
658     Res->EndLoc = Loc;
659     return Res;
660   }
661 
662   static AMDGPUOperand::Ptr CreateReg(const AMDGPUAsmParser *AsmParser,
663                                       unsigned RegNo, SMLoc S,
664                                       SMLoc E,
665                                       bool ForceVOP3) {
666     auto Op = llvm::make_unique<AMDGPUOperand>(Register, AsmParser);
667     Op->Reg.RegNo = RegNo;
668     Op->Reg.Mods = Modifiers();
669     Op->Reg.IsForcedVOP3 = ForceVOP3;
670     Op->StartLoc = S;
671     Op->EndLoc = E;
672     return Op;
673   }
674 
675   static AMDGPUOperand::Ptr CreateExpr(const AMDGPUAsmParser *AsmParser,
676                                        const class MCExpr *Expr, SMLoc S) {
677     auto Op = llvm::make_unique<AMDGPUOperand>(Expression, AsmParser);
678     Op->Expr = Expr;
679     Op->StartLoc = S;
680     Op->EndLoc = S;
681     return Op;
682   }
683 };
684 
685 raw_ostream &operator <<(raw_ostream &OS, AMDGPUOperand::Modifiers Mods) {
686   OS << "abs:" << Mods.Abs << " neg: " << Mods.Neg << " sext:" << Mods.Sext;
687   return OS;
688 }
689 
690 //===----------------------------------------------------------------------===//
691 // AsmParser
692 //===----------------------------------------------------------------------===//
693 
694 // Holds info related to the current kernel, e.g. count of SGPRs used.
695 // Kernel scope begins at .amdgpu_hsa_kernel directive, ends at next
696 // .amdgpu_hsa_kernel or at EOF.
697 class KernelScopeInfo {
698   int SgprIndexUnusedMin;
699   int VgprIndexUnusedMin;
700   MCContext *Ctx;
701 
702   void usesSgprAt(int i) {
703     if (i >= SgprIndexUnusedMin) {
704       SgprIndexUnusedMin = ++i;
705       if (Ctx) {
706         MCSymbol * const Sym = Ctx->getOrCreateSymbol(Twine(".kernel.sgpr_count"));
707         Sym->setVariableValue(MCConstantExpr::create(SgprIndexUnusedMin, *Ctx));
708       }
709     }
710   }
711   void usesVgprAt(int i) {
712     if (i >= VgprIndexUnusedMin) {
713       VgprIndexUnusedMin = ++i;
714       if (Ctx) {
715         MCSymbol * const Sym = Ctx->getOrCreateSymbol(Twine(".kernel.vgpr_count"));
716         Sym->setVariableValue(MCConstantExpr::create(VgprIndexUnusedMin, *Ctx));
717       }
718     }
719   }
720 public:
721   KernelScopeInfo() : SgprIndexUnusedMin(-1), VgprIndexUnusedMin(-1), Ctx(nullptr)
722   {}
723   void initialize(MCContext &Context) {
724     Ctx = &Context;
725     usesSgprAt(SgprIndexUnusedMin = -1);
726     usesVgprAt(VgprIndexUnusedMin = -1);
727   }
728   void usesRegister(RegisterKind RegKind, unsigned DwordRegIndex, unsigned RegWidth) {
729     switch (RegKind) {
730       case IS_SGPR: usesSgprAt(DwordRegIndex + RegWidth - 1); break;
731       case IS_VGPR: usesVgprAt(DwordRegIndex + RegWidth - 1); break;
732       default: break;
733     }
734   }
735 };
736 
737 class AMDGPUAsmParser : public MCTargetAsmParser {
738   const MCInstrInfo &MII;
739   MCAsmParser &Parser;
740 
741   unsigned ForcedEncodingSize;
742   bool ForcedDPP;
743   bool ForcedSDWA;
744   KernelScopeInfo KernelScope;
745 
746   /// @name Auto-generated Match Functions
747   /// {
748 
749 #define GET_ASSEMBLER_HEADER
750 #include "AMDGPUGenAsmMatcher.inc"
751 
752   /// }
753 
754 private:
755   bool ParseAsAbsoluteExpression(uint32_t &Ret);
756   bool ParseDirectiveMajorMinor(uint32_t &Major, uint32_t &Minor);
757   bool ParseDirectiveHSACodeObjectVersion();
758   bool ParseDirectiveHSACodeObjectISA();
759   bool ParseDirectiveRuntimeMetadata();
760   bool ParseAMDKernelCodeTValue(StringRef ID, amd_kernel_code_t &Header);
761   bool ParseDirectiveAMDKernelCodeT();
762   bool ParseSectionDirectiveHSAText();
763   bool subtargetHasRegister(const MCRegisterInfo &MRI, unsigned RegNo) const;
764   bool ParseDirectiveAMDGPUHsaKernel();
765   bool ParseDirectiveAMDGPUHsaModuleGlobal();
766   bool ParseDirectiveAMDGPUHsaProgramGlobal();
767   bool ParseSectionDirectiveHSADataGlobalAgent();
768   bool ParseSectionDirectiveHSADataGlobalProgram();
769   bool ParseSectionDirectiveHSARodataReadonlyAgent();
770   bool AddNextRegisterToList(unsigned& Reg, unsigned& RegWidth, RegisterKind RegKind, unsigned Reg1, unsigned RegNum);
771   bool ParseAMDGPURegister(RegisterKind& RegKind, unsigned& Reg, unsigned& RegNum, unsigned& RegWidth, unsigned *DwordRegIndex);
772   void cvtMubufImpl(MCInst &Inst, const OperandVector &Operands, bool IsAtomic, bool IsAtomicReturn);
773 
774 public:
775   enum AMDGPUMatchResultTy {
776     Match_PreferE32 = FIRST_TARGET_MATCH_RESULT_TY
777   };
778 
779   AMDGPUAsmParser(const MCSubtargetInfo &STI, MCAsmParser &_Parser,
780                const MCInstrInfo &MII,
781                const MCTargetOptions &Options)
782       : MCTargetAsmParser(Options, STI), MII(MII), Parser(_Parser),
783         ForcedEncodingSize(0),
784         ForcedDPP(false),
785         ForcedSDWA(false) {
786     MCAsmParserExtension::Initialize(Parser);
787 
788     if (getSTI().getFeatureBits().none()) {
789       // Set default features.
790       copySTI().ToggleFeature("SOUTHERN_ISLANDS");
791     }
792 
793     setAvailableFeatures(ComputeAvailableFeatures(getSTI().getFeatureBits()));
794 
795     {
796       // TODO: make those pre-defined variables read-only.
797       // Currently there is none suitable machinery in the core llvm-mc for this.
798       // MCSymbol::isRedefinable is intended for another purpose, and
799       // AsmParser::parseDirectiveSet() cannot be specialized for specific target.
800       AMDGPU::IsaVersion Isa = AMDGPU::getIsaVersion(getSTI().getFeatureBits());
801       MCContext &Ctx = getContext();
802       MCSymbol *Sym = Ctx.getOrCreateSymbol(Twine(".option.machine_version_major"));
803       Sym->setVariableValue(MCConstantExpr::create(Isa.Major, Ctx));
804       Sym = Ctx.getOrCreateSymbol(Twine(".option.machine_version_minor"));
805       Sym->setVariableValue(MCConstantExpr::create(Isa.Minor, Ctx));
806       Sym = Ctx.getOrCreateSymbol(Twine(".option.machine_version_stepping"));
807       Sym->setVariableValue(MCConstantExpr::create(Isa.Stepping, Ctx));
808     }
809     KernelScope.initialize(getContext());
810   }
811 
812   bool isSI() const {
813     return AMDGPU::isSI(getSTI());
814   }
815 
816   bool isCI() const {
817     return AMDGPU::isCI(getSTI());
818   }
819 
820   bool isVI() const {
821     return AMDGPU::isVI(getSTI());
822   }
823 
824   bool hasInv2PiInlineImm() const {
825     return getSTI().getFeatureBits()[AMDGPU::FeatureInv2PiInlineImm];
826   }
827 
828   bool hasSGPR102_SGPR103() const {
829     return !isVI();
830   }
831 
832   AMDGPUTargetStreamer &getTargetStreamer() {
833     MCTargetStreamer &TS = *getParser().getStreamer().getTargetStreamer();
834     return static_cast<AMDGPUTargetStreamer &>(TS);
835   }
836 
837   const MCRegisterInfo *getMRI() const {
838     // We need this const_cast because for some reason getContext() is not const
839     // in MCAsmParser.
840     return const_cast<AMDGPUAsmParser*>(this)->getContext().getRegisterInfo();
841   }
842 
843   const MCInstrInfo *getMII() const {
844     return &MII;
845   }
846 
847   void setForcedEncodingSize(unsigned Size) { ForcedEncodingSize = Size; }
848   void setForcedDPP(bool ForceDPP_) { ForcedDPP = ForceDPP_; }
849   void setForcedSDWA(bool ForceSDWA_) { ForcedSDWA = ForceSDWA_; }
850 
851   unsigned getForcedEncodingSize() const { return ForcedEncodingSize; }
852   bool isForcedVOP3() const { return ForcedEncodingSize == 64; }
853   bool isForcedDPP() const { return ForcedDPP; }
854   bool isForcedSDWA() const { return ForcedSDWA; }
855   ArrayRef<unsigned> getMatchedVariants() const;
856 
857   std::unique_ptr<AMDGPUOperand> parseRegister();
858   bool ParseRegister(unsigned &RegNo, SMLoc &StartLoc, SMLoc &EndLoc) override;
859   unsigned checkTargetMatchPredicate(MCInst &Inst) override;
860   unsigned validateTargetOperandClass(MCParsedAsmOperand &Op,
861                                       unsigned Kind) override;
862   bool MatchAndEmitInstruction(SMLoc IDLoc, unsigned &Opcode,
863                                OperandVector &Operands, MCStreamer &Out,
864                                uint64_t &ErrorInfo,
865                                bool MatchingInlineAsm) override;
866   bool ParseDirective(AsmToken DirectiveID) override;
867   OperandMatchResultTy parseOperand(OperandVector &Operands, StringRef Mnemonic);
868   StringRef parseMnemonicSuffix(StringRef Name);
869   bool ParseInstruction(ParseInstructionInfo &Info, StringRef Name,
870                         SMLoc NameLoc, OperandVector &Operands) override;
871   //bool ProcessInstruction(MCInst &Inst);
872 
873   OperandMatchResultTy parseIntWithPrefix(const char *Prefix, int64_t &Int);
874   OperandMatchResultTy
875   parseIntWithPrefix(const char *Prefix, OperandVector &Operands,
876                      enum AMDGPUOperand::ImmTy ImmTy = AMDGPUOperand::ImmTyNone,
877                      bool (*ConvertResult)(int64_t &) = nullptr);
878   OperandMatchResultTy
879   parseNamedBit(const char *Name, OperandVector &Operands,
880                 enum AMDGPUOperand::ImmTy ImmTy = AMDGPUOperand::ImmTyNone);
881   OperandMatchResultTy parseStringWithPrefix(StringRef Prefix,
882                                              StringRef &Value);
883 
884   OperandMatchResultTy parseImm(OperandVector &Operands);
885   OperandMatchResultTy parseReg(OperandVector &Operands);
886   OperandMatchResultTy parseRegOrImm(OperandVector &Operands);
887   OperandMatchResultTy parseRegOrImmWithFPInputMods(OperandVector &Operands, bool AllowImm = true);
888   OperandMatchResultTy parseRegOrImmWithIntInputMods(OperandVector &Operands, bool AllowImm = true);
889   OperandMatchResultTy parseRegWithFPInputMods(OperandVector &Operands);
890   OperandMatchResultTy parseRegWithIntInputMods(OperandVector &Operands);
891   OperandMatchResultTy parseVReg32OrOff(OperandVector &Operands);
892 
893   void cvtDSOffset01(MCInst &Inst, const OperandVector &Operands);
894   void cvtDS(MCInst &Inst, const OperandVector &Operands);
895   void cvtExp(MCInst &Inst, const OperandVector &Operands);
896 
897   bool parseCnt(int64_t &IntVal);
898   OperandMatchResultTy parseSWaitCntOps(OperandVector &Operands);
899   OperandMatchResultTy parseHwreg(OperandVector &Operands);
900 
901 private:
902   struct OperandInfoTy {
903     int64_t Id;
904     bool IsSymbolic;
905     OperandInfoTy(int64_t Id_) : Id(Id_), IsSymbolic(false) { }
906   };
907 
908   bool parseSendMsgConstruct(OperandInfoTy &Msg, OperandInfoTy &Operation, int64_t &StreamId);
909   bool parseHwregConstruct(OperandInfoTy &HwReg, int64_t &Offset, int64_t &Width);
910 
911   void errorExpTgt();
912   OperandMatchResultTy parseExpTgtImpl(StringRef Str, uint8_t &Val);
913 
914 public:
915   OperandMatchResultTy parseOptionalOperand(OperandVector &Operands);
916 
917   OperandMatchResultTy parseExpTgt(OperandVector &Operands);
918   OperandMatchResultTy parseSendMsgOp(OperandVector &Operands);
919   OperandMatchResultTy parseInterpSlot(OperandVector &Operands);
920   OperandMatchResultTy parseInterpAttr(OperandVector &Operands);
921   OperandMatchResultTy parseSOppBrTarget(OperandVector &Operands);
922 
923   void cvtMubuf(MCInst &Inst, const OperandVector &Operands) { cvtMubufImpl(Inst, Operands, false, false); }
924   void cvtMubufAtomic(MCInst &Inst, const OperandVector &Operands) { cvtMubufImpl(Inst, Operands, true, false); }
925   void cvtMubufAtomicReturn(MCInst &Inst, const OperandVector &Operands) { cvtMubufImpl(Inst, Operands, true, true); }
926   AMDGPUOperand::Ptr defaultGLC() const;
927   AMDGPUOperand::Ptr defaultSLC() const;
928   AMDGPUOperand::Ptr defaultTFE() const;
929 
930   AMDGPUOperand::Ptr defaultDMask() const;
931   AMDGPUOperand::Ptr defaultUNorm() const;
932   AMDGPUOperand::Ptr defaultDA() const;
933   AMDGPUOperand::Ptr defaultR128() const;
934   AMDGPUOperand::Ptr defaultLWE() const;
935   AMDGPUOperand::Ptr defaultSMRDOffset8() const;
936   AMDGPUOperand::Ptr defaultSMRDOffset20() const;
937   AMDGPUOperand::Ptr defaultSMRDLiteralOffset() const;
938 
939   OperandMatchResultTy parseOModOperand(OperandVector &Operands);
940 
941   void cvtId(MCInst &Inst, const OperandVector &Operands);
942   void cvtVOP3_2_mod(MCInst &Inst, const OperandVector &Operands);
943   void cvtVOP3(MCInst &Inst, const OperandVector &Operands);
944 
945   void cvtMIMG(MCInst &Inst, const OperandVector &Operands);
946   void cvtMIMGAtomic(MCInst &Inst, const OperandVector &Operands);
947 
948   OperandMatchResultTy parseDPPCtrl(OperandVector &Operands);
949   AMDGPUOperand::Ptr defaultRowMask() const;
950   AMDGPUOperand::Ptr defaultBankMask() const;
951   AMDGPUOperand::Ptr defaultBoundCtrl() const;
952   void cvtDPP(MCInst &Inst, const OperandVector &Operands);
953 
954   OperandMatchResultTy parseSDWASel(OperandVector &Operands, StringRef Prefix,
955                                     AMDGPUOperand::ImmTy Type);
956   OperandMatchResultTy parseSDWADstUnused(OperandVector &Operands);
957   void cvtSdwaVOP1(MCInst &Inst, const OperandVector &Operands);
958   void cvtSdwaVOP2(MCInst &Inst, const OperandVector &Operands);
959   void cvtSdwaVOPC(MCInst &Inst, const OperandVector &Operands);
960   void cvtSDWA(MCInst &Inst, const OperandVector &Operands,
961                uint64_t BasicInstType);
962 };
963 
964 struct OptionalOperand {
965   const char *Name;
966   AMDGPUOperand::ImmTy Type;
967   bool IsBit;
968   bool (*ConvertResult)(int64_t&);
969 };
970 
971 } // end anonymous namespace
972 
973 // May be called with integer type with equivalent bitwidth.
974 static const fltSemantics *getFltSemantics(unsigned Size) {
975   switch (Size) {
976   case 4:
977     return &APFloat::IEEEsingle();
978   case 8:
979     return &APFloat::IEEEdouble();
980   case 2:
981     return &APFloat::IEEEhalf();
982   default:
983     llvm_unreachable("unsupported fp type");
984   }
985 }
986 
987 static const fltSemantics *getFltSemantics(MVT VT) {
988   return getFltSemantics(VT.getSizeInBits() / 8);
989 }
990 
991 //===----------------------------------------------------------------------===//
992 // Operand
993 //===----------------------------------------------------------------------===//
994 
995 static bool canLosslesslyConvertToFPType(APFloat &FPLiteral, MVT VT) {
996   bool Lost;
997 
998   // Convert literal to single precision
999   APFloat::opStatus Status = FPLiteral.convert(*getFltSemantics(VT),
1000                                                APFloat::rmNearestTiesToEven,
1001                                                &Lost);
1002   // We allow precision lost but not overflow or underflow
1003   if (Status != APFloat::opOK &&
1004       Lost &&
1005       ((Status & APFloat::opOverflow)  != 0 ||
1006        (Status & APFloat::opUnderflow) != 0)) {
1007     return false;
1008   }
1009 
1010   return true;
1011 }
1012 
1013 bool AMDGPUOperand::isInlinableImm(MVT type) const {
1014   if (!isImmTy(ImmTyNone)) {
1015     // Only plain immediates are inlinable (e.g. "clamp" attribute is not)
1016     return false;
1017   }
1018   // TODO: We should avoid using host float here. It would be better to
1019   // check the float bit values which is what a few other places do.
1020   // We've had bot failures before due to weird NaN support on mips hosts.
1021 
1022   APInt Literal(64, Imm.Val);
1023 
1024   if (Imm.IsFPImm) { // We got fp literal token
1025     if (type == MVT::f64 || type == MVT::i64) { // Expected 64-bit operand
1026       return AMDGPU::isInlinableLiteral64(Imm.Val,
1027                                           AsmParser->hasInv2PiInlineImm());
1028     }
1029 
1030     APFloat FPLiteral(APFloat::IEEEdouble(), APInt(64, Imm.Val));
1031     if (!canLosslesslyConvertToFPType(FPLiteral, type))
1032       return false;
1033 
1034     // Check if single precision literal is inlinable
1035     return AMDGPU::isInlinableLiteral32(
1036       static_cast<int32_t>(FPLiteral.bitcastToAPInt().getZExtValue()),
1037       AsmParser->hasInv2PiInlineImm());
1038   }
1039 
1040 
1041   // We got int literal token.
1042   if (type == MVT::f64 || type == MVT::i64) { // Expected 64-bit operand
1043     return AMDGPU::isInlinableLiteral64(Imm.Val,
1044                                         AsmParser->hasInv2PiInlineImm());
1045   }
1046 
1047   if (type.getScalarSizeInBits() == 16) {
1048     return AMDGPU::isInlinableLiteral16(
1049       static_cast<int16_t>(Literal.getLoBits(16).getSExtValue()),
1050       AsmParser->hasInv2PiInlineImm());
1051   }
1052 
1053   return AMDGPU::isInlinableLiteral32(
1054     static_cast<int32_t>(Literal.getLoBits(32).getZExtValue()),
1055     AsmParser->hasInv2PiInlineImm());
1056 }
1057 
1058 bool AMDGPUOperand::isLiteralImm(MVT type) const {
1059   // Check that this imediate can be added as literal
1060   if (!isImmTy(ImmTyNone)) {
1061     return false;
1062   }
1063 
1064   if (!Imm.IsFPImm) {
1065     // We got int literal token.
1066 
1067     unsigned Size = type.getSizeInBits();
1068     if (Size == 64)
1069       Size = 32;
1070 
1071     // FIXME: 64-bit operands can zero extend, sign extend, or pad zeroes for FP
1072     // types.
1073     return isUIntN(Size, Imm.Val) || isIntN(Size, Imm.Val);
1074   }
1075 
1076   // We got fp literal token
1077   if (type == MVT::f64) { // Expected 64-bit fp operand
1078     // We would set low 64-bits of literal to zeroes but we accept this literals
1079     return true;
1080   }
1081 
1082   if (type == MVT::i64) { // Expected 64-bit int operand
1083     // We don't allow fp literals in 64-bit integer instructions. It is
1084     // unclear how we should encode them.
1085     return false;
1086   }
1087 
1088   APFloat FPLiteral(APFloat::IEEEdouble(), APInt(64, Imm.Val));
1089   return canLosslesslyConvertToFPType(FPLiteral, type);
1090 }
1091 
1092 bool AMDGPUOperand::isRegClass(unsigned RCID) const {
1093   return isRegKind() && AsmParser->getMRI()->getRegClass(RCID).contains(getReg());
1094 }
1095 
1096 void AMDGPUOperand::addImmOperands(MCInst &Inst, unsigned N, bool ApplyModifiers) const {
1097   int64_t Val = Imm.Val;
1098   if (isImmTy(ImmTyNone) && ApplyModifiers && Imm.Mods.hasFPModifiers() && Imm.Mods.Neg) {
1099     // Apply modifiers to immediate value. Only negate can get here
1100     if (Imm.IsFPImm) {
1101       APFloat F(BitsToDouble(Val));
1102       F.changeSign();
1103       Val = F.bitcastToAPInt().getZExtValue();
1104     } else {
1105       Val = -Val;
1106     }
1107   }
1108 
1109   if (AMDGPU::isSISrcOperand(AsmParser->getMII()->get(Inst.getOpcode()),
1110                              Inst.getNumOperands())) {
1111     addLiteralImmOperand(Inst, Val);
1112   } else {
1113     Inst.addOperand(MCOperand::createImm(Val));
1114   }
1115 }
1116 
1117 void AMDGPUOperand::addLiteralImmOperand(MCInst &Inst, int64_t Val) const {
1118   const auto& InstDesc = AsmParser->getMII()->get(Inst.getOpcode());
1119   auto OpNum = Inst.getNumOperands();
1120   // Check that this operand accepts literals
1121   assert(AMDGPU::isSISrcOperand(InstDesc, OpNum));
1122 
1123   auto OpSize = AMDGPU::getOperandSize(InstDesc, OpNum); // expected operand size
1124 
1125   if (Imm.IsFPImm) { // We got fp literal token
1126     APInt Literal(64, Val);
1127 
1128     switch (OpSize) {
1129     case 8: {
1130       if (AMDGPU::isInlinableLiteral64(Literal.getZExtValue(),
1131                                        AsmParser->hasInv2PiInlineImm())) {
1132         Inst.addOperand(MCOperand::createImm(Literal.getZExtValue()));
1133         return;
1134       }
1135 
1136       // Non-inlineable
1137       if (AMDGPU::isSISrcFPOperand(InstDesc, OpNum)) { // Expected 64-bit fp operand
1138         // For fp operands we check if low 32 bits are zeros
1139         if (Literal.getLoBits(32) != 0) {
1140           const_cast<AMDGPUAsmParser *>(AsmParser)->Warning(Inst.getLoc(),
1141           "Can't encode literal as exact 64-bit floating-point operand. "
1142           "Low 32-bits will be set to zero");
1143         }
1144 
1145         Inst.addOperand(MCOperand::createImm(Literal.lshr(32).getZExtValue()));
1146         return;
1147       }
1148 
1149       // We don't allow fp literals in 64-bit integer instructions. It is
1150       // unclear how we should encode them. This case should be checked earlier
1151       // in predicate methods (isLiteralImm())
1152       llvm_unreachable("fp literal in 64-bit integer instruction.");
1153     }
1154     case 4:
1155     case 2: {
1156       bool lost;
1157       APFloat FPLiteral(APFloat::IEEEdouble(), Literal);
1158       // Convert literal to single precision
1159       FPLiteral.convert(*getFltSemantics(OpSize),
1160                         APFloat::rmNearestTiesToEven, &lost);
1161       // We allow precision lost but not overflow or underflow. This should be
1162       // checked earlier in isLiteralImm()
1163       Inst.addOperand(MCOperand::createImm(FPLiteral.bitcastToAPInt().getZExtValue()));
1164       return;
1165     }
1166     default:
1167       llvm_unreachable("invalid operand size");
1168     }
1169 
1170     return;
1171   }
1172 
1173    // We got int literal token.
1174   // Only sign extend inline immediates.
1175   // FIXME: No errors on truncation
1176   switch (OpSize) {
1177   case 4: {
1178     if (isInt<32>(Val) &&
1179         AMDGPU::isInlinableLiteral32(static_cast<int32_t>(Val),
1180                                      AsmParser->hasInv2PiInlineImm())) {
1181       Inst.addOperand(MCOperand::createImm(Val));
1182       return;
1183     }
1184 
1185     Inst.addOperand(MCOperand::createImm(Val & 0xffffffff));
1186     return;
1187   }
1188   case 8: {
1189     if (AMDGPU::isInlinableLiteral64(Val,
1190                                      AsmParser->hasInv2PiInlineImm())) {
1191       Inst.addOperand(MCOperand::createImm(Val));
1192       return;
1193     }
1194 
1195     Inst.addOperand(MCOperand::createImm(Lo_32(Val)));
1196     return;
1197   }
1198   case 2: {
1199     if (isInt<16>(Val) &&
1200         AMDGPU::isInlinableLiteral16(static_cast<int16_t>(Val),
1201                                      AsmParser->hasInv2PiInlineImm())) {
1202       Inst.addOperand(MCOperand::createImm(Val));
1203       return;
1204     }
1205 
1206     Inst.addOperand(MCOperand::createImm(Val & 0xffff));
1207     return;
1208   }
1209   default:
1210     llvm_unreachable("invalid operand size");
1211   }
1212 }
1213 
1214 template <unsigned Bitwidth>
1215 void AMDGPUOperand::addKImmFPOperands(MCInst &Inst, unsigned N) const {
1216   APInt Literal(64, Imm.Val);
1217 
1218   if (!Imm.IsFPImm) {
1219     // We got int literal token.
1220     Inst.addOperand(MCOperand::createImm(Literal.getLoBits(Bitwidth).getZExtValue()));
1221     return;
1222   }
1223 
1224   bool Lost;
1225   APFloat FPLiteral(APFloat::IEEEdouble(), Literal);
1226   FPLiteral.convert(*getFltSemantics(Bitwidth / 8),
1227                     APFloat::rmNearestTiesToEven, &Lost);
1228   Inst.addOperand(MCOperand::createImm(FPLiteral.bitcastToAPInt().getZExtValue()));
1229 }
1230 
1231 void AMDGPUOperand::addRegOperands(MCInst &Inst, unsigned N) const {
1232   Inst.addOperand(MCOperand::createReg(AMDGPU::getMCReg(getReg(), AsmParser->getSTI())));
1233 }
1234 
1235 //===----------------------------------------------------------------------===//
1236 // AsmParser
1237 //===----------------------------------------------------------------------===//
1238 
1239 static int getRegClass(RegisterKind Is, unsigned RegWidth) {
1240   if (Is == IS_VGPR) {
1241     switch (RegWidth) {
1242       default: return -1;
1243       case 1: return AMDGPU::VGPR_32RegClassID;
1244       case 2: return AMDGPU::VReg_64RegClassID;
1245       case 3: return AMDGPU::VReg_96RegClassID;
1246       case 4: return AMDGPU::VReg_128RegClassID;
1247       case 8: return AMDGPU::VReg_256RegClassID;
1248       case 16: return AMDGPU::VReg_512RegClassID;
1249     }
1250   } else if (Is == IS_TTMP) {
1251     switch (RegWidth) {
1252       default: return -1;
1253       case 1: return AMDGPU::TTMP_32RegClassID;
1254       case 2: return AMDGPU::TTMP_64RegClassID;
1255       case 4: return AMDGPU::TTMP_128RegClassID;
1256     }
1257   } else if (Is == IS_SGPR) {
1258     switch (RegWidth) {
1259       default: return -1;
1260       case 1: return AMDGPU::SGPR_32RegClassID;
1261       case 2: return AMDGPU::SGPR_64RegClassID;
1262       case 4: return AMDGPU::SGPR_128RegClassID;
1263       case 8: return AMDGPU::SReg_256RegClassID;
1264       case 16: return AMDGPU::SReg_512RegClassID;
1265     }
1266   }
1267   return -1;
1268 }
1269 
1270 static unsigned getSpecialRegForName(StringRef RegName) {
1271   return StringSwitch<unsigned>(RegName)
1272     .Case("exec", AMDGPU::EXEC)
1273     .Case("vcc", AMDGPU::VCC)
1274     .Case("flat_scratch", AMDGPU::FLAT_SCR)
1275     .Case("m0", AMDGPU::M0)
1276     .Case("scc", AMDGPU::SCC)
1277     .Case("tba", AMDGPU::TBA)
1278     .Case("tma", AMDGPU::TMA)
1279     .Case("flat_scratch_lo", AMDGPU::FLAT_SCR_LO)
1280     .Case("flat_scratch_hi", AMDGPU::FLAT_SCR_HI)
1281     .Case("vcc_lo", AMDGPU::VCC_LO)
1282     .Case("vcc_hi", AMDGPU::VCC_HI)
1283     .Case("exec_lo", AMDGPU::EXEC_LO)
1284     .Case("exec_hi", AMDGPU::EXEC_HI)
1285     .Case("tma_lo", AMDGPU::TMA_LO)
1286     .Case("tma_hi", AMDGPU::TMA_HI)
1287     .Case("tba_lo", AMDGPU::TBA_LO)
1288     .Case("tba_hi", AMDGPU::TBA_HI)
1289     .Default(0);
1290 }
1291 
1292 bool AMDGPUAsmParser::ParseRegister(unsigned &RegNo, SMLoc &StartLoc, SMLoc &EndLoc) {
1293   auto R = parseRegister();
1294   if (!R) return true;
1295   assert(R->isReg());
1296   RegNo = R->getReg();
1297   StartLoc = R->getStartLoc();
1298   EndLoc = R->getEndLoc();
1299   return false;
1300 }
1301 
1302 bool AMDGPUAsmParser::AddNextRegisterToList(unsigned& Reg, unsigned& RegWidth, RegisterKind RegKind, unsigned Reg1, unsigned RegNum)
1303 {
1304   switch (RegKind) {
1305   case IS_SPECIAL:
1306     if (Reg == AMDGPU::EXEC_LO && Reg1 == AMDGPU::EXEC_HI) { Reg = AMDGPU::EXEC; RegWidth = 2; return true; }
1307     if (Reg == AMDGPU::FLAT_SCR_LO && Reg1 == AMDGPU::FLAT_SCR_HI) { Reg = AMDGPU::FLAT_SCR; RegWidth = 2; return true; }
1308     if (Reg == AMDGPU::VCC_LO && Reg1 == AMDGPU::VCC_HI) { Reg = AMDGPU::VCC; RegWidth = 2; return true; }
1309     if (Reg == AMDGPU::TBA_LO && Reg1 == AMDGPU::TBA_HI) { Reg = AMDGPU::TBA; RegWidth = 2; return true; }
1310     if (Reg == AMDGPU::TMA_LO && Reg1 == AMDGPU::TMA_HI) { Reg = AMDGPU::TMA; RegWidth = 2; return true; }
1311     return false;
1312   case IS_VGPR:
1313   case IS_SGPR:
1314   case IS_TTMP:
1315     if (Reg1 != Reg + RegWidth) { return false; }
1316     RegWidth++;
1317     return true;
1318   default:
1319     llvm_unreachable("unexpected register kind");
1320   }
1321 }
1322 
1323 bool AMDGPUAsmParser::ParseAMDGPURegister(RegisterKind& RegKind, unsigned& Reg, unsigned& RegNum, unsigned& RegWidth, unsigned *DwordRegIndex)
1324 {
1325   if (DwordRegIndex) { *DwordRegIndex = 0; }
1326   const MCRegisterInfo *TRI = getContext().getRegisterInfo();
1327   if (getLexer().is(AsmToken::Identifier)) {
1328     StringRef RegName = Parser.getTok().getString();
1329     if ((Reg = getSpecialRegForName(RegName))) {
1330       Parser.Lex();
1331       RegKind = IS_SPECIAL;
1332     } else {
1333       unsigned RegNumIndex = 0;
1334       if (RegName[0] == 'v') {
1335         RegNumIndex = 1;
1336         RegKind = IS_VGPR;
1337       } else if (RegName[0] == 's') {
1338         RegNumIndex = 1;
1339         RegKind = IS_SGPR;
1340       } else if (RegName.startswith("ttmp")) {
1341         RegNumIndex = strlen("ttmp");
1342         RegKind = IS_TTMP;
1343       } else {
1344         return false;
1345       }
1346       if (RegName.size() > RegNumIndex) {
1347         // Single 32-bit register: vXX.
1348         if (RegName.substr(RegNumIndex).getAsInteger(10, RegNum))
1349           return false;
1350         Parser.Lex();
1351         RegWidth = 1;
1352       } else {
1353         // Range of registers: v[XX:YY]. ":YY" is optional.
1354         Parser.Lex();
1355         int64_t RegLo, RegHi;
1356         if (getLexer().isNot(AsmToken::LBrac))
1357           return false;
1358         Parser.Lex();
1359 
1360         if (getParser().parseAbsoluteExpression(RegLo))
1361           return false;
1362 
1363         const bool isRBrace = getLexer().is(AsmToken::RBrac);
1364         if (!isRBrace && getLexer().isNot(AsmToken::Colon))
1365           return false;
1366         Parser.Lex();
1367 
1368         if (isRBrace) {
1369           RegHi = RegLo;
1370         } else {
1371           if (getParser().parseAbsoluteExpression(RegHi))
1372             return false;
1373 
1374           if (getLexer().isNot(AsmToken::RBrac))
1375             return false;
1376           Parser.Lex();
1377         }
1378         RegNum = (unsigned) RegLo;
1379         RegWidth = (RegHi - RegLo) + 1;
1380       }
1381     }
1382   } else if (getLexer().is(AsmToken::LBrac)) {
1383     // List of consecutive registers: [s0,s1,s2,s3]
1384     Parser.Lex();
1385     if (!ParseAMDGPURegister(RegKind, Reg, RegNum, RegWidth, nullptr))
1386       return false;
1387     if (RegWidth != 1)
1388       return false;
1389     RegisterKind RegKind1;
1390     unsigned Reg1, RegNum1, RegWidth1;
1391     do {
1392       if (getLexer().is(AsmToken::Comma)) {
1393         Parser.Lex();
1394       } else if (getLexer().is(AsmToken::RBrac)) {
1395         Parser.Lex();
1396         break;
1397       } else if (ParseAMDGPURegister(RegKind1, Reg1, RegNum1, RegWidth1, nullptr)) {
1398         if (RegWidth1 != 1) {
1399           return false;
1400         }
1401         if (RegKind1 != RegKind) {
1402           return false;
1403         }
1404         if (!AddNextRegisterToList(Reg, RegWidth, RegKind1, Reg1, RegNum1)) {
1405           return false;
1406         }
1407       } else {
1408         return false;
1409       }
1410     } while (true);
1411   } else {
1412     return false;
1413   }
1414   switch (RegKind) {
1415   case IS_SPECIAL:
1416     RegNum = 0;
1417     RegWidth = 1;
1418     break;
1419   case IS_VGPR:
1420   case IS_SGPR:
1421   case IS_TTMP:
1422   {
1423     unsigned Size = 1;
1424     if (RegKind == IS_SGPR || RegKind == IS_TTMP) {
1425       // SGPR and TTMP registers must be aligned. Max required alignment is 4 dwords.
1426       Size = std::min(RegWidth, 4u);
1427     }
1428     if (RegNum % Size != 0)
1429       return false;
1430     if (DwordRegIndex) { *DwordRegIndex = RegNum; }
1431     RegNum = RegNum / Size;
1432     int RCID = getRegClass(RegKind, RegWidth);
1433     if (RCID == -1)
1434       return false;
1435     const MCRegisterClass RC = TRI->getRegClass(RCID);
1436     if (RegNum >= RC.getNumRegs())
1437       return false;
1438     Reg = RC.getRegister(RegNum);
1439     break;
1440   }
1441 
1442   default:
1443     llvm_unreachable("unexpected register kind");
1444   }
1445 
1446   if (!subtargetHasRegister(*TRI, Reg))
1447     return false;
1448   return true;
1449 }
1450 
1451 std::unique_ptr<AMDGPUOperand> AMDGPUAsmParser::parseRegister() {
1452   const auto &Tok = Parser.getTok();
1453   SMLoc StartLoc = Tok.getLoc();
1454   SMLoc EndLoc = Tok.getEndLoc();
1455   RegisterKind RegKind;
1456   unsigned Reg, RegNum, RegWidth, DwordRegIndex;
1457 
1458   if (!ParseAMDGPURegister(RegKind, Reg, RegNum, RegWidth, &DwordRegIndex)) {
1459     return nullptr;
1460   }
1461   KernelScope.usesRegister(RegKind, DwordRegIndex, RegWidth);
1462   return AMDGPUOperand::CreateReg(this, Reg, StartLoc, EndLoc, false);
1463 }
1464 
1465 OperandMatchResultTy
1466 AMDGPUAsmParser::parseImm(OperandVector &Operands) {
1467   // TODO: add syntactic sugar for 1/(2*PI)
1468   bool Minus = false;
1469   if (getLexer().getKind() == AsmToken::Minus) {
1470     Minus = true;
1471     Parser.Lex();
1472   }
1473 
1474   SMLoc S = Parser.getTok().getLoc();
1475   switch(getLexer().getKind()) {
1476   case AsmToken::Integer: {
1477     int64_t IntVal;
1478     if (getParser().parseAbsoluteExpression(IntVal))
1479       return MatchOperand_ParseFail;
1480     if (Minus)
1481       IntVal *= -1;
1482     Operands.push_back(AMDGPUOperand::CreateImm(this, IntVal, S));
1483     return MatchOperand_Success;
1484   }
1485   case AsmToken::Real: {
1486     int64_t IntVal;
1487     if (getParser().parseAbsoluteExpression(IntVal))
1488       return MatchOperand_ParseFail;
1489 
1490     APFloat F(BitsToDouble(IntVal));
1491     if (Minus)
1492       F.changeSign();
1493     Operands.push_back(
1494         AMDGPUOperand::CreateImm(this, F.bitcastToAPInt().getZExtValue(), S,
1495                                  AMDGPUOperand::ImmTyNone, true));
1496     return MatchOperand_Success;
1497   }
1498   default:
1499     return Minus ? MatchOperand_ParseFail : MatchOperand_NoMatch;
1500   }
1501 }
1502 
1503 OperandMatchResultTy
1504 AMDGPUAsmParser::parseReg(OperandVector &Operands) {
1505   if (auto R = parseRegister()) {
1506     assert(R->isReg());
1507     R->Reg.IsForcedVOP3 = isForcedVOP3();
1508     Operands.push_back(std::move(R));
1509     return MatchOperand_Success;
1510   }
1511   return MatchOperand_NoMatch;
1512 }
1513 
1514 OperandMatchResultTy
1515 AMDGPUAsmParser::parseRegOrImm(OperandVector &Operands) {
1516   auto res = parseImm(Operands);
1517   if (res != MatchOperand_NoMatch) {
1518     return res;
1519   }
1520 
1521   return parseReg(Operands);
1522 }
1523 
1524 OperandMatchResultTy
1525 AMDGPUAsmParser::parseRegOrImmWithFPInputMods(OperandVector &Operands, bool AllowImm) {
1526   // XXX: During parsing we can't determine if minus sign means
1527   // negate-modifier or negative immediate value.
1528   // By default we suppose it is modifier.
1529   bool Negate = false, Abs = false, Abs2 = false;
1530 
1531   if (getLexer().getKind()== AsmToken::Minus) {
1532     Parser.Lex();
1533     Negate = true;
1534   }
1535 
1536   if (getLexer().getKind() == AsmToken::Identifier && Parser.getTok().getString() == "abs") {
1537     Parser.Lex();
1538     Abs2 = true;
1539     if (getLexer().isNot(AsmToken::LParen)) {
1540       Error(Parser.getTok().getLoc(), "expected left paren after abs");
1541       return MatchOperand_ParseFail;
1542     }
1543     Parser.Lex();
1544   }
1545 
1546   if (getLexer().getKind() == AsmToken::Pipe) {
1547     if (Abs2) {
1548       Error(Parser.getTok().getLoc(), "expected register or immediate");
1549       return MatchOperand_ParseFail;
1550     }
1551     Parser.Lex();
1552     Abs = true;
1553   }
1554 
1555   OperandMatchResultTy Res;
1556   if (AllowImm) {
1557     Res = parseRegOrImm(Operands);
1558   } else {
1559     Res = parseReg(Operands);
1560   }
1561   if (Res != MatchOperand_Success) {
1562     return Res;
1563   }
1564 
1565   AMDGPUOperand::Modifiers Mods;
1566   if (Negate) {
1567     Mods.Neg = true;
1568   }
1569   if (Abs) {
1570     if (getLexer().getKind() != AsmToken::Pipe) {
1571       Error(Parser.getTok().getLoc(), "expected vertical bar");
1572       return MatchOperand_ParseFail;
1573     }
1574     Parser.Lex();
1575     Mods.Abs = true;
1576   }
1577   if (Abs2) {
1578     if (getLexer().isNot(AsmToken::RParen)) {
1579       Error(Parser.getTok().getLoc(), "expected closing parentheses");
1580       return MatchOperand_ParseFail;
1581     }
1582     Parser.Lex();
1583     Mods.Abs = true;
1584   }
1585 
1586   if (Mods.hasFPModifiers()) {
1587     AMDGPUOperand &Op = static_cast<AMDGPUOperand &>(*Operands.back());
1588     Op.setModifiers(Mods);
1589   }
1590   return MatchOperand_Success;
1591 }
1592 
1593 OperandMatchResultTy
1594 AMDGPUAsmParser::parseRegOrImmWithIntInputMods(OperandVector &Operands, bool AllowImm) {
1595   bool Sext = false;
1596 
1597   if (getLexer().getKind() == AsmToken::Identifier && Parser.getTok().getString() == "sext") {
1598     Parser.Lex();
1599     Sext = true;
1600     if (getLexer().isNot(AsmToken::LParen)) {
1601       Error(Parser.getTok().getLoc(), "expected left paren after sext");
1602       return MatchOperand_ParseFail;
1603     }
1604     Parser.Lex();
1605   }
1606 
1607   OperandMatchResultTy Res;
1608   if (AllowImm) {
1609     Res = parseRegOrImm(Operands);
1610   } else {
1611     Res = parseReg(Operands);
1612   }
1613   if (Res != MatchOperand_Success) {
1614     return Res;
1615   }
1616 
1617   AMDGPUOperand::Modifiers Mods;
1618   if (Sext) {
1619     if (getLexer().isNot(AsmToken::RParen)) {
1620       Error(Parser.getTok().getLoc(), "expected closing parentheses");
1621       return MatchOperand_ParseFail;
1622     }
1623     Parser.Lex();
1624     Mods.Sext = true;
1625   }
1626 
1627   if (Mods.hasIntModifiers()) {
1628     AMDGPUOperand &Op = static_cast<AMDGPUOperand &>(*Operands.back());
1629     Op.setModifiers(Mods);
1630   }
1631 
1632   return MatchOperand_Success;
1633 }
1634 
1635 OperandMatchResultTy
1636 AMDGPUAsmParser::parseRegWithFPInputMods(OperandVector &Operands) {
1637   return parseRegOrImmWithFPInputMods(Operands, false);
1638 }
1639 
1640 OperandMatchResultTy
1641 AMDGPUAsmParser::parseRegWithIntInputMods(OperandVector &Operands) {
1642   return parseRegOrImmWithIntInputMods(Operands, false);
1643 }
1644 
1645 OperandMatchResultTy AMDGPUAsmParser::parseVReg32OrOff(OperandVector &Operands) {
1646   std::unique_ptr<AMDGPUOperand> Reg = parseRegister();
1647   if (Reg) {
1648     Operands.push_back(std::move(Reg));
1649     return MatchOperand_Success;
1650   }
1651 
1652   const AsmToken &Tok = Parser.getTok();
1653   if (Tok.getString() == "off") {
1654     Operands.push_back(AMDGPUOperand::CreateImm(this, 0, Tok.getLoc(),
1655                                                 AMDGPUOperand::ImmTyOff, false));
1656     Parser.Lex();
1657     return MatchOperand_Success;
1658   }
1659 
1660   return MatchOperand_NoMatch;
1661 }
1662 
1663 unsigned AMDGPUAsmParser::checkTargetMatchPredicate(MCInst &Inst) {
1664 
1665   uint64_t TSFlags = MII.get(Inst.getOpcode()).TSFlags;
1666 
1667   if ((getForcedEncodingSize() == 32 && (TSFlags & SIInstrFlags::VOP3)) ||
1668       (getForcedEncodingSize() == 64 && !(TSFlags & SIInstrFlags::VOP3)) ||
1669       (isForcedDPP() && !(TSFlags & SIInstrFlags::DPP)) ||
1670       (isForcedSDWA() && !(TSFlags & SIInstrFlags::SDWA)) )
1671     return Match_InvalidOperand;
1672 
1673   if ((TSFlags & SIInstrFlags::VOP3) &&
1674       (TSFlags & SIInstrFlags::VOPAsmPrefer32Bit) &&
1675       getForcedEncodingSize() != 64)
1676     return Match_PreferE32;
1677 
1678   if (Inst.getOpcode() == AMDGPU::V_MAC_F32_sdwa_vi ||
1679       Inst.getOpcode() == AMDGPU::V_MAC_F16_sdwa_vi) {
1680     // v_mac_f32/16 allow only dst_sel == DWORD;
1681     auto OpNum =
1682         AMDGPU::getNamedOperandIdx(Inst.getOpcode(), AMDGPU::OpName::dst_sel);
1683     const auto &Op = Inst.getOperand(OpNum);
1684     if (!Op.isImm() || Op.getImm() != AMDGPU::SDWA::SdwaSel::DWORD) {
1685       return Match_InvalidOperand;
1686     }
1687   }
1688 
1689   return Match_Success;
1690 }
1691 
1692 // What asm variants we should check
1693 ArrayRef<unsigned> AMDGPUAsmParser::getMatchedVariants() const {
1694   if (getForcedEncodingSize() == 32) {
1695     static const unsigned Variants[] = {AMDGPUAsmVariants::DEFAULT};
1696     return makeArrayRef(Variants);
1697   }
1698 
1699   if (isForcedVOP3()) {
1700     static const unsigned Variants[] = {AMDGPUAsmVariants::VOP3};
1701     return makeArrayRef(Variants);
1702   }
1703 
1704   if (isForcedSDWA()) {
1705     static const unsigned Variants[] = {AMDGPUAsmVariants::SDWA};
1706     return makeArrayRef(Variants);
1707   }
1708 
1709   if (isForcedDPP()) {
1710     static const unsigned Variants[] = {AMDGPUAsmVariants::DPP};
1711     return makeArrayRef(Variants);
1712   }
1713 
1714   static const unsigned Variants[] = {
1715     AMDGPUAsmVariants::DEFAULT, AMDGPUAsmVariants::VOP3,
1716     AMDGPUAsmVariants::SDWA, AMDGPUAsmVariants::DPP
1717   };
1718 
1719   return makeArrayRef(Variants);
1720 }
1721 
1722 bool AMDGPUAsmParser::MatchAndEmitInstruction(SMLoc IDLoc, unsigned &Opcode,
1723                                               OperandVector &Operands,
1724                                               MCStreamer &Out,
1725                                               uint64_t &ErrorInfo,
1726                                               bool MatchingInlineAsm) {
1727   MCInst Inst;
1728   unsigned Result = Match_Success;
1729   for (auto Variant : getMatchedVariants()) {
1730     uint64_t EI;
1731     auto R = MatchInstructionImpl(Operands, Inst, EI, MatchingInlineAsm,
1732                                   Variant);
1733     // We order match statuses from least to most specific. We use most specific
1734     // status as resulting
1735     // Match_MnemonicFail < Match_InvalidOperand < Match_MissingFeature < Match_PreferE32
1736     if ((R == Match_Success) ||
1737         (R == Match_PreferE32) ||
1738         (R == Match_MissingFeature && Result != Match_PreferE32) ||
1739         (R == Match_InvalidOperand && Result != Match_MissingFeature
1740                                    && Result != Match_PreferE32) ||
1741         (R == Match_MnemonicFail   && Result != Match_InvalidOperand
1742                                    && Result != Match_MissingFeature
1743                                    && Result != Match_PreferE32)) {
1744       Result = R;
1745       ErrorInfo = EI;
1746     }
1747     if (R == Match_Success)
1748       break;
1749   }
1750 
1751   switch (Result) {
1752   default: break;
1753   case Match_Success:
1754     Inst.setLoc(IDLoc);
1755     Out.EmitInstruction(Inst, getSTI());
1756     return false;
1757 
1758   case Match_MissingFeature:
1759     return Error(IDLoc, "instruction not supported on this GPU");
1760 
1761   case Match_MnemonicFail:
1762     return Error(IDLoc, "unrecognized instruction mnemonic");
1763 
1764   case Match_InvalidOperand: {
1765     SMLoc ErrorLoc = IDLoc;
1766     if (ErrorInfo != ~0ULL) {
1767       if (ErrorInfo >= Operands.size()) {
1768         return Error(IDLoc, "too few operands for instruction");
1769       }
1770       ErrorLoc = ((AMDGPUOperand &)*Operands[ErrorInfo]).getStartLoc();
1771       if (ErrorLoc == SMLoc())
1772         ErrorLoc = IDLoc;
1773     }
1774     return Error(ErrorLoc, "invalid operand for instruction");
1775   }
1776 
1777   case Match_PreferE32:
1778     return Error(IDLoc, "internal error: instruction without _e64 suffix "
1779                         "should be encoded as e32");
1780   }
1781   llvm_unreachable("Implement any new match types added!");
1782 }
1783 
1784 bool AMDGPUAsmParser::ParseAsAbsoluteExpression(uint32_t &Ret) {
1785   int64_t Tmp = -1;
1786   if (getLexer().isNot(AsmToken::Integer) && getLexer().isNot(AsmToken::Identifier)) {
1787     return true;
1788   }
1789   if (getParser().parseAbsoluteExpression(Tmp)) {
1790     return true;
1791   }
1792   Ret = static_cast<uint32_t>(Tmp);
1793   return false;
1794 }
1795 
1796 
1797 bool AMDGPUAsmParser::ParseDirectiveMajorMinor(uint32_t &Major,
1798                                                uint32_t &Minor) {
1799   if (ParseAsAbsoluteExpression(Major))
1800     return TokError("invalid major version");
1801 
1802   if (getLexer().isNot(AsmToken::Comma))
1803     return TokError("minor version number required, comma expected");
1804   Lex();
1805 
1806   if (ParseAsAbsoluteExpression(Minor))
1807     return TokError("invalid minor version");
1808 
1809   return false;
1810 }
1811 
1812 bool AMDGPUAsmParser::ParseDirectiveHSACodeObjectVersion() {
1813 
1814   uint32_t Major;
1815   uint32_t Minor;
1816 
1817   if (ParseDirectiveMajorMinor(Major, Minor))
1818     return true;
1819 
1820   getTargetStreamer().EmitDirectiveHSACodeObjectVersion(Major, Minor);
1821   return false;
1822 }
1823 
1824 bool AMDGPUAsmParser::ParseDirectiveHSACodeObjectISA() {
1825   uint32_t Major;
1826   uint32_t Minor;
1827   uint32_t Stepping;
1828   StringRef VendorName;
1829   StringRef ArchName;
1830 
1831   // If this directive has no arguments, then use the ISA version for the
1832   // targeted GPU.
1833   if (getLexer().is(AsmToken::EndOfStatement)) {
1834     AMDGPU::IsaVersion Isa = AMDGPU::getIsaVersion(getSTI().getFeatureBits());
1835     getTargetStreamer().EmitDirectiveHSACodeObjectISA(Isa.Major, Isa.Minor,
1836                                                       Isa.Stepping,
1837                                                       "AMD", "AMDGPU");
1838     return false;
1839   }
1840 
1841   if (ParseDirectiveMajorMinor(Major, Minor))
1842     return true;
1843 
1844   if (getLexer().isNot(AsmToken::Comma))
1845     return TokError("stepping version number required, comma expected");
1846   Lex();
1847 
1848   if (ParseAsAbsoluteExpression(Stepping))
1849     return TokError("invalid stepping version");
1850 
1851   if (getLexer().isNot(AsmToken::Comma))
1852     return TokError("vendor name required, comma expected");
1853   Lex();
1854 
1855   if (getLexer().isNot(AsmToken::String))
1856     return TokError("invalid vendor name");
1857 
1858   VendorName = getLexer().getTok().getStringContents();
1859   Lex();
1860 
1861   if (getLexer().isNot(AsmToken::Comma))
1862     return TokError("arch name required, comma expected");
1863   Lex();
1864 
1865   if (getLexer().isNot(AsmToken::String))
1866     return TokError("invalid arch name");
1867 
1868   ArchName = getLexer().getTok().getStringContents();
1869   Lex();
1870 
1871   getTargetStreamer().EmitDirectiveHSACodeObjectISA(Major, Minor, Stepping,
1872                                                     VendorName, ArchName);
1873   return false;
1874 }
1875 
1876 bool AMDGPUAsmParser::ParseDirectiveRuntimeMetadata() {
1877   std::string Metadata;
1878   raw_string_ostream MS(Metadata);
1879 
1880   getLexer().setSkipSpace(false);
1881 
1882   bool FoundEnd = false;
1883   while (!getLexer().is(AsmToken::Eof)) {
1884     while (getLexer().is(AsmToken::Space)) {
1885       MS << ' ';
1886       Lex();
1887     }
1888 
1889     if (getLexer().is(AsmToken::Identifier)) {
1890       StringRef ID = getLexer().getTok().getIdentifier();
1891       if (ID == ".end_amdgpu_runtime_metadata") {
1892         Lex();
1893         FoundEnd = true;
1894         break;
1895       }
1896     }
1897 
1898     MS << Parser.parseStringToEndOfStatement()
1899        << getContext().getAsmInfo()->getSeparatorString();
1900 
1901     Parser.eatToEndOfStatement();
1902   }
1903 
1904   getLexer().setSkipSpace(true);
1905 
1906   if (getLexer().is(AsmToken::Eof) && !FoundEnd)
1907     return TokError("expected directive .end_amdgpu_runtime_metadata not found");
1908 
1909   MS.flush();
1910 
1911   getTargetStreamer().EmitRuntimeMetadata(Metadata);
1912 
1913   return false;
1914 }
1915 
1916 bool AMDGPUAsmParser::ParseAMDKernelCodeTValue(StringRef ID,
1917                                                amd_kernel_code_t &Header) {
1918   SmallString<40> ErrStr;
1919   raw_svector_ostream Err(ErrStr);
1920   if (!parseAmdKernelCodeField(ID, getParser(), Header, Err)) {
1921     return TokError(Err.str());
1922   }
1923   Lex();
1924   return false;
1925 }
1926 
1927 bool AMDGPUAsmParser::ParseDirectiveAMDKernelCodeT() {
1928   amd_kernel_code_t Header;
1929   AMDGPU::initDefaultAMDKernelCodeT(Header, getSTI().getFeatureBits());
1930 
1931   while (true) {
1932     // Lex EndOfStatement.  This is in a while loop, because lexing a comment
1933     // will set the current token to EndOfStatement.
1934     while(getLexer().is(AsmToken::EndOfStatement))
1935       Lex();
1936 
1937     if (getLexer().isNot(AsmToken::Identifier))
1938       return TokError("expected value identifier or .end_amd_kernel_code_t");
1939 
1940     StringRef ID = getLexer().getTok().getIdentifier();
1941     Lex();
1942 
1943     if (ID == ".end_amd_kernel_code_t")
1944       break;
1945 
1946     if (ParseAMDKernelCodeTValue(ID, Header))
1947       return true;
1948   }
1949 
1950   getTargetStreamer().EmitAMDKernelCodeT(Header);
1951 
1952   return false;
1953 }
1954 
1955 bool AMDGPUAsmParser::ParseSectionDirectiveHSAText() {
1956   getParser().getStreamer().SwitchSection(
1957       AMDGPU::getHSATextSection(getContext()));
1958   return false;
1959 }
1960 
1961 bool AMDGPUAsmParser::ParseDirectiveAMDGPUHsaKernel() {
1962   if (getLexer().isNot(AsmToken::Identifier))
1963     return TokError("expected symbol name");
1964 
1965   StringRef KernelName = Parser.getTok().getString();
1966 
1967   getTargetStreamer().EmitAMDGPUSymbolType(KernelName,
1968                                            ELF::STT_AMDGPU_HSA_KERNEL);
1969   Lex();
1970   KernelScope.initialize(getContext());
1971   return false;
1972 }
1973 
1974 bool AMDGPUAsmParser::ParseDirectiveAMDGPUHsaModuleGlobal() {
1975   if (getLexer().isNot(AsmToken::Identifier))
1976     return TokError("expected symbol name");
1977 
1978   StringRef GlobalName = Parser.getTok().getIdentifier();
1979 
1980   getTargetStreamer().EmitAMDGPUHsaModuleScopeGlobal(GlobalName);
1981   Lex();
1982   return false;
1983 }
1984 
1985 bool AMDGPUAsmParser::ParseDirectiveAMDGPUHsaProgramGlobal() {
1986   if (getLexer().isNot(AsmToken::Identifier))
1987     return TokError("expected symbol name");
1988 
1989   StringRef GlobalName = Parser.getTok().getIdentifier();
1990 
1991   getTargetStreamer().EmitAMDGPUHsaProgramScopeGlobal(GlobalName);
1992   Lex();
1993   return false;
1994 }
1995 
1996 bool AMDGPUAsmParser::ParseSectionDirectiveHSADataGlobalAgent() {
1997   getParser().getStreamer().SwitchSection(
1998       AMDGPU::getHSADataGlobalAgentSection(getContext()));
1999   return false;
2000 }
2001 
2002 bool AMDGPUAsmParser::ParseSectionDirectiveHSADataGlobalProgram() {
2003   getParser().getStreamer().SwitchSection(
2004       AMDGPU::getHSADataGlobalProgramSection(getContext()));
2005   return false;
2006 }
2007 
2008 bool AMDGPUAsmParser::ParseSectionDirectiveHSARodataReadonlyAgent() {
2009   getParser().getStreamer().SwitchSection(
2010       AMDGPU::getHSARodataReadonlyAgentSection(getContext()));
2011   return false;
2012 }
2013 
2014 bool AMDGPUAsmParser::ParseDirective(AsmToken DirectiveID) {
2015   StringRef IDVal = DirectiveID.getString();
2016 
2017   if (IDVal == ".hsa_code_object_version")
2018     return ParseDirectiveHSACodeObjectVersion();
2019 
2020   if (IDVal == ".hsa_code_object_isa")
2021     return ParseDirectiveHSACodeObjectISA();
2022 
2023   if (IDVal == ".amdgpu_runtime_metadata")
2024     return ParseDirectiveRuntimeMetadata();
2025 
2026   if (IDVal == ".amd_kernel_code_t")
2027     return ParseDirectiveAMDKernelCodeT();
2028 
2029   if (IDVal == ".hsatext")
2030     return ParseSectionDirectiveHSAText();
2031 
2032   if (IDVal == ".amdgpu_hsa_kernel")
2033     return ParseDirectiveAMDGPUHsaKernel();
2034 
2035   if (IDVal == ".amdgpu_hsa_module_global")
2036     return ParseDirectiveAMDGPUHsaModuleGlobal();
2037 
2038   if (IDVal == ".amdgpu_hsa_program_global")
2039     return ParseDirectiveAMDGPUHsaProgramGlobal();
2040 
2041   if (IDVal == ".hsadata_global_agent")
2042     return ParseSectionDirectiveHSADataGlobalAgent();
2043 
2044   if (IDVal == ".hsadata_global_program")
2045     return ParseSectionDirectiveHSADataGlobalProgram();
2046 
2047   if (IDVal == ".hsarodata_readonly_agent")
2048     return ParseSectionDirectiveHSARodataReadonlyAgent();
2049 
2050   return true;
2051 }
2052 
2053 bool AMDGPUAsmParser::subtargetHasRegister(const MCRegisterInfo &MRI,
2054                                            unsigned RegNo) const {
2055   if (isCI())
2056     return true;
2057 
2058   if (isSI()) {
2059     // No flat_scr
2060     switch (RegNo) {
2061     case AMDGPU::FLAT_SCR:
2062     case AMDGPU::FLAT_SCR_LO:
2063     case AMDGPU::FLAT_SCR_HI:
2064       return false;
2065     default:
2066       return true;
2067     }
2068   }
2069 
2070   // VI only has 102 SGPRs, so make sure we aren't trying to use the 2 more that
2071   // SI/CI have.
2072   for (MCRegAliasIterator R(AMDGPU::SGPR102_SGPR103, &MRI, true);
2073        R.isValid(); ++R) {
2074     if (*R == RegNo)
2075       return false;
2076   }
2077 
2078   return true;
2079 }
2080 
2081 OperandMatchResultTy
2082 AMDGPUAsmParser::parseOperand(OperandVector &Operands, StringRef Mnemonic) {
2083 
2084   // Try to parse with a custom parser
2085   OperandMatchResultTy ResTy = MatchOperandParserImpl(Operands, Mnemonic);
2086 
2087   // If we successfully parsed the operand or if there as an error parsing,
2088   // we are done.
2089   //
2090   // If we are parsing after we reach EndOfStatement then this means we
2091   // are appending default values to the Operands list.  This is only done
2092   // by custom parser, so we shouldn't continue on to the generic parsing.
2093   if (ResTy == MatchOperand_Success || ResTy == MatchOperand_ParseFail ||
2094       getLexer().is(AsmToken::EndOfStatement))
2095     return ResTy;
2096 
2097   ResTy = parseRegOrImm(Operands);
2098 
2099   if (ResTy == MatchOperand_Success)
2100     return ResTy;
2101 
2102   if (getLexer().getKind() == AsmToken::Identifier) {
2103     // If this identifier is a symbol, we want to create an expression for it.
2104     // It is a little difficult to distinguish between a symbol name, and
2105     // an instruction flag like 'gds'.  In order to do this, we parse
2106     // all tokens as expressions and then treate the symbol name as the token
2107     // string when we want to interpret the operand as a token.
2108     const auto &Tok = Parser.getTok();
2109     SMLoc S = Tok.getLoc();
2110     const MCExpr *Expr = nullptr;
2111     if (!Parser.parseExpression(Expr)) {
2112       Operands.push_back(AMDGPUOperand::CreateExpr(this, Expr, S));
2113       return MatchOperand_Success;
2114     }
2115 
2116     Operands.push_back(AMDGPUOperand::CreateToken(this, Tok.getString(), Tok.getLoc()));
2117     Parser.Lex();
2118     return MatchOperand_Success;
2119   }
2120   return MatchOperand_NoMatch;
2121 }
2122 
2123 StringRef AMDGPUAsmParser::parseMnemonicSuffix(StringRef Name) {
2124   // Clear any forced encodings from the previous instruction.
2125   setForcedEncodingSize(0);
2126   setForcedDPP(false);
2127   setForcedSDWA(false);
2128 
2129   if (Name.endswith("_e64")) {
2130     setForcedEncodingSize(64);
2131     return Name.substr(0, Name.size() - 4);
2132   } else if (Name.endswith("_e32")) {
2133     setForcedEncodingSize(32);
2134     return Name.substr(0, Name.size() - 4);
2135   } else if (Name.endswith("_dpp")) {
2136     setForcedDPP(true);
2137     return Name.substr(0, Name.size() - 4);
2138   } else if (Name.endswith("_sdwa")) {
2139     setForcedSDWA(true);
2140     return Name.substr(0, Name.size() - 5);
2141   }
2142   return Name;
2143 }
2144 
2145 bool AMDGPUAsmParser::ParseInstruction(ParseInstructionInfo &Info,
2146                                        StringRef Name,
2147                                        SMLoc NameLoc, OperandVector &Operands) {
2148   // Add the instruction mnemonic
2149   Name = parseMnemonicSuffix(Name);
2150   Operands.push_back(AMDGPUOperand::CreateToken(this, Name, NameLoc));
2151 
2152   while (!getLexer().is(AsmToken::EndOfStatement)) {
2153     OperandMatchResultTy Res = parseOperand(Operands, Name);
2154 
2155     // Eat the comma or space if there is one.
2156     if (getLexer().is(AsmToken::Comma))
2157       Parser.Lex();
2158 
2159     switch (Res) {
2160       case MatchOperand_Success: break;
2161       case MatchOperand_ParseFail:
2162         Error(getLexer().getLoc(), "failed parsing operand.");
2163         while (!getLexer().is(AsmToken::EndOfStatement)) {
2164           Parser.Lex();
2165         }
2166         return true;
2167       case MatchOperand_NoMatch:
2168         Error(getLexer().getLoc(), "not a valid operand.");
2169         while (!getLexer().is(AsmToken::EndOfStatement)) {
2170           Parser.Lex();
2171         }
2172         return true;
2173     }
2174   }
2175 
2176   return false;
2177 }
2178 
2179 //===----------------------------------------------------------------------===//
2180 // Utility functions
2181 //===----------------------------------------------------------------------===//
2182 
2183 OperandMatchResultTy
2184 AMDGPUAsmParser::parseIntWithPrefix(const char *Prefix, int64_t &Int) {
2185   switch(getLexer().getKind()) {
2186     default: return MatchOperand_NoMatch;
2187     case AsmToken::Identifier: {
2188       StringRef Name = Parser.getTok().getString();
2189       if (!Name.equals(Prefix)) {
2190         return MatchOperand_NoMatch;
2191       }
2192 
2193       Parser.Lex();
2194       if (getLexer().isNot(AsmToken::Colon))
2195         return MatchOperand_ParseFail;
2196 
2197       Parser.Lex();
2198       if (getLexer().isNot(AsmToken::Integer))
2199         return MatchOperand_ParseFail;
2200 
2201       if (getParser().parseAbsoluteExpression(Int))
2202         return MatchOperand_ParseFail;
2203       break;
2204     }
2205   }
2206   return MatchOperand_Success;
2207 }
2208 
2209 OperandMatchResultTy
2210 AMDGPUAsmParser::parseIntWithPrefix(const char *Prefix, OperandVector &Operands,
2211                                     enum AMDGPUOperand::ImmTy ImmTy,
2212                                     bool (*ConvertResult)(int64_t&)) {
2213   SMLoc S = Parser.getTok().getLoc();
2214   int64_t Value = 0;
2215 
2216   OperandMatchResultTy Res = parseIntWithPrefix(Prefix, Value);
2217   if (Res != MatchOperand_Success)
2218     return Res;
2219 
2220   if (ConvertResult && !ConvertResult(Value)) {
2221     return MatchOperand_ParseFail;
2222   }
2223 
2224   Operands.push_back(AMDGPUOperand::CreateImm(this, Value, S, ImmTy));
2225   return MatchOperand_Success;
2226 }
2227 
2228 OperandMatchResultTy
2229 AMDGPUAsmParser::parseNamedBit(const char *Name, OperandVector &Operands,
2230                                enum AMDGPUOperand::ImmTy ImmTy) {
2231   int64_t Bit = 0;
2232   SMLoc S = Parser.getTok().getLoc();
2233 
2234   // We are at the end of the statement, and this is a default argument, so
2235   // use a default value.
2236   if (getLexer().isNot(AsmToken::EndOfStatement)) {
2237     switch(getLexer().getKind()) {
2238       case AsmToken::Identifier: {
2239         StringRef Tok = Parser.getTok().getString();
2240         if (Tok == Name) {
2241           Bit = 1;
2242           Parser.Lex();
2243         } else if (Tok.startswith("no") && Tok.endswith(Name)) {
2244           Bit = 0;
2245           Parser.Lex();
2246         } else {
2247           return MatchOperand_NoMatch;
2248         }
2249         break;
2250       }
2251       default:
2252         return MatchOperand_NoMatch;
2253     }
2254   }
2255 
2256   Operands.push_back(AMDGPUOperand::CreateImm(this, Bit, S, ImmTy));
2257   return MatchOperand_Success;
2258 }
2259 
2260 typedef std::map<enum AMDGPUOperand::ImmTy, unsigned> OptionalImmIndexMap;
2261 
2262 void addOptionalImmOperand(MCInst& Inst, const OperandVector& Operands,
2263                            OptionalImmIndexMap& OptionalIdx,
2264                            enum AMDGPUOperand::ImmTy ImmT, int64_t Default = 0) {
2265   auto i = OptionalIdx.find(ImmT);
2266   if (i != OptionalIdx.end()) {
2267     unsigned Idx = i->second;
2268     ((AMDGPUOperand &)*Operands[Idx]).addImmOperands(Inst, 1);
2269   } else {
2270     Inst.addOperand(MCOperand::createImm(Default));
2271   }
2272 }
2273 
2274 OperandMatchResultTy
2275 AMDGPUAsmParser::parseStringWithPrefix(StringRef Prefix, StringRef &Value) {
2276   if (getLexer().isNot(AsmToken::Identifier)) {
2277     return MatchOperand_NoMatch;
2278   }
2279   StringRef Tok = Parser.getTok().getString();
2280   if (Tok != Prefix) {
2281     return MatchOperand_NoMatch;
2282   }
2283 
2284   Parser.Lex();
2285   if (getLexer().isNot(AsmToken::Colon)) {
2286     return MatchOperand_ParseFail;
2287   }
2288 
2289   Parser.Lex();
2290   if (getLexer().isNot(AsmToken::Identifier)) {
2291     return MatchOperand_ParseFail;
2292   }
2293 
2294   Value = Parser.getTok().getString();
2295   return MatchOperand_Success;
2296 }
2297 
2298 //===----------------------------------------------------------------------===//
2299 // ds
2300 //===----------------------------------------------------------------------===//
2301 
2302 void AMDGPUAsmParser::cvtDSOffset01(MCInst &Inst,
2303                                     const OperandVector &Operands) {
2304   OptionalImmIndexMap OptionalIdx;
2305 
2306   for (unsigned i = 1, e = Operands.size(); i != e; ++i) {
2307     AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[i]);
2308 
2309     // Add the register arguments
2310     if (Op.isReg()) {
2311       Op.addRegOperands(Inst, 1);
2312       continue;
2313     }
2314 
2315     // Handle optional arguments
2316     OptionalIdx[Op.getImmTy()] = i;
2317   }
2318 
2319   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyOffset0);
2320   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyOffset1);
2321   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyGDS);
2322 
2323   Inst.addOperand(MCOperand::createReg(AMDGPU::M0)); // m0
2324 }
2325 
2326 void AMDGPUAsmParser::cvtDS(MCInst &Inst, const OperandVector &Operands) {
2327   std::map<enum AMDGPUOperand::ImmTy, unsigned> OptionalIdx;
2328   bool GDSOnly = false;
2329 
2330   for (unsigned i = 1, e = Operands.size(); i != e; ++i) {
2331     AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[i]);
2332 
2333     // Add the register arguments
2334     if (Op.isReg()) {
2335       Op.addRegOperands(Inst, 1);
2336       continue;
2337     }
2338 
2339     if (Op.isToken() && Op.getToken() == "gds") {
2340       GDSOnly = true;
2341       continue;
2342     }
2343 
2344     // Handle optional arguments
2345     OptionalIdx[Op.getImmTy()] = i;
2346   }
2347 
2348   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyOffset);
2349   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyGDS);
2350 
2351   if (!GDSOnly) {
2352     addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyGDS);
2353   }
2354   Inst.addOperand(MCOperand::createReg(AMDGPU::M0)); // m0
2355 }
2356 
2357 void AMDGPUAsmParser::cvtExp(MCInst &Inst, const OperandVector &Operands) {
2358   OptionalImmIndexMap OptionalIdx;
2359 
2360   unsigned EnMask = 0;
2361   int SrcIdx = 0;
2362 
2363   for (unsigned i = 1, e = Operands.size(); i != e; ++i) {
2364     AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[i]);
2365 
2366     // Add the register arguments
2367     if (Op.isReg()) {
2368       EnMask |= (1 << SrcIdx);
2369       Op.addRegOperands(Inst, 1);
2370       ++SrcIdx;
2371       continue;
2372     }
2373 
2374     if (Op.isOff()) {
2375       ++SrcIdx;
2376       Inst.addOperand(MCOperand::createReg(AMDGPU::NoRegister));
2377       continue;
2378     }
2379 
2380     if (Op.isImm() && Op.getImmTy() == AMDGPUOperand::ImmTyExpTgt) {
2381       Op.addImmOperands(Inst, 1);
2382       continue;
2383     }
2384 
2385     if (Op.isToken() && Op.getToken() == "done")
2386       continue;
2387 
2388     // Handle optional arguments
2389     OptionalIdx[Op.getImmTy()] = i;
2390   }
2391 
2392   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyExpVM);
2393   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyExpCompr);
2394 
2395   Inst.addOperand(MCOperand::createImm(EnMask));
2396 }
2397 
2398 //===----------------------------------------------------------------------===//
2399 // s_waitcnt
2400 //===----------------------------------------------------------------------===//
2401 
2402 bool AMDGPUAsmParser::parseCnt(int64_t &IntVal) {
2403   StringRef CntName = Parser.getTok().getString();
2404   int64_t CntVal;
2405 
2406   Parser.Lex();
2407   if (getLexer().isNot(AsmToken::LParen))
2408     return true;
2409 
2410   Parser.Lex();
2411   if (getLexer().isNot(AsmToken::Integer))
2412     return true;
2413 
2414   if (getParser().parseAbsoluteExpression(CntVal))
2415     return true;
2416 
2417   if (getLexer().isNot(AsmToken::RParen))
2418     return true;
2419 
2420   Parser.Lex();
2421   if (getLexer().is(AsmToken::Amp) || getLexer().is(AsmToken::Comma))
2422     Parser.Lex();
2423 
2424   IsaVersion IV = getIsaVersion(getSTI().getFeatureBits());
2425   if (CntName == "vmcnt")
2426     IntVal = encodeVmcnt(IV, IntVal, CntVal);
2427   else if (CntName == "expcnt")
2428     IntVal = encodeExpcnt(IV, IntVal, CntVal);
2429   else if (CntName == "lgkmcnt")
2430     IntVal = encodeLgkmcnt(IV, IntVal, CntVal);
2431   else
2432     return true;
2433 
2434   return false;
2435 }
2436 
2437 OperandMatchResultTy
2438 AMDGPUAsmParser::parseSWaitCntOps(OperandVector &Operands) {
2439   IsaVersion IV = getIsaVersion(getSTI().getFeatureBits());
2440   int64_t Waitcnt = getWaitcntBitMask(IV);
2441   SMLoc S = Parser.getTok().getLoc();
2442 
2443   switch(getLexer().getKind()) {
2444     default: return MatchOperand_ParseFail;
2445     case AsmToken::Integer:
2446       // The operand can be an integer value.
2447       if (getParser().parseAbsoluteExpression(Waitcnt))
2448         return MatchOperand_ParseFail;
2449       break;
2450 
2451     case AsmToken::Identifier:
2452       do {
2453         if (parseCnt(Waitcnt))
2454           return MatchOperand_ParseFail;
2455       } while(getLexer().isNot(AsmToken::EndOfStatement));
2456       break;
2457   }
2458   Operands.push_back(AMDGPUOperand::CreateImm(this, Waitcnt, S));
2459   return MatchOperand_Success;
2460 }
2461 
2462 bool AMDGPUAsmParser::parseHwregConstruct(OperandInfoTy &HwReg, int64_t &Offset, int64_t &Width) {
2463   using namespace llvm::AMDGPU::Hwreg;
2464 
2465   if (Parser.getTok().getString() != "hwreg")
2466     return true;
2467   Parser.Lex();
2468 
2469   if (getLexer().isNot(AsmToken::LParen))
2470     return true;
2471   Parser.Lex();
2472 
2473   if (getLexer().is(AsmToken::Identifier)) {
2474     HwReg.IsSymbolic = true;
2475     HwReg.Id = ID_UNKNOWN_;
2476     const StringRef tok = Parser.getTok().getString();
2477     for (int i = ID_SYMBOLIC_FIRST_; i < ID_SYMBOLIC_LAST_; ++i) {
2478       if (tok == IdSymbolic[i]) {
2479         HwReg.Id = i;
2480         break;
2481       }
2482     }
2483     Parser.Lex();
2484   } else {
2485     HwReg.IsSymbolic = false;
2486     if (getLexer().isNot(AsmToken::Integer))
2487       return true;
2488     if (getParser().parseAbsoluteExpression(HwReg.Id))
2489       return true;
2490   }
2491 
2492   if (getLexer().is(AsmToken::RParen)) {
2493     Parser.Lex();
2494     return false;
2495   }
2496 
2497   // optional params
2498   if (getLexer().isNot(AsmToken::Comma))
2499     return true;
2500   Parser.Lex();
2501 
2502   if (getLexer().isNot(AsmToken::Integer))
2503     return true;
2504   if (getParser().parseAbsoluteExpression(Offset))
2505     return true;
2506 
2507   if (getLexer().isNot(AsmToken::Comma))
2508     return true;
2509   Parser.Lex();
2510 
2511   if (getLexer().isNot(AsmToken::Integer))
2512     return true;
2513   if (getParser().parseAbsoluteExpression(Width))
2514     return true;
2515 
2516   if (getLexer().isNot(AsmToken::RParen))
2517     return true;
2518   Parser.Lex();
2519 
2520   return false;
2521 }
2522 
2523 OperandMatchResultTy
2524 AMDGPUAsmParser::parseHwreg(OperandVector &Operands) {
2525   using namespace llvm::AMDGPU::Hwreg;
2526 
2527   int64_t Imm16Val = 0;
2528   SMLoc S = Parser.getTok().getLoc();
2529 
2530   switch(getLexer().getKind()) {
2531     default: return MatchOperand_NoMatch;
2532     case AsmToken::Integer:
2533       // The operand can be an integer value.
2534       if (getParser().parseAbsoluteExpression(Imm16Val))
2535         return MatchOperand_NoMatch;
2536       if (Imm16Val < 0 || !isUInt<16>(Imm16Val)) {
2537         Error(S, "invalid immediate: only 16-bit values are legal");
2538         // Do not return error code, but create an imm operand anyway and proceed
2539         // to the next operand, if any. That avoids unneccessary error messages.
2540       }
2541       break;
2542 
2543     case AsmToken::Identifier: {
2544         OperandInfoTy HwReg(ID_UNKNOWN_);
2545         int64_t Offset = OFFSET_DEFAULT_;
2546         int64_t Width = WIDTH_M1_DEFAULT_ + 1;
2547         if (parseHwregConstruct(HwReg, Offset, Width))
2548           return MatchOperand_ParseFail;
2549         if (HwReg.Id < 0 || !isUInt<ID_WIDTH_>(HwReg.Id)) {
2550           if (HwReg.IsSymbolic)
2551             Error(S, "invalid symbolic name of hardware register");
2552           else
2553             Error(S, "invalid code of hardware register: only 6-bit values are legal");
2554         }
2555         if (Offset < 0 || !isUInt<OFFSET_WIDTH_>(Offset))
2556           Error(S, "invalid bit offset: only 5-bit values are legal");
2557         if ((Width-1) < 0 || !isUInt<WIDTH_M1_WIDTH_>(Width-1))
2558           Error(S, "invalid bitfield width: only values from 1 to 32 are legal");
2559         Imm16Val = (HwReg.Id << ID_SHIFT_) | (Offset << OFFSET_SHIFT_) | ((Width-1) << WIDTH_M1_SHIFT_);
2560       }
2561       break;
2562   }
2563   Operands.push_back(AMDGPUOperand::CreateImm(this, Imm16Val, S, AMDGPUOperand::ImmTyHwreg));
2564   return MatchOperand_Success;
2565 }
2566 
2567 bool AMDGPUOperand::isSWaitCnt() const {
2568   return isImm();
2569 }
2570 
2571 bool AMDGPUOperand::isHwreg() const {
2572   return isImmTy(ImmTyHwreg);
2573 }
2574 
2575 bool AMDGPUAsmParser::parseSendMsgConstruct(OperandInfoTy &Msg, OperandInfoTy &Operation, int64_t &StreamId) {
2576   using namespace llvm::AMDGPU::SendMsg;
2577 
2578   if (Parser.getTok().getString() != "sendmsg")
2579     return true;
2580   Parser.Lex();
2581 
2582   if (getLexer().isNot(AsmToken::LParen))
2583     return true;
2584   Parser.Lex();
2585 
2586   if (getLexer().is(AsmToken::Identifier)) {
2587     Msg.IsSymbolic = true;
2588     Msg.Id = ID_UNKNOWN_;
2589     const std::string tok = Parser.getTok().getString();
2590     for (int i = ID_GAPS_FIRST_; i < ID_GAPS_LAST_; ++i) {
2591       switch(i) {
2592         default: continue; // Omit gaps.
2593         case ID_INTERRUPT: case ID_GS: case ID_GS_DONE:  case ID_SYSMSG: break;
2594       }
2595       if (tok == IdSymbolic[i]) {
2596         Msg.Id = i;
2597         break;
2598       }
2599     }
2600     Parser.Lex();
2601   } else {
2602     Msg.IsSymbolic = false;
2603     if (getLexer().isNot(AsmToken::Integer))
2604       return true;
2605     if (getParser().parseAbsoluteExpression(Msg.Id))
2606       return true;
2607     if (getLexer().is(AsmToken::Integer))
2608       if (getParser().parseAbsoluteExpression(Msg.Id))
2609         Msg.Id = ID_UNKNOWN_;
2610   }
2611   if (Msg.Id == ID_UNKNOWN_) // Don't know how to parse the rest.
2612     return false;
2613 
2614   if (!(Msg.Id == ID_GS || Msg.Id == ID_GS_DONE || Msg.Id == ID_SYSMSG)) {
2615     if (getLexer().isNot(AsmToken::RParen))
2616       return true;
2617     Parser.Lex();
2618     return false;
2619   }
2620 
2621   if (getLexer().isNot(AsmToken::Comma))
2622     return true;
2623   Parser.Lex();
2624 
2625   assert(Msg.Id == ID_GS || Msg.Id == ID_GS_DONE || Msg.Id == ID_SYSMSG);
2626   Operation.Id = ID_UNKNOWN_;
2627   if (getLexer().is(AsmToken::Identifier)) {
2628     Operation.IsSymbolic = true;
2629     const char* const *S = (Msg.Id == ID_SYSMSG) ? OpSysSymbolic : OpGsSymbolic;
2630     const int F = (Msg.Id == ID_SYSMSG) ? OP_SYS_FIRST_ : OP_GS_FIRST_;
2631     const int L = (Msg.Id == ID_SYSMSG) ? OP_SYS_LAST_ : OP_GS_LAST_;
2632     const StringRef Tok = Parser.getTok().getString();
2633     for (int i = F; i < L; ++i) {
2634       if (Tok == S[i]) {
2635         Operation.Id = i;
2636         break;
2637       }
2638     }
2639     Parser.Lex();
2640   } else {
2641     Operation.IsSymbolic = false;
2642     if (getLexer().isNot(AsmToken::Integer))
2643       return true;
2644     if (getParser().parseAbsoluteExpression(Operation.Id))
2645       return true;
2646   }
2647 
2648   if ((Msg.Id == ID_GS || Msg.Id == ID_GS_DONE) && Operation.Id != OP_GS_NOP) {
2649     // Stream id is optional.
2650     if (getLexer().is(AsmToken::RParen)) {
2651       Parser.Lex();
2652       return false;
2653     }
2654 
2655     if (getLexer().isNot(AsmToken::Comma))
2656       return true;
2657     Parser.Lex();
2658 
2659     if (getLexer().isNot(AsmToken::Integer))
2660       return true;
2661     if (getParser().parseAbsoluteExpression(StreamId))
2662       return true;
2663   }
2664 
2665   if (getLexer().isNot(AsmToken::RParen))
2666     return true;
2667   Parser.Lex();
2668   return false;
2669 }
2670 
2671 OperandMatchResultTy AMDGPUAsmParser::parseInterpSlot(OperandVector &Operands) {
2672   if (getLexer().getKind() != AsmToken::Identifier)
2673     return MatchOperand_NoMatch;
2674 
2675   StringRef Str = Parser.getTok().getString();
2676   int Slot = StringSwitch<int>(Str)
2677     .Case("p10", 0)
2678     .Case("p20", 1)
2679     .Case("p0", 2)
2680     .Default(-1);
2681 
2682   SMLoc S = Parser.getTok().getLoc();
2683   if (Slot == -1)
2684     return MatchOperand_ParseFail;
2685 
2686   Parser.Lex();
2687   Operands.push_back(AMDGPUOperand::CreateImm(this, Slot, S,
2688                                               AMDGPUOperand::ImmTyInterpSlot));
2689   return MatchOperand_Success;
2690 }
2691 
2692 OperandMatchResultTy AMDGPUAsmParser::parseInterpAttr(OperandVector &Operands) {
2693   if (getLexer().getKind() != AsmToken::Identifier)
2694     return MatchOperand_NoMatch;
2695 
2696   StringRef Str = Parser.getTok().getString();
2697   if (!Str.startswith("attr"))
2698     return MatchOperand_NoMatch;
2699 
2700   StringRef Chan = Str.take_back(2);
2701   int AttrChan = StringSwitch<int>(Chan)
2702     .Case(".x", 0)
2703     .Case(".y", 1)
2704     .Case(".z", 2)
2705     .Case(".w", 3)
2706     .Default(-1);
2707   if (AttrChan == -1)
2708     return MatchOperand_ParseFail;
2709 
2710   Str = Str.drop_back(2).drop_front(4);
2711 
2712   uint8_t Attr;
2713   if (Str.getAsInteger(10, Attr))
2714     return MatchOperand_ParseFail;
2715 
2716   SMLoc S = Parser.getTok().getLoc();
2717   Parser.Lex();
2718   if (Attr > 63) {
2719     Error(S, "out of bounds attr");
2720     return MatchOperand_Success;
2721   }
2722 
2723   SMLoc SChan = SMLoc::getFromPointer(Chan.data());
2724 
2725   Operands.push_back(AMDGPUOperand::CreateImm(this, Attr, S,
2726                                               AMDGPUOperand::ImmTyInterpAttr));
2727   Operands.push_back(AMDGPUOperand::CreateImm(this, AttrChan, SChan,
2728                                               AMDGPUOperand::ImmTyAttrChan));
2729   return MatchOperand_Success;
2730 }
2731 
2732 void AMDGPUAsmParser::errorExpTgt() {
2733   Error(Parser.getTok().getLoc(), "invalid exp target");
2734 }
2735 
2736 OperandMatchResultTy AMDGPUAsmParser::parseExpTgtImpl(StringRef Str,
2737                                                       uint8_t &Val) {
2738   if (Str == "null") {
2739     Val = 9;
2740     return MatchOperand_Success;
2741   }
2742 
2743   if (Str.startswith("mrt")) {
2744     Str = Str.drop_front(3);
2745     if (Str == "z") { // == mrtz
2746       Val = 8;
2747       return MatchOperand_Success;
2748     }
2749 
2750     if (Str.getAsInteger(10, Val))
2751       return MatchOperand_ParseFail;
2752 
2753     if (Val > 7)
2754       errorExpTgt();
2755 
2756     return MatchOperand_Success;
2757   }
2758 
2759   if (Str.startswith("pos")) {
2760     Str = Str.drop_front(3);
2761     if (Str.getAsInteger(10, Val))
2762       return MatchOperand_ParseFail;
2763 
2764     if (Val > 3)
2765       errorExpTgt();
2766 
2767     Val += 12;
2768     return MatchOperand_Success;
2769   }
2770 
2771   if (Str.startswith("param")) {
2772     Str = Str.drop_front(5);
2773     if (Str.getAsInteger(10, Val))
2774       return MatchOperand_ParseFail;
2775 
2776     if (Val >= 32)
2777       errorExpTgt();
2778 
2779     Val += 32;
2780     return MatchOperand_Success;
2781   }
2782 
2783   if (Str.startswith("invalid_target_")) {
2784     Str = Str.drop_front(15);
2785     if (Str.getAsInteger(10, Val))
2786       return MatchOperand_ParseFail;
2787 
2788     errorExpTgt();
2789     return MatchOperand_Success;
2790   }
2791 
2792   return MatchOperand_NoMatch;
2793 }
2794 
2795 OperandMatchResultTy AMDGPUAsmParser::parseExpTgt(OperandVector &Operands) {
2796   uint8_t Val;
2797   StringRef Str = Parser.getTok().getString();
2798 
2799   auto Res = parseExpTgtImpl(Str, Val);
2800   if (Res != MatchOperand_Success)
2801     return Res;
2802 
2803   SMLoc S = Parser.getTok().getLoc();
2804   Parser.Lex();
2805 
2806   Operands.push_back(AMDGPUOperand::CreateImm(this, Val, S,
2807                                               AMDGPUOperand::ImmTyExpTgt));
2808   return MatchOperand_Success;
2809 }
2810 
2811 OperandMatchResultTy
2812 AMDGPUAsmParser::parseSendMsgOp(OperandVector &Operands) {
2813   using namespace llvm::AMDGPU::SendMsg;
2814 
2815   int64_t Imm16Val = 0;
2816   SMLoc S = Parser.getTok().getLoc();
2817 
2818   switch(getLexer().getKind()) {
2819   default:
2820     return MatchOperand_NoMatch;
2821   case AsmToken::Integer:
2822     // The operand can be an integer value.
2823     if (getParser().parseAbsoluteExpression(Imm16Val))
2824       return MatchOperand_NoMatch;
2825     if (Imm16Val < 0 || !isUInt<16>(Imm16Val)) {
2826       Error(S, "invalid immediate: only 16-bit values are legal");
2827       // Do not return error code, but create an imm operand anyway and proceed
2828       // to the next operand, if any. That avoids unneccessary error messages.
2829     }
2830     break;
2831   case AsmToken::Identifier: {
2832       OperandInfoTy Msg(ID_UNKNOWN_);
2833       OperandInfoTy Operation(OP_UNKNOWN_);
2834       int64_t StreamId = STREAM_ID_DEFAULT_;
2835       if (parseSendMsgConstruct(Msg, Operation, StreamId))
2836         return MatchOperand_ParseFail;
2837       do {
2838         // Validate and encode message ID.
2839         if (! ((ID_INTERRUPT <= Msg.Id && Msg.Id <= ID_GS_DONE)
2840                 || Msg.Id == ID_SYSMSG)) {
2841           if (Msg.IsSymbolic)
2842             Error(S, "invalid/unsupported symbolic name of message");
2843           else
2844             Error(S, "invalid/unsupported code of message");
2845           break;
2846         }
2847         Imm16Val = (Msg.Id << ID_SHIFT_);
2848         // Validate and encode operation ID.
2849         if (Msg.Id == ID_GS || Msg.Id == ID_GS_DONE) {
2850           if (! (OP_GS_FIRST_ <= Operation.Id && Operation.Id < OP_GS_LAST_)) {
2851             if (Operation.IsSymbolic)
2852               Error(S, "invalid symbolic name of GS_OP");
2853             else
2854               Error(S, "invalid code of GS_OP: only 2-bit values are legal");
2855             break;
2856           }
2857           if (Operation.Id == OP_GS_NOP
2858               && Msg.Id != ID_GS_DONE) {
2859             Error(S, "invalid GS_OP: NOP is for GS_DONE only");
2860             break;
2861           }
2862           Imm16Val |= (Operation.Id << OP_SHIFT_);
2863         }
2864         if (Msg.Id == ID_SYSMSG) {
2865           if (! (OP_SYS_FIRST_ <= Operation.Id && Operation.Id < OP_SYS_LAST_)) {
2866             if (Operation.IsSymbolic)
2867               Error(S, "invalid/unsupported symbolic name of SYSMSG_OP");
2868             else
2869               Error(S, "invalid/unsupported code of SYSMSG_OP");
2870             break;
2871           }
2872           Imm16Val |= (Operation.Id << OP_SHIFT_);
2873         }
2874         // Validate and encode stream ID.
2875         if ((Msg.Id == ID_GS || Msg.Id == ID_GS_DONE) && Operation.Id != OP_GS_NOP) {
2876           if (! (STREAM_ID_FIRST_ <= StreamId && StreamId < STREAM_ID_LAST_)) {
2877             Error(S, "invalid stream id: only 2-bit values are legal");
2878             break;
2879           }
2880           Imm16Val |= (StreamId << STREAM_ID_SHIFT_);
2881         }
2882       } while (false);
2883     }
2884     break;
2885   }
2886   Operands.push_back(AMDGPUOperand::CreateImm(this, Imm16Val, S, AMDGPUOperand::ImmTySendMsg));
2887   return MatchOperand_Success;
2888 }
2889 
2890 bool AMDGPUOperand::isSendMsg() const {
2891   return isImmTy(ImmTySendMsg);
2892 }
2893 
2894 //===----------------------------------------------------------------------===//
2895 // sopp branch targets
2896 //===----------------------------------------------------------------------===//
2897 
2898 OperandMatchResultTy
2899 AMDGPUAsmParser::parseSOppBrTarget(OperandVector &Operands) {
2900   SMLoc S = Parser.getTok().getLoc();
2901 
2902   switch (getLexer().getKind()) {
2903     default: return MatchOperand_ParseFail;
2904     case AsmToken::Integer: {
2905       int64_t Imm;
2906       if (getParser().parseAbsoluteExpression(Imm))
2907         return MatchOperand_ParseFail;
2908       Operands.push_back(AMDGPUOperand::CreateImm(this, Imm, S));
2909       return MatchOperand_Success;
2910     }
2911 
2912     case AsmToken::Identifier:
2913       Operands.push_back(AMDGPUOperand::CreateExpr(this,
2914           MCSymbolRefExpr::create(getContext().getOrCreateSymbol(
2915                                   Parser.getTok().getString()), getContext()), S));
2916       Parser.Lex();
2917       return MatchOperand_Success;
2918   }
2919 }
2920 
2921 //===----------------------------------------------------------------------===//
2922 // mubuf
2923 //===----------------------------------------------------------------------===//
2924 
2925 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultGLC() const {
2926   return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTyGLC);
2927 }
2928 
2929 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultSLC() const {
2930   return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTySLC);
2931 }
2932 
2933 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultTFE() const {
2934   return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTyTFE);
2935 }
2936 
2937 void AMDGPUAsmParser::cvtMubufImpl(MCInst &Inst,
2938                                const OperandVector &Operands,
2939                                bool IsAtomic, bool IsAtomicReturn) {
2940   OptionalImmIndexMap OptionalIdx;
2941   assert(IsAtomicReturn ? IsAtomic : true);
2942 
2943   for (unsigned i = 1, e = Operands.size(); i != e; ++i) {
2944     AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[i]);
2945 
2946     // Add the register arguments
2947     if (Op.isReg()) {
2948       Op.addRegOperands(Inst, 1);
2949       continue;
2950     }
2951 
2952     // Handle the case where soffset is an immediate
2953     if (Op.isImm() && Op.getImmTy() == AMDGPUOperand::ImmTyNone) {
2954       Op.addImmOperands(Inst, 1);
2955       continue;
2956     }
2957 
2958     // Handle tokens like 'offen' which are sometimes hard-coded into the
2959     // asm string.  There are no MCInst operands for these.
2960     if (Op.isToken()) {
2961       continue;
2962     }
2963     assert(Op.isImm());
2964 
2965     // Handle optional arguments
2966     OptionalIdx[Op.getImmTy()] = i;
2967   }
2968 
2969   // Copy $vdata_in operand and insert as $vdata for MUBUF_Atomic RTN insns.
2970   if (IsAtomicReturn) {
2971     MCInst::iterator I = Inst.begin(); // $vdata_in is always at the beginning.
2972     Inst.insert(I, *I);
2973   }
2974 
2975   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyOffset);
2976   if (!IsAtomic) { // glc is hard-coded.
2977     addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyGLC);
2978   }
2979   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySLC);
2980   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyTFE);
2981 }
2982 
2983 //===----------------------------------------------------------------------===//
2984 // mimg
2985 //===----------------------------------------------------------------------===//
2986 
2987 void AMDGPUAsmParser::cvtMIMG(MCInst &Inst, const OperandVector &Operands) {
2988   unsigned I = 1;
2989   const MCInstrDesc &Desc = MII.get(Inst.getOpcode());
2990   for (unsigned J = 0; J < Desc.getNumDefs(); ++J) {
2991     ((AMDGPUOperand &)*Operands[I++]).addRegOperands(Inst, 1);
2992   }
2993 
2994   OptionalImmIndexMap OptionalIdx;
2995 
2996   for (unsigned E = Operands.size(); I != E; ++I) {
2997     AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[I]);
2998 
2999     // Add the register arguments
3000     if (Op.isRegOrImm()) {
3001       Op.addRegOrImmOperands(Inst, 1);
3002       continue;
3003     } else if (Op.isImmModifier()) {
3004       OptionalIdx[Op.getImmTy()] = I;
3005     } else {
3006       llvm_unreachable("unexpected operand type");
3007     }
3008   }
3009 
3010   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyDMask);
3011   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyUNorm);
3012   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyGLC);
3013   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyDA);
3014   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyR128);
3015   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyTFE);
3016   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyLWE);
3017   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySLC);
3018 }
3019 
3020 void AMDGPUAsmParser::cvtMIMGAtomic(MCInst &Inst, const OperandVector &Operands) {
3021   unsigned I = 1;
3022   const MCInstrDesc &Desc = MII.get(Inst.getOpcode());
3023   for (unsigned J = 0; J < Desc.getNumDefs(); ++J) {
3024     ((AMDGPUOperand &)*Operands[I++]).addRegOperands(Inst, 1);
3025   }
3026 
3027   // Add src, same as dst
3028   ((AMDGPUOperand &)*Operands[I]).addRegOperands(Inst, 1);
3029 
3030   OptionalImmIndexMap OptionalIdx;
3031 
3032   for (unsigned E = Operands.size(); I != E; ++I) {
3033     AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[I]);
3034 
3035     // Add the register arguments
3036     if (Op.isRegOrImm()) {
3037       Op.addRegOrImmOperands(Inst, 1);
3038       continue;
3039     } else if (Op.isImmModifier()) {
3040       OptionalIdx[Op.getImmTy()] = I;
3041     } else {
3042       llvm_unreachable("unexpected operand type");
3043     }
3044   }
3045 
3046   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyDMask);
3047   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyUNorm);
3048   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyGLC);
3049   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyDA);
3050   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyR128);
3051   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyTFE);
3052   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyLWE);
3053   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySLC);
3054 }
3055 
3056 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultDMask() const {
3057   return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTyDMask);
3058 }
3059 
3060 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultUNorm() const {
3061   return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTyUNorm);
3062 }
3063 
3064 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultDA() const {
3065   return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTyDA);
3066 }
3067 
3068 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultR128() const {
3069   return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTyR128);
3070 }
3071 
3072 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultLWE() const {
3073   return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTyLWE);
3074 }
3075 
3076 //===----------------------------------------------------------------------===//
3077 // smrd
3078 //===----------------------------------------------------------------------===//
3079 
3080 bool AMDGPUOperand::isSMRDOffset8() const {
3081   return isImm() && isUInt<8>(getImm());
3082 }
3083 
3084 bool AMDGPUOperand::isSMRDOffset20() const {
3085   return isImm() && isUInt<20>(getImm());
3086 }
3087 
3088 bool AMDGPUOperand::isSMRDLiteralOffset() const {
3089   // 32-bit literals are only supported on CI and we only want to use them
3090   // when the offset is > 8-bits.
3091   return isImm() && !isUInt<8>(getImm()) && isUInt<32>(getImm());
3092 }
3093 
3094 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultSMRDOffset8() const {
3095   return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTyOffset);
3096 }
3097 
3098 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultSMRDOffset20() const {
3099   return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTyOffset);
3100 }
3101 
3102 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultSMRDLiteralOffset() const {
3103   return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTyOffset);
3104 }
3105 
3106 //===----------------------------------------------------------------------===//
3107 // vop3
3108 //===----------------------------------------------------------------------===//
3109 
3110 static bool ConvertOmodMul(int64_t &Mul) {
3111   if (Mul != 1 && Mul != 2 && Mul != 4)
3112     return false;
3113 
3114   Mul >>= 1;
3115   return true;
3116 }
3117 
3118 static bool ConvertOmodDiv(int64_t &Div) {
3119   if (Div == 1) {
3120     Div = 0;
3121     return true;
3122   }
3123 
3124   if (Div == 2) {
3125     Div = 3;
3126     return true;
3127   }
3128 
3129   return false;
3130 }
3131 
3132 static bool ConvertBoundCtrl(int64_t &BoundCtrl) {
3133   if (BoundCtrl == 0) {
3134     BoundCtrl = 1;
3135     return true;
3136   }
3137 
3138   if (BoundCtrl == -1) {
3139     BoundCtrl = 0;
3140     return true;
3141   }
3142 
3143   return false;
3144 }
3145 
3146 // Note: the order in this table matches the order of operands in AsmString.
3147 static const OptionalOperand AMDGPUOptionalOperandTable[] = {
3148   {"offen",   AMDGPUOperand::ImmTyOffen, true, nullptr},
3149   {"idxen",   AMDGPUOperand::ImmTyIdxen, true, nullptr},
3150   {"addr64",  AMDGPUOperand::ImmTyAddr64, true, nullptr},
3151   {"offset0", AMDGPUOperand::ImmTyOffset0, false, nullptr},
3152   {"offset1", AMDGPUOperand::ImmTyOffset1, false, nullptr},
3153   {"gds",     AMDGPUOperand::ImmTyGDS, true, nullptr},
3154   {"offset",  AMDGPUOperand::ImmTyOffset, false, nullptr},
3155   {"glc",     AMDGPUOperand::ImmTyGLC, true, nullptr},
3156   {"slc",     AMDGPUOperand::ImmTySLC, true, nullptr},
3157   {"tfe",     AMDGPUOperand::ImmTyTFE, true, nullptr},
3158   {"clamp",   AMDGPUOperand::ImmTyClampSI, true, nullptr},
3159   {"omod",    AMDGPUOperand::ImmTyOModSI, false, ConvertOmodMul},
3160   {"unorm",   AMDGPUOperand::ImmTyUNorm, true, nullptr},
3161   {"da",      AMDGPUOperand::ImmTyDA,    true, nullptr},
3162   {"r128",    AMDGPUOperand::ImmTyR128,  true, nullptr},
3163   {"lwe",     AMDGPUOperand::ImmTyLWE,   true, nullptr},
3164   {"dmask",   AMDGPUOperand::ImmTyDMask, false, nullptr},
3165   {"row_mask",   AMDGPUOperand::ImmTyDppRowMask, false, nullptr},
3166   {"bank_mask",  AMDGPUOperand::ImmTyDppBankMask, false, nullptr},
3167   {"bound_ctrl", AMDGPUOperand::ImmTyDppBoundCtrl, false, ConvertBoundCtrl},
3168   {"dst_sel",    AMDGPUOperand::ImmTySdwaDstSel, false, nullptr},
3169   {"src0_sel",   AMDGPUOperand::ImmTySdwaSrc0Sel, false, nullptr},
3170   {"src1_sel",   AMDGPUOperand::ImmTySdwaSrc1Sel, false, nullptr},
3171   {"dst_unused", AMDGPUOperand::ImmTySdwaDstUnused, false, nullptr},
3172   {"vm", AMDGPUOperand::ImmTyExpVM, true, nullptr},
3173 };
3174 
3175 OperandMatchResultTy AMDGPUAsmParser::parseOptionalOperand(OperandVector &Operands) {
3176   OperandMatchResultTy res;
3177   for (const OptionalOperand &Op : AMDGPUOptionalOperandTable) {
3178     // try to parse any optional operand here
3179     if (Op.IsBit) {
3180       res = parseNamedBit(Op.Name, Operands, Op.Type);
3181     } else if (Op.Type == AMDGPUOperand::ImmTyOModSI) {
3182       res = parseOModOperand(Operands);
3183     } else if (Op.Type == AMDGPUOperand::ImmTySdwaDstSel ||
3184                Op.Type == AMDGPUOperand::ImmTySdwaSrc0Sel ||
3185                Op.Type == AMDGPUOperand::ImmTySdwaSrc1Sel) {
3186       res = parseSDWASel(Operands, Op.Name, Op.Type);
3187     } else if (Op.Type == AMDGPUOperand::ImmTySdwaDstUnused) {
3188       res = parseSDWADstUnused(Operands);
3189     } else {
3190       res = parseIntWithPrefix(Op.Name, Operands, Op.Type, Op.ConvertResult);
3191     }
3192     if (res != MatchOperand_NoMatch) {
3193       return res;
3194     }
3195   }
3196   return MatchOperand_NoMatch;
3197 }
3198 
3199 OperandMatchResultTy AMDGPUAsmParser::parseOModOperand(OperandVector &Operands) {
3200   StringRef Name = Parser.getTok().getString();
3201   if (Name == "mul") {
3202     return parseIntWithPrefix("mul", Operands,
3203                               AMDGPUOperand::ImmTyOModSI, ConvertOmodMul);
3204   }
3205 
3206   if (Name == "div") {
3207     return parseIntWithPrefix("div", Operands,
3208                               AMDGPUOperand::ImmTyOModSI, ConvertOmodDiv);
3209   }
3210 
3211   return MatchOperand_NoMatch;
3212 }
3213 
3214 void AMDGPUAsmParser::cvtId(MCInst &Inst, const OperandVector &Operands) {
3215   unsigned I = 1;
3216   const MCInstrDesc &Desc = MII.get(Inst.getOpcode());
3217   for (unsigned J = 0; J < Desc.getNumDefs(); ++J) {
3218     ((AMDGPUOperand &)*Operands[I++]).addRegOperands(Inst, 1);
3219   }
3220   for (unsigned E = Operands.size(); I != E; ++I)
3221     ((AMDGPUOperand &)*Operands[I]).addRegOrImmOperands(Inst, 1);
3222 }
3223 
3224 void AMDGPUAsmParser::cvtVOP3_2_mod(MCInst &Inst, const OperandVector &Operands) {
3225   uint64_t TSFlags = MII.get(Inst.getOpcode()).TSFlags;
3226   if (TSFlags & SIInstrFlags::VOP3) {
3227     cvtVOP3(Inst, Operands);
3228   } else {
3229     cvtId(Inst, Operands);
3230   }
3231 }
3232 
3233 static bool isRegOrImmWithInputMods(const MCInstrDesc &Desc, unsigned OpNum) {
3234       // 1. This operand is input modifiers
3235   return Desc.OpInfo[OpNum].OperandType == AMDGPU::OPERAND_INPUT_MODS
3236       // 2. This is not last operand
3237       && Desc.NumOperands > (OpNum + 1)
3238       // 3. Next operand is register class
3239       && Desc.OpInfo[OpNum + 1].RegClass != -1
3240       // 4. Next register is not tied to any other operand
3241       && Desc.getOperandConstraint(OpNum + 1, MCOI::OperandConstraint::TIED_TO) == -1;
3242 }
3243 
3244 void AMDGPUAsmParser::cvtVOP3(MCInst &Inst, const OperandVector &Operands) {
3245   OptionalImmIndexMap OptionalIdx;
3246   unsigned I = 1;
3247   const MCInstrDesc &Desc = MII.get(Inst.getOpcode());
3248   for (unsigned J = 0; J < Desc.getNumDefs(); ++J) {
3249     ((AMDGPUOperand &)*Operands[I++]).addRegOperands(Inst, 1);
3250   }
3251 
3252   for (unsigned E = Operands.size(); I != E; ++I) {
3253     AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[I]);
3254     if (isRegOrImmWithInputMods(Desc, Inst.getNumOperands())) {
3255       Op.addRegOrImmWithFPInputModsOperands(Inst, 2);
3256     } else if (Op.isImm()) {
3257       OptionalIdx[Op.getImmTy()] = I;
3258     } else {
3259       llvm_unreachable("unhandled operand type");
3260     }
3261   }
3262 
3263   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyClampSI);
3264   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyOModSI);
3265 
3266   // special case v_mac_{f16, f32}:
3267   // it has src2 register operand that is tied to dst operand
3268   // we don't allow modifiers for this operand in assembler so src2_modifiers
3269   // should be 0
3270   if (Inst.getOpcode() == AMDGPU::V_MAC_F32_e64_si ||
3271       Inst.getOpcode() == AMDGPU::V_MAC_F32_e64_vi ||
3272       Inst.getOpcode() == AMDGPU::V_MAC_F16_e64_vi) {
3273     auto it = Inst.begin();
3274     std::advance(
3275       it,
3276       AMDGPU::getNamedOperandIdx(Inst.getOpcode() == AMDGPU::V_MAC_F16_e64_vi ?
3277                                      AMDGPU::V_MAC_F16_e64 :
3278                                      AMDGPU::V_MAC_F32_e64,
3279                                  AMDGPU::OpName::src2_modifiers));
3280     it = Inst.insert(it, MCOperand::createImm(0)); // no modifiers for src2
3281     ++it;
3282     Inst.insert(it, Inst.getOperand(0)); // src2 = dst
3283   }
3284 }
3285 
3286 //===----------------------------------------------------------------------===//
3287 // dpp
3288 //===----------------------------------------------------------------------===//
3289 
3290 bool AMDGPUOperand::isDPPCtrl() const {
3291   bool result = isImm() && getImmTy() == ImmTyDppCtrl && isUInt<9>(getImm());
3292   if (result) {
3293     int64_t Imm = getImm();
3294     return ((Imm >= 0x000) && (Imm <= 0x0ff)) ||
3295            ((Imm >= 0x101) && (Imm <= 0x10f)) ||
3296            ((Imm >= 0x111) && (Imm <= 0x11f)) ||
3297            ((Imm >= 0x121) && (Imm <= 0x12f)) ||
3298            (Imm == 0x130) ||
3299            (Imm == 0x134) ||
3300            (Imm == 0x138) ||
3301            (Imm == 0x13c) ||
3302            (Imm == 0x140) ||
3303            (Imm == 0x141) ||
3304            (Imm == 0x142) ||
3305            (Imm == 0x143);
3306   }
3307   return false;
3308 }
3309 
3310 bool AMDGPUOperand::isGPRIdxMode() const {
3311   return isImm() && isUInt<4>(getImm());
3312 }
3313 
3314 OperandMatchResultTy
3315 AMDGPUAsmParser::parseDPPCtrl(OperandVector &Operands) {
3316   SMLoc S = Parser.getTok().getLoc();
3317   StringRef Prefix;
3318   int64_t Int;
3319 
3320   if (getLexer().getKind() == AsmToken::Identifier) {
3321     Prefix = Parser.getTok().getString();
3322   } else {
3323     return MatchOperand_NoMatch;
3324   }
3325 
3326   if (Prefix == "row_mirror") {
3327     Int = 0x140;
3328     Parser.Lex();
3329   } else if (Prefix == "row_half_mirror") {
3330     Int = 0x141;
3331     Parser.Lex();
3332   } else {
3333     // Check to prevent parseDPPCtrlOps from eating invalid tokens
3334     if (Prefix != "quad_perm"
3335         && Prefix != "row_shl"
3336         && Prefix != "row_shr"
3337         && Prefix != "row_ror"
3338         && Prefix != "wave_shl"
3339         && Prefix != "wave_rol"
3340         && Prefix != "wave_shr"
3341         && Prefix != "wave_ror"
3342         && Prefix != "row_bcast") {
3343       return MatchOperand_NoMatch;
3344     }
3345 
3346     Parser.Lex();
3347     if (getLexer().isNot(AsmToken::Colon))
3348       return MatchOperand_ParseFail;
3349 
3350     if (Prefix == "quad_perm") {
3351       // quad_perm:[%d,%d,%d,%d]
3352       Parser.Lex();
3353       if (getLexer().isNot(AsmToken::LBrac))
3354         return MatchOperand_ParseFail;
3355       Parser.Lex();
3356 
3357       if (getParser().parseAbsoluteExpression(Int) || !(0 <= Int && Int <=3))
3358         return MatchOperand_ParseFail;
3359 
3360       for (int i = 0; i < 3; ++i) {
3361         if (getLexer().isNot(AsmToken::Comma))
3362           return MatchOperand_ParseFail;
3363         Parser.Lex();
3364 
3365         int64_t Temp;
3366         if (getParser().parseAbsoluteExpression(Temp) || !(0 <= Temp && Temp <=3))
3367           return MatchOperand_ParseFail;
3368         const int shift = i*2 + 2;
3369         Int += (Temp << shift);
3370       }
3371 
3372       if (getLexer().isNot(AsmToken::RBrac))
3373         return MatchOperand_ParseFail;
3374       Parser.Lex();
3375 
3376     } else {
3377       // sel:%d
3378       Parser.Lex();
3379       if (getParser().parseAbsoluteExpression(Int))
3380         return MatchOperand_ParseFail;
3381 
3382       if (Prefix == "row_shl" && 1 <= Int && Int <= 15) {
3383         Int |= 0x100;
3384       } else if (Prefix == "row_shr" && 1 <= Int && Int <= 15) {
3385         Int |= 0x110;
3386       } else if (Prefix == "row_ror" && 1 <= Int && Int <= 15) {
3387         Int |= 0x120;
3388       } else if (Prefix == "wave_shl" && 1 == Int) {
3389         Int = 0x130;
3390       } else if (Prefix == "wave_rol" && 1 == Int) {
3391         Int = 0x134;
3392       } else if (Prefix == "wave_shr" && 1 == Int) {
3393         Int = 0x138;
3394       } else if (Prefix == "wave_ror" && 1 == Int) {
3395         Int = 0x13C;
3396       } else if (Prefix == "row_bcast") {
3397         if (Int == 15) {
3398           Int = 0x142;
3399         } else if (Int == 31) {
3400           Int = 0x143;
3401         } else {
3402           return MatchOperand_ParseFail;
3403         }
3404       } else {
3405         return MatchOperand_ParseFail;
3406       }
3407     }
3408   }
3409 
3410   Operands.push_back(AMDGPUOperand::CreateImm(this, Int, S, AMDGPUOperand::ImmTyDppCtrl));
3411   return MatchOperand_Success;
3412 }
3413 
3414 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultRowMask() const {
3415   return AMDGPUOperand::CreateImm(this, 0xf, SMLoc(), AMDGPUOperand::ImmTyDppRowMask);
3416 }
3417 
3418 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultBankMask() const {
3419   return AMDGPUOperand::CreateImm(this, 0xf, SMLoc(), AMDGPUOperand::ImmTyDppBankMask);
3420 }
3421 
3422 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultBoundCtrl() const {
3423   return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTyDppBoundCtrl);
3424 }
3425 
3426 void AMDGPUAsmParser::cvtDPP(MCInst &Inst, const OperandVector &Operands) {
3427   OptionalImmIndexMap OptionalIdx;
3428 
3429   unsigned I = 1;
3430   const MCInstrDesc &Desc = MII.get(Inst.getOpcode());
3431   for (unsigned J = 0; J < Desc.getNumDefs(); ++J) {
3432     ((AMDGPUOperand &)*Operands[I++]).addRegOperands(Inst, 1);
3433   }
3434 
3435   for (unsigned E = Operands.size(); I != E; ++I) {
3436     AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[I]);
3437     // Add the register arguments
3438     if (Op.isReg() && Op.Reg.RegNo == AMDGPU::VCC) {
3439       // VOP2b (v_add_u32, v_sub_u32 ...) sdwa use "vcc" token.
3440       // Skip it.
3441       continue;
3442     } if (isRegOrImmWithInputMods(Desc, Inst.getNumOperands())) {
3443       Op.addRegWithFPInputModsOperands(Inst, 2);
3444     } else if (Op.isDPPCtrl()) {
3445       Op.addImmOperands(Inst, 1);
3446     } else if (Op.isImm()) {
3447       // Handle optional arguments
3448       OptionalIdx[Op.getImmTy()] = I;
3449     } else {
3450       llvm_unreachable("Invalid operand type");
3451     }
3452   }
3453 
3454   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyDppRowMask, 0xf);
3455   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyDppBankMask, 0xf);
3456   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyDppBoundCtrl);
3457 
3458   // special case v_mac_{f16, f32}:
3459   // it has src2 register operand that is tied to dst operand
3460   if (Inst.getOpcode() == AMDGPU::V_MAC_F32_dpp ||
3461       Inst.getOpcode() == AMDGPU::V_MAC_F16_dpp) {
3462     auto it = Inst.begin();
3463     std::advance(
3464         it, AMDGPU::getNamedOperandIdx(Inst.getOpcode(), AMDGPU::OpName::src2));
3465     Inst.insert(it, Inst.getOperand(0)); // src2 = dst
3466   }
3467 }
3468 
3469 //===----------------------------------------------------------------------===//
3470 // sdwa
3471 //===----------------------------------------------------------------------===//
3472 
3473 OperandMatchResultTy
3474 AMDGPUAsmParser::parseSDWASel(OperandVector &Operands, StringRef Prefix,
3475                               AMDGPUOperand::ImmTy Type) {
3476   using namespace llvm::AMDGPU::SDWA;
3477 
3478   SMLoc S = Parser.getTok().getLoc();
3479   StringRef Value;
3480   OperandMatchResultTy res;
3481 
3482   res = parseStringWithPrefix(Prefix, Value);
3483   if (res != MatchOperand_Success) {
3484     return res;
3485   }
3486 
3487   int64_t Int;
3488   Int = StringSwitch<int64_t>(Value)
3489         .Case("BYTE_0", SdwaSel::BYTE_0)
3490         .Case("BYTE_1", SdwaSel::BYTE_1)
3491         .Case("BYTE_2", SdwaSel::BYTE_2)
3492         .Case("BYTE_3", SdwaSel::BYTE_3)
3493         .Case("WORD_0", SdwaSel::WORD_0)
3494         .Case("WORD_1", SdwaSel::WORD_1)
3495         .Case("DWORD", SdwaSel::DWORD)
3496         .Default(0xffffffff);
3497   Parser.Lex(); // eat last token
3498 
3499   if (Int == 0xffffffff) {
3500     return MatchOperand_ParseFail;
3501   }
3502 
3503   Operands.push_back(AMDGPUOperand::CreateImm(this, Int, S, Type));
3504   return MatchOperand_Success;
3505 }
3506 
3507 OperandMatchResultTy
3508 AMDGPUAsmParser::parseSDWADstUnused(OperandVector &Operands) {
3509   using namespace llvm::AMDGPU::SDWA;
3510 
3511   SMLoc S = Parser.getTok().getLoc();
3512   StringRef Value;
3513   OperandMatchResultTy res;
3514 
3515   res = parseStringWithPrefix("dst_unused", Value);
3516   if (res != MatchOperand_Success) {
3517     return res;
3518   }
3519 
3520   int64_t Int;
3521   Int = StringSwitch<int64_t>(Value)
3522         .Case("UNUSED_PAD", DstUnused::UNUSED_PAD)
3523         .Case("UNUSED_SEXT", DstUnused::UNUSED_SEXT)
3524         .Case("UNUSED_PRESERVE", DstUnused::UNUSED_PRESERVE)
3525         .Default(0xffffffff);
3526   Parser.Lex(); // eat last token
3527 
3528   if (Int == 0xffffffff) {
3529     return MatchOperand_ParseFail;
3530   }
3531 
3532   Operands.push_back(AMDGPUOperand::CreateImm(this, Int, S, AMDGPUOperand::ImmTySdwaDstUnused));
3533   return MatchOperand_Success;
3534 }
3535 
3536 void AMDGPUAsmParser::cvtSdwaVOP1(MCInst &Inst, const OperandVector &Operands) {
3537   cvtSDWA(Inst, Operands, SIInstrFlags::VOP1);
3538 }
3539 
3540 void AMDGPUAsmParser::cvtSdwaVOP2(MCInst &Inst, const OperandVector &Operands) {
3541   cvtSDWA(Inst, Operands, SIInstrFlags::VOP2);
3542 }
3543 
3544 void AMDGPUAsmParser::cvtSdwaVOPC(MCInst &Inst, const OperandVector &Operands) {
3545   cvtSDWA(Inst, Operands, SIInstrFlags::VOPC);
3546 }
3547 
3548 void AMDGPUAsmParser::cvtSDWA(MCInst &Inst, const OperandVector &Operands,
3549                               uint64_t BasicInstType) {
3550   OptionalImmIndexMap OptionalIdx;
3551 
3552   unsigned I = 1;
3553   const MCInstrDesc &Desc = MII.get(Inst.getOpcode());
3554   for (unsigned J = 0; J < Desc.getNumDefs(); ++J) {
3555     ((AMDGPUOperand &)*Operands[I++]).addRegOperands(Inst, 1);
3556   }
3557 
3558   for (unsigned E = Operands.size(); I != E; ++I) {
3559     AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[I]);
3560     // Add the register arguments
3561     if ((BasicInstType == SIInstrFlags::VOPC ||
3562          BasicInstType == SIInstrFlags::VOP2)&&
3563         Op.isReg() &&
3564         Op.Reg.RegNo == AMDGPU::VCC) {
3565       // VOPC and VOP2b (v_add_u32, v_sub_u32 ...) sdwa use "vcc" token as dst.
3566       // Skip it.
3567       continue;
3568     } else if (isRegOrImmWithInputMods(Desc, Inst.getNumOperands())) {
3569       Op.addRegWithInputModsOperands(Inst, 2);
3570     } else if (Op.isImm()) {
3571       // Handle optional arguments
3572       OptionalIdx[Op.getImmTy()] = I;
3573     } else {
3574       llvm_unreachable("Invalid operand type");
3575     }
3576   }
3577 
3578   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyClampSI, 0);
3579 
3580   if (Inst.getOpcode() != AMDGPU::V_NOP_sdwa_vi) {
3581     // V_NOP_sdwa_vi has no optional sdwa arguments
3582     switch (BasicInstType) {
3583     case SIInstrFlags::VOP1:
3584       addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySdwaDstSel, 6);
3585       addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySdwaDstUnused, 2);
3586       addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySdwaSrc0Sel, 6);
3587       break;
3588 
3589     case SIInstrFlags::VOP2:
3590       addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySdwaDstSel, 6);
3591       addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySdwaDstUnused, 2);
3592       addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySdwaSrc0Sel, 6);
3593       addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySdwaSrc1Sel, 6);
3594       break;
3595 
3596     case SIInstrFlags::VOPC:
3597       addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySdwaSrc0Sel, 6);
3598       addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySdwaSrc1Sel, 6);
3599       break;
3600 
3601     default:
3602       llvm_unreachable("Invalid instruction type. Only VOP1, VOP2 and VOPC allowed");
3603     }
3604   }
3605 
3606   // special case v_mac_{f16, f32}:
3607   // it has src2 register operand that is tied to dst operand
3608   if (Inst.getOpcode() == AMDGPU::V_MAC_F32_sdwa_vi ||
3609       Inst.getOpcode() == AMDGPU::V_MAC_F16_sdwa_vi)  {
3610     auto it = Inst.begin();
3611     std::advance(
3612         it, AMDGPU::getNamedOperandIdx(Inst.getOpcode(), AMDGPU::OpName::src2));
3613     Inst.insert(it, Inst.getOperand(0)); // src2 = dst
3614   }
3615 
3616 }
3617 
3618 /// Force static initialization.
3619 extern "C" void LLVMInitializeAMDGPUAsmParser() {
3620   RegisterMCAsmParser<AMDGPUAsmParser> A(getTheAMDGPUTarget());
3621   RegisterMCAsmParser<AMDGPUAsmParser> B(getTheGCNTarget());
3622 }
3623 
3624 #define GET_REGISTER_MATCHER
3625 #define GET_MATCHER_IMPLEMENTATION
3626 #include "AMDGPUGenAsmMatcher.inc"
3627 
3628 // This fuction should be defined after auto-generated include so that we have
3629 // MatchClassKind enum defined
3630 unsigned AMDGPUAsmParser::validateTargetOperandClass(MCParsedAsmOperand &Op,
3631                                                      unsigned Kind) {
3632   // Tokens like "glc" would be parsed as immediate operands in ParseOperand().
3633   // But MatchInstructionImpl() expects to meet token and fails to validate
3634   // operand. This method checks if we are given immediate operand but expect to
3635   // get corresponding token.
3636   AMDGPUOperand &Operand = (AMDGPUOperand&)Op;
3637   switch (Kind) {
3638   case MCK_addr64:
3639     return Operand.isAddr64() ? Match_Success : Match_InvalidOperand;
3640   case MCK_gds:
3641     return Operand.isGDS() ? Match_Success : Match_InvalidOperand;
3642   case MCK_glc:
3643     return Operand.isGLC() ? Match_Success : Match_InvalidOperand;
3644   case MCK_idxen:
3645     return Operand.isIdxen() ? Match_Success : Match_InvalidOperand;
3646   case MCK_offen:
3647     return Operand.isOffen() ? Match_Success : Match_InvalidOperand;
3648   case MCK_SSrcB32:
3649     // When operands have expression values, they will return true for isToken,
3650     // because it is not possible to distinguish between a token and an
3651     // expression at parse time. MatchInstructionImpl() will always try to
3652     // match an operand as a token, when isToken returns true, and when the
3653     // name of the expression is not a valid token, the match will fail,
3654     // so we need to handle it here.
3655     return Operand.isSSrcB32() ? Match_Success : Match_InvalidOperand;
3656   case MCK_SSrcF32:
3657     return Operand.isSSrcF32() ? Match_Success : Match_InvalidOperand;
3658   case MCK_SoppBrTarget:
3659     return Operand.isSoppBrTarget() ? Match_Success : Match_InvalidOperand;
3660   case MCK_VReg32OrOff:
3661     return Operand.isVReg32OrOff() ? Match_Success : Match_InvalidOperand;
3662   case MCK_InterpSlot:
3663     return Operand.isInterpSlot() ? Match_Success : Match_InvalidOperand;
3664   case MCK_Attr:
3665     return Operand.isInterpAttr() ? Match_Success : Match_InvalidOperand;
3666   case MCK_AttrChan:
3667     return Operand.isAttrChan() ? Match_Success : Match_InvalidOperand;
3668   default:
3669     return Match_InvalidOperand;
3670   }
3671 }
3672