1 //===-- AMDGPUAsmParser.cpp - Parse SI asm to MCInst instructions ---------===//
2 //
3 //                     The LLVM Compiler Infrastructure
4 //
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
7 //
8 //===----------------------------------------------------------------------===//
9 
10 #include "AMDKernelCodeT.h"
11 #include "MCTargetDesc/AMDGPUMCTargetDesc.h"
12 #include "MCTargetDesc/AMDGPUTargetStreamer.h"
13 #include "SIDefines.h"
14 #include "Utils/AMDGPUBaseInfo.h"
15 #include "Utils/AMDKernelCodeTUtils.h"
16 #include "Utils/AMDGPUAsmUtils.h"
17 #include "llvm/ADT/APFloat.h"
18 #include "llvm/ADT/APInt.h"
19 #include "llvm/ADT/ArrayRef.h"
20 #include "llvm/ADT/SmallBitVector.h"
21 #include "llvm/ADT/SmallString.h"
22 #include "llvm/ADT/STLExtras.h"
23 #include "llvm/ADT/StringRef.h"
24 #include "llvm/ADT/StringSwitch.h"
25 #include "llvm/ADT/Twine.h"
26 #include "llvm/CodeGen/MachineValueType.h"
27 #include "llvm/MC/MCAsmInfo.h"
28 #include "llvm/MC/MCContext.h"
29 #include "llvm/MC/MCExpr.h"
30 #include "llvm/MC/MCInst.h"
31 #include "llvm/MC/MCInstrDesc.h"
32 #include "llvm/MC/MCInstrInfo.h"
33 #include "llvm/MC/MCParser/MCAsmLexer.h"
34 #include "llvm/MC/MCParser/MCAsmParser.h"
35 #include "llvm/MC/MCParser/MCAsmParserExtension.h"
36 #include "llvm/MC/MCParser/MCParsedAsmOperand.h"
37 #include "llvm/MC/MCParser/MCTargetAsmParser.h"
38 #include "llvm/MC/MCRegisterInfo.h"
39 #include "llvm/MC/MCStreamer.h"
40 #include "llvm/MC/MCSubtargetInfo.h"
41 #include "llvm/MC/MCSymbol.h"
42 #include "llvm/Support/Casting.h"
43 #include "llvm/Support/ELF.h"
44 #include "llvm/Support/ErrorHandling.h"
45 #include "llvm/Support/MathExtras.h"
46 #include "llvm/Support/raw_ostream.h"
47 #include "llvm/Support/SMLoc.h"
48 #include "llvm/Support/TargetRegistry.h"
49 #include <algorithm>
50 #include <cassert>
51 #include <cstdint>
52 #include <cstring>
53 #include <iterator>
54 #include <map>
55 #include <memory>
56 #include <string>
57 
58 using namespace llvm;
59 using namespace llvm::AMDGPU;
60 
61 namespace {
62 
63 class AMDGPUAsmParser;
64 
65 enum RegisterKind { IS_UNKNOWN, IS_VGPR, IS_SGPR, IS_TTMP, IS_SPECIAL };
66 
67 //===----------------------------------------------------------------------===//
68 // Operand
69 //===----------------------------------------------------------------------===//
70 
71 class AMDGPUOperand : public MCParsedAsmOperand {
72   enum KindTy {
73     Token,
74     Immediate,
75     Register,
76     Expression
77   } Kind;
78 
79   SMLoc StartLoc, EndLoc;
80   const AMDGPUAsmParser *AsmParser;
81 
82 public:
83   AMDGPUOperand(KindTy Kind_, const AMDGPUAsmParser *AsmParser_)
84     : MCParsedAsmOperand(), Kind(Kind_), AsmParser(AsmParser_) {}
85 
86   typedef std::unique_ptr<AMDGPUOperand> Ptr;
87 
88   struct Modifiers {
89     bool Abs = false;
90     bool Neg = false;
91     bool Sext = false;
92 
93     bool hasFPModifiers() const { return Abs || Neg; }
94     bool hasIntModifiers() const { return Sext; }
95     bool hasModifiers() const { return hasFPModifiers() || hasIntModifiers(); }
96 
97     int64_t getFPModifiersOperand() const {
98       int64_t Operand = 0;
99       Operand |= Abs ? SISrcMods::ABS : 0;
100       Operand |= Neg ? SISrcMods::NEG : 0;
101       return Operand;
102     }
103 
104     int64_t getIntModifiersOperand() const {
105       int64_t Operand = 0;
106       Operand |= Sext ? SISrcMods::SEXT : 0;
107       return Operand;
108     }
109 
110     int64_t getModifiersOperand() const {
111       assert(!(hasFPModifiers() && hasIntModifiers())
112            && "fp and int modifiers should not be used simultaneously");
113       if (hasFPModifiers()) {
114         return getFPModifiersOperand();
115       } else if (hasIntModifiers()) {
116         return getIntModifiersOperand();
117       } else {
118         return 0;
119       }
120     }
121 
122     friend raw_ostream &operator <<(raw_ostream &OS, AMDGPUOperand::Modifiers Mods);
123   };
124 
125   enum ImmTy {
126     ImmTyNone,
127     ImmTyGDS,
128     ImmTyOffen,
129     ImmTyIdxen,
130     ImmTyAddr64,
131     ImmTyOffset,
132     ImmTyOffset0,
133     ImmTyOffset1,
134     ImmTyGLC,
135     ImmTySLC,
136     ImmTyTFE,
137     ImmTyClampSI,
138     ImmTyOModSI,
139     ImmTyDppCtrl,
140     ImmTyDppRowMask,
141     ImmTyDppBankMask,
142     ImmTyDppBoundCtrl,
143     ImmTySdwaDstSel,
144     ImmTySdwaSrc0Sel,
145     ImmTySdwaSrc1Sel,
146     ImmTySdwaDstUnused,
147     ImmTyDMask,
148     ImmTyUNorm,
149     ImmTyDA,
150     ImmTyR128,
151     ImmTyLWE,
152     ImmTyExpTgt,
153     ImmTyExpCompr,
154     ImmTyExpVM,
155     ImmTyHwreg,
156     ImmTyOff,
157     ImmTySendMsg,
158     ImmTyInterpSlot,
159     ImmTyInterpAttr,
160     ImmTyAttrChan
161   };
162 
163   struct TokOp {
164     const char *Data;
165     unsigned Length;
166   };
167 
168   struct ImmOp {
169     int64_t Val;
170     ImmTy Type;
171     bool IsFPImm;
172     Modifiers Mods;
173   };
174 
175   struct RegOp {
176     unsigned RegNo;
177     bool IsForcedVOP3;
178     Modifiers Mods;
179   };
180 
181   union {
182     TokOp Tok;
183     ImmOp Imm;
184     RegOp Reg;
185     const MCExpr *Expr;
186   };
187 
188   bool isToken() const override {
189     if (Kind == Token)
190       return true;
191 
192     if (Kind != Expression || !Expr)
193       return false;
194 
195     // When parsing operands, we can't always tell if something was meant to be
196     // a token, like 'gds', or an expression that references a global variable.
197     // In this case, we assume the string is an expression, and if we need to
198     // interpret is a token, then we treat the symbol name as the token.
199     return isa<MCSymbolRefExpr>(Expr);
200   }
201 
202   bool isImm() const override {
203     return Kind == Immediate;
204   }
205 
206   bool isInlinableImm(MVT type) const;
207   bool isLiteralImm(MVT type) const;
208 
209   bool isRegKind() const {
210     return Kind == Register;
211   }
212 
213   bool isReg() const override {
214     return isRegKind() && !hasModifiers();
215   }
216 
217   bool isRegOrImmWithInputMods(MVT type) const {
218     return isRegKind() || isInlinableImm(type);
219   }
220 
221   bool isRegOrImmWithInt16InputMods() const {
222     return isRegOrImmWithInputMods(MVT::i16);
223   }
224 
225   bool isRegOrImmWithInt32InputMods() const {
226     return isRegOrImmWithInputMods(MVT::i32);
227   }
228 
229   bool isRegOrImmWithInt64InputMods() const {
230     return isRegOrImmWithInputMods(MVT::i64);
231   }
232 
233   bool isRegOrImmWithFP16InputMods() const {
234     return isRegOrImmWithInputMods(MVT::f16);
235   }
236 
237   bool isRegOrImmWithFP32InputMods() const {
238     return isRegOrImmWithInputMods(MVT::f32);
239   }
240 
241   bool isRegOrImmWithFP64InputMods() const {
242     return isRegOrImmWithInputMods(MVT::f64);
243   }
244 
245   bool isVReg() const {
246     return isRegClass(AMDGPU::VGPR_32RegClassID) ||
247            isRegClass(AMDGPU::VReg_64RegClassID) ||
248            isRegClass(AMDGPU::VReg_96RegClassID) ||
249            isRegClass(AMDGPU::VReg_128RegClassID) ||
250            isRegClass(AMDGPU::VReg_256RegClassID) ||
251            isRegClass(AMDGPU::VReg_512RegClassID);
252   }
253 
254   bool isVReg32OrOff() const {
255     return isOff() || isRegClass(AMDGPU::VGPR_32RegClassID);
256   }
257 
258   bool isImmTy(ImmTy ImmT) const {
259     return isImm() && Imm.Type == ImmT;
260   }
261 
262   bool isImmModifier() const {
263     return isImm() && Imm.Type != ImmTyNone;
264   }
265 
266   bool isClampSI() const { return isImmTy(ImmTyClampSI); }
267   bool isOModSI() const { return isImmTy(ImmTyOModSI); }
268   bool isDMask() const { return isImmTy(ImmTyDMask); }
269   bool isUNorm() const { return isImmTy(ImmTyUNorm); }
270   bool isDA() const { return isImmTy(ImmTyDA); }
271   bool isR128() const { return isImmTy(ImmTyUNorm); }
272   bool isLWE() const { return isImmTy(ImmTyLWE); }
273   bool isOff() const { return isImmTy(ImmTyOff); }
274   bool isExpTgt() const { return isImmTy(ImmTyExpTgt); }
275   bool isExpVM() const { return isImmTy(ImmTyExpVM); }
276   bool isExpCompr() const { return isImmTy(ImmTyExpCompr); }
277   bool isOffen() const { return isImmTy(ImmTyOffen); }
278   bool isIdxen() const { return isImmTy(ImmTyIdxen); }
279   bool isAddr64() const { return isImmTy(ImmTyAddr64); }
280   bool isOffset() const { return isImmTy(ImmTyOffset) && isUInt<16>(getImm()); }
281   bool isOffset0() const { return isImmTy(ImmTyOffset0) && isUInt<16>(getImm()); }
282   bool isOffset1() const { return isImmTy(ImmTyOffset1) && isUInt<8>(getImm()); }
283   bool isGDS() const { return isImmTy(ImmTyGDS); }
284   bool isGLC() const { return isImmTy(ImmTyGLC); }
285   bool isSLC() const { return isImmTy(ImmTySLC); }
286   bool isTFE() const { return isImmTy(ImmTyTFE); }
287   bool isBankMask() const { return isImmTy(ImmTyDppBankMask); }
288   bool isRowMask() const { return isImmTy(ImmTyDppRowMask); }
289   bool isBoundCtrl() const { return isImmTy(ImmTyDppBoundCtrl); }
290   bool isSDWADstSel() const { return isImmTy(ImmTySdwaDstSel); }
291   bool isSDWASrc0Sel() const { return isImmTy(ImmTySdwaSrc0Sel); }
292   bool isSDWASrc1Sel() const { return isImmTy(ImmTySdwaSrc1Sel); }
293   bool isSDWADstUnused() const { return isImmTy(ImmTySdwaDstUnused); }
294   bool isInterpSlot() const { return isImmTy(ImmTyInterpSlot); }
295   bool isInterpAttr() const { return isImmTy(ImmTyInterpAttr); }
296   bool isAttrChan() const { return isImmTy(ImmTyAttrChan); }
297 
298   bool isMod() const {
299     return isClampSI() || isOModSI();
300   }
301 
302   bool isRegOrImm() const {
303     return isReg() || isImm();
304   }
305 
306   bool isRegClass(unsigned RCID) const;
307 
308   bool isRegOrInlineNoMods(unsigned RCID, MVT type) const {
309     return (isRegClass(RCID) || isInlinableImm(type)) && !hasModifiers();
310   }
311 
312   bool isSCSrcB16() const {
313     return isRegOrInlineNoMods(AMDGPU::SReg_32RegClassID, MVT::i16);
314   }
315 
316   bool isSCSrcB32() const {
317     return isRegOrInlineNoMods(AMDGPU::SReg_32RegClassID, MVT::i32);
318   }
319 
320   bool isSCSrcB64() const {
321     return isRegOrInlineNoMods(AMDGPU::SReg_64RegClassID, MVT::i64);
322   }
323 
324   bool isSCSrcF16() const {
325     return isRegOrInlineNoMods(AMDGPU::SReg_32RegClassID, MVT::f16);
326   }
327 
328   bool isSCSrcF32() const {
329     return isRegOrInlineNoMods(AMDGPU::SReg_32RegClassID, MVT::f32);
330   }
331 
332   bool isSCSrcF64() const {
333     return isRegOrInlineNoMods(AMDGPU::SReg_64RegClassID, MVT::f64);
334   }
335 
336   bool isSSrcB32() const {
337     return isSCSrcB32() || isLiteralImm(MVT::i32) || isExpr();
338   }
339 
340   bool isSSrcB16() const {
341     return isSCSrcB16() || isLiteralImm(MVT::i16);
342   }
343 
344   bool isSSrcB64() const {
345     // TODO: Find out how SALU supports extension of 32-bit literals to 64 bits.
346     // See isVSrc64().
347     return isSCSrcB64() || isLiteralImm(MVT::i64);
348   }
349 
350   bool isSSrcF32() const {
351     return isSCSrcB32() || isLiteralImm(MVT::f32) || isExpr();
352   }
353 
354   bool isSSrcF64() const {
355     return isSCSrcB64() || isLiteralImm(MVT::f64);
356   }
357 
358   bool isSSrcF16() const {
359     return isSCSrcB16() || isLiteralImm(MVT::f16);
360   }
361 
362   bool isVCSrcB32() const {
363     return isRegOrInlineNoMods(AMDGPU::VS_32RegClassID, MVT::i32);
364   }
365 
366   bool isVCSrcB64() const {
367     return isRegOrInlineNoMods(AMDGPU::VS_64RegClassID, MVT::i64);
368   }
369 
370   bool isVCSrcB16() const {
371     return isRegOrInlineNoMods(AMDGPU::VS_32RegClassID, MVT::i16);
372   }
373 
374   bool isVCSrcF32() const {
375     return isRegOrInlineNoMods(AMDGPU::VS_32RegClassID, MVT::f32);
376   }
377 
378   bool isVCSrcF64() const {
379     return isRegOrInlineNoMods(AMDGPU::VS_64RegClassID, MVT::f64);
380   }
381 
382   bool isVCSrcF16() const {
383     return isRegOrInlineNoMods(AMDGPU::VS_32RegClassID, MVT::f16);
384   }
385 
386   bool isVSrcB32() const {
387     return isVCSrcF32() || isLiteralImm(MVT::i32);
388   }
389 
390   bool isVSrcB64() const {
391     return isVCSrcF64() || isLiteralImm(MVT::i64);
392   }
393 
394   bool isVSrcB16() const {
395     return isVCSrcF16() || isLiteralImm(MVT::i16);
396   }
397 
398   bool isVSrcF32() const {
399     return isVCSrcF32() || isLiteralImm(MVT::f32);
400   }
401 
402   bool isVSrcF64() const {
403     return isVCSrcF64() || isLiteralImm(MVT::f64);
404   }
405 
406   bool isVSrcF16() const {
407     return isVCSrcF16() || isLiteralImm(MVT::f16);
408   }
409 
410   bool isKImmFP32() const {
411     return isLiteralImm(MVT::f32);
412   }
413 
414   bool isKImmFP16() const {
415     return isLiteralImm(MVT::f16);
416   }
417 
418   bool isMem() const override {
419     return false;
420   }
421 
422   bool isExpr() const {
423     return Kind == Expression;
424   }
425 
426   bool isSoppBrTarget() const {
427     return isExpr() || isImm();
428   }
429 
430   bool isSWaitCnt() const;
431   bool isHwreg() const;
432   bool isSendMsg() const;
433   bool isSMRDOffset8() const;
434   bool isSMRDOffset20() const;
435   bool isSMRDLiteralOffset() const;
436   bool isDPPCtrl() const;
437   bool isGPRIdxMode() const;
438 
439   StringRef getExpressionAsToken() const {
440     assert(isExpr());
441     const MCSymbolRefExpr *S = cast<MCSymbolRefExpr>(Expr);
442     return S->getSymbol().getName();
443   }
444 
445   StringRef getToken() const {
446     assert(isToken());
447 
448     if (Kind == Expression)
449       return getExpressionAsToken();
450 
451     return StringRef(Tok.Data, Tok.Length);
452   }
453 
454   int64_t getImm() const {
455     assert(isImm());
456     return Imm.Val;
457   }
458 
459   ImmTy getImmTy() const {
460     assert(isImm());
461     return Imm.Type;
462   }
463 
464   unsigned getReg() const override {
465     return Reg.RegNo;
466   }
467 
468   SMLoc getStartLoc() const override {
469     return StartLoc;
470   }
471 
472   SMLoc getEndLoc() const override {
473     return EndLoc;
474   }
475 
476   Modifiers getModifiers() const {
477     assert(isRegKind() || isImmTy(ImmTyNone));
478     return isRegKind() ? Reg.Mods : Imm.Mods;
479   }
480 
481   void setModifiers(Modifiers Mods) {
482     assert(isRegKind() || isImmTy(ImmTyNone));
483     if (isRegKind())
484       Reg.Mods = Mods;
485     else
486       Imm.Mods = Mods;
487   }
488 
489   bool hasModifiers() const {
490     return getModifiers().hasModifiers();
491   }
492 
493   bool hasFPModifiers() const {
494     return getModifiers().hasFPModifiers();
495   }
496 
497   bool hasIntModifiers() const {
498     return getModifiers().hasIntModifiers();
499   }
500 
501   void addImmOperands(MCInst &Inst, unsigned N, bool ApplyModifiers = true) const;
502 
503   void addLiteralImmOperand(MCInst &Inst, int64_t Val) const;
504 
505   template <unsigned Bitwidth>
506   void addKImmFPOperands(MCInst &Inst, unsigned N) const;
507 
508   void addKImmFP16Operands(MCInst &Inst, unsigned N) const {
509     addKImmFPOperands<16>(Inst, N);
510   }
511 
512   void addKImmFP32Operands(MCInst &Inst, unsigned N) const {
513     addKImmFPOperands<32>(Inst, N);
514   }
515 
516   void addRegOperands(MCInst &Inst, unsigned N) const;
517 
518   void addRegOrImmOperands(MCInst &Inst, unsigned N) const {
519     if (isRegKind())
520       addRegOperands(Inst, N);
521     else if (isExpr())
522       Inst.addOperand(MCOperand::createExpr(Expr));
523     else
524       addImmOperands(Inst, N);
525   }
526 
527   void addRegOrImmWithInputModsOperands(MCInst &Inst, unsigned N) const {
528     Modifiers Mods = getModifiers();
529     Inst.addOperand(MCOperand::createImm(Mods.getModifiersOperand()));
530     if (isRegKind()) {
531       addRegOperands(Inst, N);
532     } else {
533       addImmOperands(Inst, N, false);
534     }
535   }
536 
537   void addRegOrImmWithFPInputModsOperands(MCInst &Inst, unsigned N) const {
538     assert(!hasIntModifiers());
539     addRegOrImmWithInputModsOperands(Inst, N);
540   }
541 
542   void addRegOrImmWithIntInputModsOperands(MCInst &Inst, unsigned N) const {
543     assert(!hasFPModifiers());
544     addRegOrImmWithInputModsOperands(Inst, N);
545   }
546 
547   void addRegWithInputModsOperands(MCInst &Inst, unsigned N) const {
548     Modifiers Mods = getModifiers();
549     Inst.addOperand(MCOperand::createImm(Mods.getModifiersOperand()));
550     assert(isRegKind());
551     addRegOperands(Inst, N);
552   }
553 
554   void addRegWithFPInputModsOperands(MCInst &Inst, unsigned N) const {
555     assert(!hasIntModifiers());
556     addRegWithInputModsOperands(Inst, N);
557   }
558 
559   void addRegWithIntInputModsOperands(MCInst &Inst, unsigned N) const {
560     assert(!hasFPModifiers());
561     addRegWithInputModsOperands(Inst, N);
562   }
563 
564   void addSoppBrTargetOperands(MCInst &Inst, unsigned N) const {
565     if (isImm())
566       addImmOperands(Inst, N);
567     else {
568       assert(isExpr());
569       Inst.addOperand(MCOperand::createExpr(Expr));
570     }
571   }
572 
573   static void printImmTy(raw_ostream& OS, ImmTy Type) {
574     switch (Type) {
575     case ImmTyNone: OS << "None"; break;
576     case ImmTyGDS: OS << "GDS"; break;
577     case ImmTyOffen: OS << "Offen"; break;
578     case ImmTyIdxen: OS << "Idxen"; break;
579     case ImmTyAddr64: OS << "Addr64"; break;
580     case ImmTyOffset: OS << "Offset"; break;
581     case ImmTyOffset0: OS << "Offset0"; break;
582     case ImmTyOffset1: OS << "Offset1"; break;
583     case ImmTyGLC: OS << "GLC"; break;
584     case ImmTySLC: OS << "SLC"; break;
585     case ImmTyTFE: OS << "TFE"; break;
586     case ImmTyClampSI: OS << "ClampSI"; break;
587     case ImmTyOModSI: OS << "OModSI"; break;
588     case ImmTyDppCtrl: OS << "DppCtrl"; break;
589     case ImmTyDppRowMask: OS << "DppRowMask"; break;
590     case ImmTyDppBankMask: OS << "DppBankMask"; break;
591     case ImmTyDppBoundCtrl: OS << "DppBoundCtrl"; break;
592     case ImmTySdwaDstSel: OS << "SdwaDstSel"; break;
593     case ImmTySdwaSrc0Sel: OS << "SdwaSrc0Sel"; break;
594     case ImmTySdwaSrc1Sel: OS << "SdwaSrc1Sel"; break;
595     case ImmTySdwaDstUnused: OS << "SdwaDstUnused"; break;
596     case ImmTyDMask: OS << "DMask"; break;
597     case ImmTyUNorm: OS << "UNorm"; break;
598     case ImmTyDA: OS << "DA"; break;
599     case ImmTyR128: OS << "R128"; break;
600     case ImmTyLWE: OS << "LWE"; break;
601     case ImmTyOff: OS << "Off"; break;
602     case ImmTyExpTgt: OS << "ExpTgt"; break;
603     case ImmTyExpCompr: OS << "ExpCompr"; break;
604     case ImmTyExpVM: OS << "ExpVM"; break;
605     case ImmTyHwreg: OS << "Hwreg"; break;
606     case ImmTySendMsg: OS << "SendMsg"; break;
607     case ImmTyInterpSlot: OS << "InterpSlot"; break;
608     case ImmTyInterpAttr: OS << "InterpAttr"; break;
609     case ImmTyAttrChan: OS << "AttrChan"; break;
610     }
611   }
612 
613   void print(raw_ostream &OS) const override {
614     switch (Kind) {
615     case Register:
616       OS << "<register " << getReg() << " mods: " << Reg.Mods << '>';
617       break;
618     case Immediate:
619       OS << '<' << getImm();
620       if (getImmTy() != ImmTyNone) {
621         OS << " type: "; printImmTy(OS, getImmTy());
622       }
623       OS << " mods: " << Imm.Mods << '>';
624       break;
625     case Token:
626       OS << '\'' << getToken() << '\'';
627       break;
628     case Expression:
629       OS << "<expr " << *Expr << '>';
630       break;
631     }
632   }
633 
634   static AMDGPUOperand::Ptr CreateImm(const AMDGPUAsmParser *AsmParser,
635                                       int64_t Val, SMLoc Loc,
636                                       ImmTy Type = ImmTyNone,
637                                       bool IsFPImm = false) {
638     auto Op = llvm::make_unique<AMDGPUOperand>(Immediate, AsmParser);
639     Op->Imm.Val = Val;
640     Op->Imm.IsFPImm = IsFPImm;
641     Op->Imm.Type = Type;
642     Op->Imm.Mods = Modifiers();
643     Op->StartLoc = Loc;
644     Op->EndLoc = Loc;
645     return Op;
646   }
647 
648   static AMDGPUOperand::Ptr CreateToken(const AMDGPUAsmParser *AsmParser,
649                                         StringRef Str, SMLoc Loc,
650                                         bool HasExplicitEncodingSize = true) {
651     auto Res = llvm::make_unique<AMDGPUOperand>(Token, AsmParser);
652     Res->Tok.Data = Str.data();
653     Res->Tok.Length = Str.size();
654     Res->StartLoc = Loc;
655     Res->EndLoc = Loc;
656     return Res;
657   }
658 
659   static AMDGPUOperand::Ptr CreateReg(const AMDGPUAsmParser *AsmParser,
660                                       unsigned RegNo, SMLoc S,
661                                       SMLoc E,
662                                       bool ForceVOP3) {
663     auto Op = llvm::make_unique<AMDGPUOperand>(Register, AsmParser);
664     Op->Reg.RegNo = RegNo;
665     Op->Reg.Mods = Modifiers();
666     Op->Reg.IsForcedVOP3 = ForceVOP3;
667     Op->StartLoc = S;
668     Op->EndLoc = E;
669     return Op;
670   }
671 
672   static AMDGPUOperand::Ptr CreateExpr(const AMDGPUAsmParser *AsmParser,
673                                        const class MCExpr *Expr, SMLoc S) {
674     auto Op = llvm::make_unique<AMDGPUOperand>(Expression, AsmParser);
675     Op->Expr = Expr;
676     Op->StartLoc = S;
677     Op->EndLoc = S;
678     return Op;
679   }
680 };
681 
682 raw_ostream &operator <<(raw_ostream &OS, AMDGPUOperand::Modifiers Mods) {
683   OS << "abs:" << Mods.Abs << " neg: " << Mods.Neg << " sext:" << Mods.Sext;
684   return OS;
685 }
686 
687 //===----------------------------------------------------------------------===//
688 // AsmParser
689 //===----------------------------------------------------------------------===//
690 
691 // Holds info related to the current kernel, e.g. count of SGPRs used.
692 // Kernel scope begins at .amdgpu_hsa_kernel directive, ends at next
693 // .amdgpu_hsa_kernel or at EOF.
694 class KernelScopeInfo {
695   int SgprIndexUnusedMin = -1;
696   int VgprIndexUnusedMin = -1;
697   MCContext *Ctx = nullptr;
698 
699   void usesSgprAt(int i) {
700     if (i >= SgprIndexUnusedMin) {
701       SgprIndexUnusedMin = ++i;
702       if (Ctx) {
703         MCSymbol * const Sym = Ctx->getOrCreateSymbol(Twine(".kernel.sgpr_count"));
704         Sym->setVariableValue(MCConstantExpr::create(SgprIndexUnusedMin, *Ctx));
705       }
706     }
707   }
708 
709   void usesVgprAt(int i) {
710     if (i >= VgprIndexUnusedMin) {
711       VgprIndexUnusedMin = ++i;
712       if (Ctx) {
713         MCSymbol * const Sym = Ctx->getOrCreateSymbol(Twine(".kernel.vgpr_count"));
714         Sym->setVariableValue(MCConstantExpr::create(VgprIndexUnusedMin, *Ctx));
715       }
716     }
717   }
718 
719 public:
720   KernelScopeInfo() = default;
721 
722   void initialize(MCContext &Context) {
723     Ctx = &Context;
724     usesSgprAt(SgprIndexUnusedMin = -1);
725     usesVgprAt(VgprIndexUnusedMin = -1);
726   }
727 
728   void usesRegister(RegisterKind RegKind, unsigned DwordRegIndex, unsigned RegWidth) {
729     switch (RegKind) {
730       case IS_SGPR: usesSgprAt(DwordRegIndex + RegWidth - 1); break;
731       case IS_VGPR: usesVgprAt(DwordRegIndex + RegWidth - 1); break;
732       default: break;
733     }
734   }
735 };
736 
737 class AMDGPUAsmParser : public MCTargetAsmParser {
738   const MCInstrInfo &MII;
739   MCAsmParser &Parser;
740 
741   unsigned ForcedEncodingSize = 0;
742   bool ForcedDPP = false;
743   bool ForcedSDWA = false;
744   KernelScopeInfo KernelScope;
745 
746   /// @name Auto-generated Match Functions
747   /// {
748 
749 #define GET_ASSEMBLER_HEADER
750 #include "AMDGPUGenAsmMatcher.inc"
751 
752   /// }
753 
754 private:
755   bool ParseAsAbsoluteExpression(uint32_t &Ret);
756   bool ParseDirectiveMajorMinor(uint32_t &Major, uint32_t &Minor);
757   bool ParseDirectiveHSACodeObjectVersion();
758   bool ParseDirectiveHSACodeObjectISA();
759   bool ParseDirectiveRuntimeMetadata();
760   bool ParseAMDKernelCodeTValue(StringRef ID, amd_kernel_code_t &Header);
761   bool ParseDirectiveAMDKernelCodeT();
762   bool ParseSectionDirectiveHSAText();
763   bool subtargetHasRegister(const MCRegisterInfo &MRI, unsigned RegNo) const;
764   bool ParseDirectiveAMDGPUHsaKernel();
765   bool ParseDirectiveAMDGPUHsaModuleGlobal();
766   bool ParseDirectiveAMDGPUHsaProgramGlobal();
767   bool ParseSectionDirectiveHSADataGlobalAgent();
768   bool ParseSectionDirectiveHSADataGlobalProgram();
769   bool ParseSectionDirectiveHSARodataReadonlyAgent();
770   bool AddNextRegisterToList(unsigned& Reg, unsigned& RegWidth,
771                              RegisterKind RegKind, unsigned Reg1,
772                              unsigned RegNum);
773   bool ParseAMDGPURegister(RegisterKind& RegKind, unsigned& Reg,
774                            unsigned& RegNum, unsigned& RegWidth,
775                            unsigned *DwordRegIndex);
776   void cvtMubufImpl(MCInst &Inst, const OperandVector &Operands,
777                     bool IsAtomic, bool IsAtomicReturn);
778   void cvtDSImpl(MCInst &Inst, const OperandVector &Operands,
779                  bool IsGdsHardcoded);
780 
781 public:
782   enum AMDGPUMatchResultTy {
783     Match_PreferE32 = FIRST_TARGET_MATCH_RESULT_TY
784   };
785 
786   AMDGPUAsmParser(const MCSubtargetInfo &STI, MCAsmParser &_Parser,
787                const MCInstrInfo &MII,
788                const MCTargetOptions &Options)
789       : MCTargetAsmParser(Options, STI), MII(MII), Parser(_Parser) {
790     MCAsmParserExtension::Initialize(Parser);
791 
792     if (getSTI().getFeatureBits().none()) {
793       // Set default features.
794       copySTI().ToggleFeature("SOUTHERN_ISLANDS");
795     }
796 
797     setAvailableFeatures(ComputeAvailableFeatures(getSTI().getFeatureBits()));
798 
799     {
800       // TODO: make those pre-defined variables read-only.
801       // Currently there is none suitable machinery in the core llvm-mc for this.
802       // MCSymbol::isRedefinable is intended for another purpose, and
803       // AsmParser::parseDirectiveSet() cannot be specialized for specific target.
804       AMDGPU::IsaInfo::IsaVersion ISA =
805           AMDGPU::IsaInfo::getIsaVersion(getSTI().getFeatureBits());
806       MCContext &Ctx = getContext();
807       MCSymbol *Sym =
808           Ctx.getOrCreateSymbol(Twine(".option.machine_version_major"));
809       Sym->setVariableValue(MCConstantExpr::create(ISA.Major, Ctx));
810       Sym = Ctx.getOrCreateSymbol(Twine(".option.machine_version_minor"));
811       Sym->setVariableValue(MCConstantExpr::create(ISA.Minor, Ctx));
812       Sym = Ctx.getOrCreateSymbol(Twine(".option.machine_version_stepping"));
813       Sym->setVariableValue(MCConstantExpr::create(ISA.Stepping, Ctx));
814     }
815     KernelScope.initialize(getContext());
816   }
817 
818   bool isSI() const {
819     return AMDGPU::isSI(getSTI());
820   }
821 
822   bool isCI() const {
823     return AMDGPU::isCI(getSTI());
824   }
825 
826   bool isVI() const {
827     return AMDGPU::isVI(getSTI());
828   }
829 
830   bool hasInv2PiInlineImm() const {
831     return getSTI().getFeatureBits()[AMDGPU::FeatureInv2PiInlineImm];
832   }
833 
834   bool hasSGPR102_SGPR103() const {
835     return !isVI();
836   }
837 
838   AMDGPUTargetStreamer &getTargetStreamer() {
839     MCTargetStreamer &TS = *getParser().getStreamer().getTargetStreamer();
840     return static_cast<AMDGPUTargetStreamer &>(TS);
841   }
842 
843   const MCRegisterInfo *getMRI() const {
844     // We need this const_cast because for some reason getContext() is not const
845     // in MCAsmParser.
846     return const_cast<AMDGPUAsmParser*>(this)->getContext().getRegisterInfo();
847   }
848 
849   const MCInstrInfo *getMII() const {
850     return &MII;
851   }
852 
853   void setForcedEncodingSize(unsigned Size) { ForcedEncodingSize = Size; }
854   void setForcedDPP(bool ForceDPP_) { ForcedDPP = ForceDPP_; }
855   void setForcedSDWA(bool ForceSDWA_) { ForcedSDWA = ForceSDWA_; }
856 
857   unsigned getForcedEncodingSize() const { return ForcedEncodingSize; }
858   bool isForcedVOP3() const { return ForcedEncodingSize == 64; }
859   bool isForcedDPP() const { return ForcedDPP; }
860   bool isForcedSDWA() const { return ForcedSDWA; }
861   ArrayRef<unsigned> getMatchedVariants() const;
862 
863   std::unique_ptr<AMDGPUOperand> parseRegister();
864   bool ParseRegister(unsigned &RegNo, SMLoc &StartLoc, SMLoc &EndLoc) override;
865   unsigned checkTargetMatchPredicate(MCInst &Inst) override;
866   unsigned validateTargetOperandClass(MCParsedAsmOperand &Op,
867                                       unsigned Kind) override;
868   bool MatchAndEmitInstruction(SMLoc IDLoc, unsigned &Opcode,
869                                OperandVector &Operands, MCStreamer &Out,
870                                uint64_t &ErrorInfo,
871                                bool MatchingInlineAsm) override;
872   bool ParseDirective(AsmToken DirectiveID) override;
873   OperandMatchResultTy parseOperand(OperandVector &Operands, StringRef Mnemonic);
874   StringRef parseMnemonicSuffix(StringRef Name);
875   bool ParseInstruction(ParseInstructionInfo &Info, StringRef Name,
876                         SMLoc NameLoc, OperandVector &Operands) override;
877   //bool ProcessInstruction(MCInst &Inst);
878 
879   OperandMatchResultTy parseIntWithPrefix(const char *Prefix, int64_t &Int);
880   OperandMatchResultTy
881   parseIntWithPrefix(const char *Prefix, OperandVector &Operands,
882                      AMDGPUOperand::ImmTy ImmTy = AMDGPUOperand::ImmTyNone,
883                      bool (*ConvertResult)(int64_t &) = nullptr);
884   OperandMatchResultTy
885   parseNamedBit(const char *Name, OperandVector &Operands,
886                 AMDGPUOperand::ImmTy ImmTy = AMDGPUOperand::ImmTyNone);
887   OperandMatchResultTy parseStringWithPrefix(StringRef Prefix,
888                                              StringRef &Value);
889 
890   OperandMatchResultTy parseImm(OperandVector &Operands);
891   OperandMatchResultTy parseReg(OperandVector &Operands);
892   OperandMatchResultTy parseRegOrImm(OperandVector &Operands);
893   OperandMatchResultTy parseRegOrImmWithFPInputMods(OperandVector &Operands, bool AllowImm = true);
894   OperandMatchResultTy parseRegOrImmWithIntInputMods(OperandVector &Operands, bool AllowImm = true);
895   OperandMatchResultTy parseRegWithFPInputMods(OperandVector &Operands);
896   OperandMatchResultTy parseRegWithIntInputMods(OperandVector &Operands);
897   OperandMatchResultTy parseVReg32OrOff(OperandVector &Operands);
898 
899   void cvtDSOffset01(MCInst &Inst, const OperandVector &Operands);
900   void cvtDS(MCInst &Inst, const OperandVector &Operands) { cvtDSImpl(Inst, Operands, false); }
901   void cvtDSGds(MCInst &Inst, const OperandVector &Operands) { cvtDSImpl(Inst, Operands, true); }
902   void cvtExp(MCInst &Inst, const OperandVector &Operands);
903 
904   bool parseCnt(int64_t &IntVal);
905   OperandMatchResultTy parseSWaitCntOps(OperandVector &Operands);
906   OperandMatchResultTy parseHwreg(OperandVector &Operands);
907 
908 private:
909   struct OperandInfoTy {
910     int64_t Id;
911     bool IsSymbolic;
912     OperandInfoTy(int64_t Id_) : Id(Id_), IsSymbolic(false) { }
913   };
914 
915   bool parseSendMsgConstruct(OperandInfoTy &Msg, OperandInfoTy &Operation, int64_t &StreamId);
916   bool parseHwregConstruct(OperandInfoTy &HwReg, int64_t &Offset, int64_t &Width);
917 
918   void errorExpTgt();
919   OperandMatchResultTy parseExpTgtImpl(StringRef Str, uint8_t &Val);
920 
921 public:
922   OperandMatchResultTy parseOptionalOperand(OperandVector &Operands);
923 
924   OperandMatchResultTy parseExpTgt(OperandVector &Operands);
925   OperandMatchResultTy parseSendMsgOp(OperandVector &Operands);
926   OperandMatchResultTy parseInterpSlot(OperandVector &Operands);
927   OperandMatchResultTy parseInterpAttr(OperandVector &Operands);
928   OperandMatchResultTy parseSOppBrTarget(OperandVector &Operands);
929 
930   void cvtMubuf(MCInst &Inst, const OperandVector &Operands) { cvtMubufImpl(Inst, Operands, false, false); }
931   void cvtMubufAtomic(MCInst &Inst, const OperandVector &Operands) { cvtMubufImpl(Inst, Operands, true, false); }
932   void cvtMubufAtomicReturn(MCInst &Inst, const OperandVector &Operands) { cvtMubufImpl(Inst, Operands, true, true); }
933   AMDGPUOperand::Ptr defaultGLC() const;
934   AMDGPUOperand::Ptr defaultSLC() const;
935   AMDGPUOperand::Ptr defaultTFE() const;
936 
937   AMDGPUOperand::Ptr defaultDMask() const;
938   AMDGPUOperand::Ptr defaultUNorm() const;
939   AMDGPUOperand::Ptr defaultDA() const;
940   AMDGPUOperand::Ptr defaultR128() const;
941   AMDGPUOperand::Ptr defaultLWE() const;
942   AMDGPUOperand::Ptr defaultSMRDOffset8() const;
943   AMDGPUOperand::Ptr defaultSMRDOffset20() const;
944   AMDGPUOperand::Ptr defaultSMRDLiteralOffset() const;
945 
946   OperandMatchResultTy parseOModOperand(OperandVector &Operands);
947 
948   void cvtId(MCInst &Inst, const OperandVector &Operands);
949   void cvtVOP3_2_mod(MCInst &Inst, const OperandVector &Operands);
950   void cvtVOP3(MCInst &Inst, const OperandVector &Operands);
951 
952   void cvtMIMG(MCInst &Inst, const OperandVector &Operands);
953   void cvtMIMGAtomic(MCInst &Inst, const OperandVector &Operands);
954 
955   OperandMatchResultTy parseDPPCtrl(OperandVector &Operands);
956   AMDGPUOperand::Ptr defaultRowMask() const;
957   AMDGPUOperand::Ptr defaultBankMask() const;
958   AMDGPUOperand::Ptr defaultBoundCtrl() const;
959   void cvtDPP(MCInst &Inst, const OperandVector &Operands);
960 
961   OperandMatchResultTy parseSDWASel(OperandVector &Operands, StringRef Prefix,
962                                     AMDGPUOperand::ImmTy Type);
963   OperandMatchResultTy parseSDWADstUnused(OperandVector &Operands);
964   void cvtSdwaVOP1(MCInst &Inst, const OperandVector &Operands);
965   void cvtSdwaVOP2(MCInst &Inst, const OperandVector &Operands);
966   void cvtSdwaVOPC(MCInst &Inst, const OperandVector &Operands);
967   void cvtSDWA(MCInst &Inst, const OperandVector &Operands,
968                uint64_t BasicInstType);
969 };
970 
971 struct OptionalOperand {
972   const char *Name;
973   AMDGPUOperand::ImmTy Type;
974   bool IsBit;
975   bool (*ConvertResult)(int64_t&);
976 };
977 
978 } // end anonymous namespace
979 
980 // May be called with integer type with equivalent bitwidth.
981 static const fltSemantics *getFltSemantics(unsigned Size) {
982   switch (Size) {
983   case 4:
984     return &APFloat::IEEEsingle();
985   case 8:
986     return &APFloat::IEEEdouble();
987   case 2:
988     return &APFloat::IEEEhalf();
989   default:
990     llvm_unreachable("unsupported fp type");
991   }
992 }
993 
994 static const fltSemantics *getFltSemantics(MVT VT) {
995   return getFltSemantics(VT.getSizeInBits() / 8);
996 }
997 
998 //===----------------------------------------------------------------------===//
999 // Operand
1000 //===----------------------------------------------------------------------===//
1001 
1002 static bool canLosslesslyConvertToFPType(APFloat &FPLiteral, MVT VT) {
1003   bool Lost;
1004 
1005   // Convert literal to single precision
1006   APFloat::opStatus Status = FPLiteral.convert(*getFltSemantics(VT),
1007                                                APFloat::rmNearestTiesToEven,
1008                                                &Lost);
1009   // We allow precision lost but not overflow or underflow
1010   if (Status != APFloat::opOK &&
1011       Lost &&
1012       ((Status & APFloat::opOverflow)  != 0 ||
1013        (Status & APFloat::opUnderflow) != 0)) {
1014     return false;
1015   }
1016 
1017   return true;
1018 }
1019 
1020 bool AMDGPUOperand::isInlinableImm(MVT type) const {
1021   if (!isImmTy(ImmTyNone)) {
1022     // Only plain immediates are inlinable (e.g. "clamp" attribute is not)
1023     return false;
1024   }
1025   // TODO: We should avoid using host float here. It would be better to
1026   // check the float bit values which is what a few other places do.
1027   // We've had bot failures before due to weird NaN support on mips hosts.
1028 
1029   APInt Literal(64, Imm.Val);
1030 
1031   if (Imm.IsFPImm) { // We got fp literal token
1032     if (type == MVT::f64 || type == MVT::i64) { // Expected 64-bit operand
1033       return AMDGPU::isInlinableLiteral64(Imm.Val,
1034                                           AsmParser->hasInv2PiInlineImm());
1035     }
1036 
1037     APFloat FPLiteral(APFloat::IEEEdouble(), APInt(64, Imm.Val));
1038     if (!canLosslesslyConvertToFPType(FPLiteral, type))
1039       return false;
1040 
1041     if (type.getScalarSizeInBits() == 16) {
1042       return AMDGPU::isInlinableLiteral16(
1043         static_cast<int32_t>(FPLiteral.bitcastToAPInt().getZExtValue()),
1044         AsmParser->hasInv2PiInlineImm());
1045     }
1046 
1047     // Check if single precision literal is inlinable
1048     return AMDGPU::isInlinableLiteral32(
1049       static_cast<int32_t>(FPLiteral.bitcastToAPInt().getZExtValue()),
1050       AsmParser->hasInv2PiInlineImm());
1051   }
1052 
1053   // We got int literal token.
1054   if (type == MVT::f64 || type == MVT::i64) { // Expected 64-bit operand
1055     return AMDGPU::isInlinableLiteral64(Imm.Val,
1056                                         AsmParser->hasInv2PiInlineImm());
1057   }
1058 
1059   if (type.getScalarSizeInBits() == 16) {
1060     return AMDGPU::isInlinableLiteral16(
1061       static_cast<int16_t>(Literal.getLoBits(16).getSExtValue()),
1062       AsmParser->hasInv2PiInlineImm());
1063   }
1064 
1065   return AMDGPU::isInlinableLiteral32(
1066     static_cast<int32_t>(Literal.getLoBits(32).getZExtValue()),
1067     AsmParser->hasInv2PiInlineImm());
1068 }
1069 
1070 bool AMDGPUOperand::isLiteralImm(MVT type) const {
1071   // Check that this imediate can be added as literal
1072   if (!isImmTy(ImmTyNone)) {
1073     return false;
1074   }
1075 
1076   if (!Imm.IsFPImm) {
1077     // We got int literal token.
1078 
1079     unsigned Size = type.getSizeInBits();
1080     if (Size == 64)
1081       Size = 32;
1082 
1083     // FIXME: 64-bit operands can zero extend, sign extend, or pad zeroes for FP
1084     // types.
1085     return isUIntN(Size, Imm.Val) || isIntN(Size, Imm.Val);
1086   }
1087 
1088   // We got fp literal token
1089   if (type == MVT::f64) { // Expected 64-bit fp operand
1090     // We would set low 64-bits of literal to zeroes but we accept this literals
1091     return true;
1092   }
1093 
1094   if (type == MVT::i64) { // Expected 64-bit int operand
1095     // We don't allow fp literals in 64-bit integer instructions. It is
1096     // unclear how we should encode them.
1097     return false;
1098   }
1099 
1100   APFloat FPLiteral(APFloat::IEEEdouble(), APInt(64, Imm.Val));
1101   return canLosslesslyConvertToFPType(FPLiteral, type);
1102 }
1103 
1104 bool AMDGPUOperand::isRegClass(unsigned RCID) const {
1105   return isRegKind() && AsmParser->getMRI()->getRegClass(RCID).contains(getReg());
1106 }
1107 
1108 void AMDGPUOperand::addImmOperands(MCInst &Inst, unsigned N, bool ApplyModifiers) const {
1109   int64_t Val = Imm.Val;
1110   if (isImmTy(ImmTyNone) && ApplyModifiers && Imm.Mods.hasFPModifiers() && Imm.Mods.Neg) {
1111     // Apply modifiers to immediate value. Only negate can get here
1112     if (Imm.IsFPImm) {
1113       APFloat F(BitsToDouble(Val));
1114       F.changeSign();
1115       Val = F.bitcastToAPInt().getZExtValue();
1116     } else {
1117       Val = -Val;
1118     }
1119   }
1120 
1121   if (AMDGPU::isSISrcOperand(AsmParser->getMII()->get(Inst.getOpcode()),
1122                              Inst.getNumOperands())) {
1123     addLiteralImmOperand(Inst, Val);
1124   } else {
1125     Inst.addOperand(MCOperand::createImm(Val));
1126   }
1127 }
1128 
1129 void AMDGPUOperand::addLiteralImmOperand(MCInst &Inst, int64_t Val) const {
1130   const auto& InstDesc = AsmParser->getMII()->get(Inst.getOpcode());
1131   auto OpNum = Inst.getNumOperands();
1132   // Check that this operand accepts literals
1133   assert(AMDGPU::isSISrcOperand(InstDesc, OpNum));
1134 
1135   auto OpSize = AMDGPU::getOperandSize(InstDesc, OpNum); // expected operand size
1136 
1137   if (Imm.IsFPImm) { // We got fp literal token
1138     APInt Literal(64, Val);
1139 
1140     switch (OpSize) {
1141     case 8:
1142       if (AMDGPU::isInlinableLiteral64(Literal.getZExtValue(),
1143                                        AsmParser->hasInv2PiInlineImm())) {
1144         Inst.addOperand(MCOperand::createImm(Literal.getZExtValue()));
1145         return;
1146       }
1147 
1148       // Non-inlineable
1149       if (AMDGPU::isSISrcFPOperand(InstDesc, OpNum)) { // Expected 64-bit fp operand
1150         // For fp operands we check if low 32 bits are zeros
1151         if (Literal.getLoBits(32) != 0) {
1152           const_cast<AMDGPUAsmParser *>(AsmParser)->Warning(Inst.getLoc(),
1153           "Can't encode literal as exact 64-bit floating-point operand. "
1154           "Low 32-bits will be set to zero");
1155         }
1156 
1157         Inst.addOperand(MCOperand::createImm(Literal.lshr(32).getZExtValue()));
1158         return;
1159       }
1160 
1161       // We don't allow fp literals in 64-bit integer instructions. It is
1162       // unclear how we should encode them. This case should be checked earlier
1163       // in predicate methods (isLiteralImm())
1164       llvm_unreachable("fp literal in 64-bit integer instruction.");
1165 
1166     case 4:
1167     case 2: {
1168       bool lost;
1169       APFloat FPLiteral(APFloat::IEEEdouble(), Literal);
1170       // Convert literal to single precision
1171       FPLiteral.convert(*getFltSemantics(OpSize),
1172                         APFloat::rmNearestTiesToEven, &lost);
1173       // We allow precision lost but not overflow or underflow. This should be
1174       // checked earlier in isLiteralImm()
1175       Inst.addOperand(MCOperand::createImm(FPLiteral.bitcastToAPInt().getZExtValue()));
1176       return;
1177     }
1178     default:
1179       llvm_unreachable("invalid operand size");
1180     }
1181 
1182     return;
1183   }
1184 
1185    // We got int literal token.
1186   // Only sign extend inline immediates.
1187   // FIXME: No errors on truncation
1188   switch (OpSize) {
1189   case 4:
1190     if (isInt<32>(Val) &&
1191         AMDGPU::isInlinableLiteral32(static_cast<int32_t>(Val),
1192                                      AsmParser->hasInv2PiInlineImm())) {
1193       Inst.addOperand(MCOperand::createImm(Val));
1194       return;
1195     }
1196 
1197     Inst.addOperand(MCOperand::createImm(Val & 0xffffffff));
1198     return;
1199 
1200   case 8:
1201     if (AMDGPU::isInlinableLiteral64(Val,
1202                                      AsmParser->hasInv2PiInlineImm())) {
1203       Inst.addOperand(MCOperand::createImm(Val));
1204       return;
1205     }
1206 
1207     Inst.addOperand(MCOperand::createImm(Lo_32(Val)));
1208     return;
1209 
1210   case 2:
1211     if (isInt<16>(Val) &&
1212         AMDGPU::isInlinableLiteral16(static_cast<int16_t>(Val),
1213                                      AsmParser->hasInv2PiInlineImm())) {
1214       Inst.addOperand(MCOperand::createImm(Val));
1215       return;
1216     }
1217 
1218     Inst.addOperand(MCOperand::createImm(Val & 0xffff));
1219     return;
1220 
1221   default:
1222     llvm_unreachable("invalid operand size");
1223   }
1224 }
1225 
1226 template <unsigned Bitwidth>
1227 void AMDGPUOperand::addKImmFPOperands(MCInst &Inst, unsigned N) const {
1228   APInt Literal(64, Imm.Val);
1229 
1230   if (!Imm.IsFPImm) {
1231     // We got int literal token.
1232     Inst.addOperand(MCOperand::createImm(Literal.getLoBits(Bitwidth).getZExtValue()));
1233     return;
1234   }
1235 
1236   bool Lost;
1237   APFloat FPLiteral(APFloat::IEEEdouble(), Literal);
1238   FPLiteral.convert(*getFltSemantics(Bitwidth / 8),
1239                     APFloat::rmNearestTiesToEven, &Lost);
1240   Inst.addOperand(MCOperand::createImm(FPLiteral.bitcastToAPInt().getZExtValue()));
1241 }
1242 
1243 void AMDGPUOperand::addRegOperands(MCInst &Inst, unsigned N) const {
1244   Inst.addOperand(MCOperand::createReg(AMDGPU::getMCReg(getReg(), AsmParser->getSTI())));
1245 }
1246 
1247 //===----------------------------------------------------------------------===//
1248 // AsmParser
1249 //===----------------------------------------------------------------------===//
1250 
1251 static int getRegClass(RegisterKind Is, unsigned RegWidth) {
1252   if (Is == IS_VGPR) {
1253     switch (RegWidth) {
1254       default: return -1;
1255       case 1: return AMDGPU::VGPR_32RegClassID;
1256       case 2: return AMDGPU::VReg_64RegClassID;
1257       case 3: return AMDGPU::VReg_96RegClassID;
1258       case 4: return AMDGPU::VReg_128RegClassID;
1259       case 8: return AMDGPU::VReg_256RegClassID;
1260       case 16: return AMDGPU::VReg_512RegClassID;
1261     }
1262   } else if (Is == IS_TTMP) {
1263     switch (RegWidth) {
1264       default: return -1;
1265       case 1: return AMDGPU::TTMP_32RegClassID;
1266       case 2: return AMDGPU::TTMP_64RegClassID;
1267       case 4: return AMDGPU::TTMP_128RegClassID;
1268     }
1269   } else if (Is == IS_SGPR) {
1270     switch (RegWidth) {
1271       default: return -1;
1272       case 1: return AMDGPU::SGPR_32RegClassID;
1273       case 2: return AMDGPU::SGPR_64RegClassID;
1274       case 4: return AMDGPU::SGPR_128RegClassID;
1275       case 8: return AMDGPU::SReg_256RegClassID;
1276       case 16: return AMDGPU::SReg_512RegClassID;
1277     }
1278   }
1279   return -1;
1280 }
1281 
1282 static unsigned getSpecialRegForName(StringRef RegName) {
1283   return StringSwitch<unsigned>(RegName)
1284     .Case("exec", AMDGPU::EXEC)
1285     .Case("vcc", AMDGPU::VCC)
1286     .Case("flat_scratch", AMDGPU::FLAT_SCR)
1287     .Case("m0", AMDGPU::M0)
1288     .Case("scc", AMDGPU::SCC)
1289     .Case("tba", AMDGPU::TBA)
1290     .Case("tma", AMDGPU::TMA)
1291     .Case("flat_scratch_lo", AMDGPU::FLAT_SCR_LO)
1292     .Case("flat_scratch_hi", AMDGPU::FLAT_SCR_HI)
1293     .Case("vcc_lo", AMDGPU::VCC_LO)
1294     .Case("vcc_hi", AMDGPU::VCC_HI)
1295     .Case("exec_lo", AMDGPU::EXEC_LO)
1296     .Case("exec_hi", AMDGPU::EXEC_HI)
1297     .Case("tma_lo", AMDGPU::TMA_LO)
1298     .Case("tma_hi", AMDGPU::TMA_HI)
1299     .Case("tba_lo", AMDGPU::TBA_LO)
1300     .Case("tba_hi", AMDGPU::TBA_HI)
1301     .Default(0);
1302 }
1303 
1304 bool AMDGPUAsmParser::ParseRegister(unsigned &RegNo, SMLoc &StartLoc,
1305                                     SMLoc &EndLoc) {
1306   auto R = parseRegister();
1307   if (!R) return true;
1308   assert(R->isReg());
1309   RegNo = R->getReg();
1310   StartLoc = R->getStartLoc();
1311   EndLoc = R->getEndLoc();
1312   return false;
1313 }
1314 
1315 bool AMDGPUAsmParser::AddNextRegisterToList(unsigned &Reg, unsigned &RegWidth,
1316                                             RegisterKind RegKind, unsigned Reg1,
1317                                             unsigned RegNum) {
1318   switch (RegKind) {
1319   case IS_SPECIAL:
1320     if (Reg == AMDGPU::EXEC_LO && Reg1 == AMDGPU::EXEC_HI) {
1321       Reg = AMDGPU::EXEC;
1322       RegWidth = 2;
1323       return true;
1324     }
1325     if (Reg == AMDGPU::FLAT_SCR_LO && Reg1 == AMDGPU::FLAT_SCR_HI) {
1326       Reg = AMDGPU::FLAT_SCR;
1327       RegWidth = 2;
1328       return true;
1329     }
1330     if (Reg == AMDGPU::VCC_LO && Reg1 == AMDGPU::VCC_HI) {
1331       Reg = AMDGPU::VCC;
1332       RegWidth = 2;
1333       return true;
1334     }
1335     if (Reg == AMDGPU::TBA_LO && Reg1 == AMDGPU::TBA_HI) {
1336       Reg = AMDGPU::TBA;
1337       RegWidth = 2;
1338       return true;
1339     }
1340     if (Reg == AMDGPU::TMA_LO && Reg1 == AMDGPU::TMA_HI) {
1341       Reg = AMDGPU::TMA;
1342       RegWidth = 2;
1343       return true;
1344     }
1345     return false;
1346   case IS_VGPR:
1347   case IS_SGPR:
1348   case IS_TTMP:
1349     if (Reg1 != Reg + RegWidth) {
1350       return false;
1351     }
1352     RegWidth++;
1353     return true;
1354   default:
1355     llvm_unreachable("unexpected register kind");
1356   }
1357 }
1358 
1359 bool AMDGPUAsmParser::ParseAMDGPURegister(RegisterKind &RegKind, unsigned &Reg,
1360                                           unsigned &RegNum, unsigned &RegWidth,
1361                                           unsigned *DwordRegIndex) {
1362   if (DwordRegIndex) { *DwordRegIndex = 0; }
1363   const MCRegisterInfo *TRI = getContext().getRegisterInfo();
1364   if (getLexer().is(AsmToken::Identifier)) {
1365     StringRef RegName = Parser.getTok().getString();
1366     if ((Reg = getSpecialRegForName(RegName))) {
1367       Parser.Lex();
1368       RegKind = IS_SPECIAL;
1369     } else {
1370       unsigned RegNumIndex = 0;
1371       if (RegName[0] == 'v') {
1372         RegNumIndex = 1;
1373         RegKind = IS_VGPR;
1374       } else if (RegName[0] == 's') {
1375         RegNumIndex = 1;
1376         RegKind = IS_SGPR;
1377       } else if (RegName.startswith("ttmp")) {
1378         RegNumIndex = strlen("ttmp");
1379         RegKind = IS_TTMP;
1380       } else {
1381         return false;
1382       }
1383       if (RegName.size() > RegNumIndex) {
1384         // Single 32-bit register: vXX.
1385         if (RegName.substr(RegNumIndex).getAsInteger(10, RegNum))
1386           return false;
1387         Parser.Lex();
1388         RegWidth = 1;
1389       } else {
1390         // Range of registers: v[XX:YY]. ":YY" is optional.
1391         Parser.Lex();
1392         int64_t RegLo, RegHi;
1393         if (getLexer().isNot(AsmToken::LBrac))
1394           return false;
1395         Parser.Lex();
1396 
1397         if (getParser().parseAbsoluteExpression(RegLo))
1398           return false;
1399 
1400         const bool isRBrace = getLexer().is(AsmToken::RBrac);
1401         if (!isRBrace && getLexer().isNot(AsmToken::Colon))
1402           return false;
1403         Parser.Lex();
1404 
1405         if (isRBrace) {
1406           RegHi = RegLo;
1407         } else {
1408           if (getParser().parseAbsoluteExpression(RegHi))
1409             return false;
1410 
1411           if (getLexer().isNot(AsmToken::RBrac))
1412             return false;
1413           Parser.Lex();
1414         }
1415         RegNum = (unsigned) RegLo;
1416         RegWidth = (RegHi - RegLo) + 1;
1417       }
1418     }
1419   } else if (getLexer().is(AsmToken::LBrac)) {
1420     // List of consecutive registers: [s0,s1,s2,s3]
1421     Parser.Lex();
1422     if (!ParseAMDGPURegister(RegKind, Reg, RegNum, RegWidth, nullptr))
1423       return false;
1424     if (RegWidth != 1)
1425       return false;
1426     RegisterKind RegKind1;
1427     unsigned Reg1, RegNum1, RegWidth1;
1428     do {
1429       if (getLexer().is(AsmToken::Comma)) {
1430         Parser.Lex();
1431       } else if (getLexer().is(AsmToken::RBrac)) {
1432         Parser.Lex();
1433         break;
1434       } else if (ParseAMDGPURegister(RegKind1, Reg1, RegNum1, RegWidth1, nullptr)) {
1435         if (RegWidth1 != 1) {
1436           return false;
1437         }
1438         if (RegKind1 != RegKind) {
1439           return false;
1440         }
1441         if (!AddNextRegisterToList(Reg, RegWidth, RegKind1, Reg1, RegNum1)) {
1442           return false;
1443         }
1444       } else {
1445         return false;
1446       }
1447     } while (true);
1448   } else {
1449     return false;
1450   }
1451   switch (RegKind) {
1452   case IS_SPECIAL:
1453     RegNum = 0;
1454     RegWidth = 1;
1455     break;
1456   case IS_VGPR:
1457   case IS_SGPR:
1458   case IS_TTMP:
1459   {
1460     unsigned Size = 1;
1461     if (RegKind == IS_SGPR || RegKind == IS_TTMP) {
1462       // SGPR and TTMP registers must be aligned. Max required alignment is 4 dwords.
1463       Size = std::min(RegWidth, 4u);
1464     }
1465     if (RegNum % Size != 0)
1466       return false;
1467     if (DwordRegIndex) { *DwordRegIndex = RegNum; }
1468     RegNum = RegNum / Size;
1469     int RCID = getRegClass(RegKind, RegWidth);
1470     if (RCID == -1)
1471       return false;
1472     const MCRegisterClass RC = TRI->getRegClass(RCID);
1473     if (RegNum >= RC.getNumRegs())
1474       return false;
1475     Reg = RC.getRegister(RegNum);
1476     break;
1477   }
1478 
1479   default:
1480     llvm_unreachable("unexpected register kind");
1481   }
1482 
1483   if (!subtargetHasRegister(*TRI, Reg))
1484     return false;
1485   return true;
1486 }
1487 
1488 std::unique_ptr<AMDGPUOperand> AMDGPUAsmParser::parseRegister() {
1489   const auto &Tok = Parser.getTok();
1490   SMLoc StartLoc = Tok.getLoc();
1491   SMLoc EndLoc = Tok.getEndLoc();
1492   RegisterKind RegKind;
1493   unsigned Reg, RegNum, RegWidth, DwordRegIndex;
1494 
1495   if (!ParseAMDGPURegister(RegKind, Reg, RegNum, RegWidth, &DwordRegIndex)) {
1496     return nullptr;
1497   }
1498   KernelScope.usesRegister(RegKind, DwordRegIndex, RegWidth);
1499   return AMDGPUOperand::CreateReg(this, Reg, StartLoc, EndLoc, false);
1500 }
1501 
1502 OperandMatchResultTy
1503 AMDGPUAsmParser::parseImm(OperandVector &Operands) {
1504   // TODO: add syntactic sugar for 1/(2*PI)
1505   bool Minus = false;
1506   if (getLexer().getKind() == AsmToken::Minus) {
1507     Minus = true;
1508     Parser.Lex();
1509   }
1510 
1511   SMLoc S = Parser.getTok().getLoc();
1512   switch(getLexer().getKind()) {
1513   case AsmToken::Integer: {
1514     int64_t IntVal;
1515     if (getParser().parseAbsoluteExpression(IntVal))
1516       return MatchOperand_ParseFail;
1517     if (Minus)
1518       IntVal *= -1;
1519     Operands.push_back(AMDGPUOperand::CreateImm(this, IntVal, S));
1520     return MatchOperand_Success;
1521   }
1522   case AsmToken::Real: {
1523     int64_t IntVal;
1524     if (getParser().parseAbsoluteExpression(IntVal))
1525       return MatchOperand_ParseFail;
1526 
1527     APFloat F(BitsToDouble(IntVal));
1528     if (Minus)
1529       F.changeSign();
1530     Operands.push_back(
1531         AMDGPUOperand::CreateImm(this, F.bitcastToAPInt().getZExtValue(), S,
1532                                  AMDGPUOperand::ImmTyNone, true));
1533     return MatchOperand_Success;
1534   }
1535   default:
1536     return Minus ? MatchOperand_ParseFail : MatchOperand_NoMatch;
1537   }
1538 }
1539 
1540 OperandMatchResultTy
1541 AMDGPUAsmParser::parseReg(OperandVector &Operands) {
1542   if (auto R = parseRegister()) {
1543     assert(R->isReg());
1544     R->Reg.IsForcedVOP3 = isForcedVOP3();
1545     Operands.push_back(std::move(R));
1546     return MatchOperand_Success;
1547   }
1548   return MatchOperand_NoMatch;
1549 }
1550 
1551 OperandMatchResultTy
1552 AMDGPUAsmParser::parseRegOrImm(OperandVector &Operands) {
1553   auto res = parseImm(Operands);
1554   if (res != MatchOperand_NoMatch) {
1555     return res;
1556   }
1557 
1558   return parseReg(Operands);
1559 }
1560 
1561 OperandMatchResultTy
1562 AMDGPUAsmParser::parseRegOrImmWithFPInputMods(OperandVector &Operands,
1563                                               bool AllowImm) {
1564   // XXX: During parsing we can't determine if minus sign means
1565   // negate-modifier or negative immediate value.
1566   // By default we suppose it is modifier.
1567   bool Negate = false, Abs = false, Abs2 = false;
1568 
1569   if (getLexer().getKind()== AsmToken::Minus) {
1570     Parser.Lex();
1571     Negate = true;
1572   }
1573 
1574   if (getLexer().getKind() == AsmToken::Identifier &&
1575       Parser.getTok().getString() == "abs") {
1576     Parser.Lex();
1577     Abs2 = true;
1578     if (getLexer().isNot(AsmToken::LParen)) {
1579       Error(Parser.getTok().getLoc(), "expected left paren after abs");
1580       return MatchOperand_ParseFail;
1581     }
1582     Parser.Lex();
1583   }
1584 
1585   if (getLexer().getKind() == AsmToken::Pipe) {
1586     if (Abs2) {
1587       Error(Parser.getTok().getLoc(), "expected register or immediate");
1588       return MatchOperand_ParseFail;
1589     }
1590     Parser.Lex();
1591     Abs = true;
1592   }
1593 
1594   OperandMatchResultTy Res;
1595   if (AllowImm) {
1596     Res = parseRegOrImm(Operands);
1597   } else {
1598     Res = parseReg(Operands);
1599   }
1600   if (Res != MatchOperand_Success) {
1601     return Res;
1602   }
1603 
1604   AMDGPUOperand::Modifiers Mods;
1605   if (Negate) {
1606     Mods.Neg = true;
1607   }
1608   if (Abs) {
1609     if (getLexer().getKind() != AsmToken::Pipe) {
1610       Error(Parser.getTok().getLoc(), "expected vertical bar");
1611       return MatchOperand_ParseFail;
1612     }
1613     Parser.Lex();
1614     Mods.Abs = true;
1615   }
1616   if (Abs2) {
1617     if (getLexer().isNot(AsmToken::RParen)) {
1618       Error(Parser.getTok().getLoc(), "expected closing parentheses");
1619       return MatchOperand_ParseFail;
1620     }
1621     Parser.Lex();
1622     Mods.Abs = true;
1623   }
1624 
1625   if (Mods.hasFPModifiers()) {
1626     AMDGPUOperand &Op = static_cast<AMDGPUOperand &>(*Operands.back());
1627     Op.setModifiers(Mods);
1628   }
1629   return MatchOperand_Success;
1630 }
1631 
1632 OperandMatchResultTy
1633 AMDGPUAsmParser::parseRegOrImmWithIntInputMods(OperandVector &Operands,
1634                                                bool AllowImm) {
1635   bool Sext = false;
1636 
1637   if (getLexer().getKind() == AsmToken::Identifier &&
1638       Parser.getTok().getString() == "sext") {
1639     Parser.Lex();
1640     Sext = true;
1641     if (getLexer().isNot(AsmToken::LParen)) {
1642       Error(Parser.getTok().getLoc(), "expected left paren after sext");
1643       return MatchOperand_ParseFail;
1644     }
1645     Parser.Lex();
1646   }
1647 
1648   OperandMatchResultTy Res;
1649   if (AllowImm) {
1650     Res = parseRegOrImm(Operands);
1651   } else {
1652     Res = parseReg(Operands);
1653   }
1654   if (Res != MatchOperand_Success) {
1655     return Res;
1656   }
1657 
1658   AMDGPUOperand::Modifiers Mods;
1659   if (Sext) {
1660     if (getLexer().isNot(AsmToken::RParen)) {
1661       Error(Parser.getTok().getLoc(), "expected closing parentheses");
1662       return MatchOperand_ParseFail;
1663     }
1664     Parser.Lex();
1665     Mods.Sext = true;
1666   }
1667 
1668   if (Mods.hasIntModifiers()) {
1669     AMDGPUOperand &Op = static_cast<AMDGPUOperand &>(*Operands.back());
1670     Op.setModifiers(Mods);
1671   }
1672 
1673   return MatchOperand_Success;
1674 }
1675 
1676 OperandMatchResultTy
1677 AMDGPUAsmParser::parseRegWithFPInputMods(OperandVector &Operands) {
1678   return parseRegOrImmWithFPInputMods(Operands, false);
1679 }
1680 
1681 OperandMatchResultTy
1682 AMDGPUAsmParser::parseRegWithIntInputMods(OperandVector &Operands) {
1683   return parseRegOrImmWithIntInputMods(Operands, false);
1684 }
1685 
1686 OperandMatchResultTy AMDGPUAsmParser::parseVReg32OrOff(OperandVector &Operands) {
1687   std::unique_ptr<AMDGPUOperand> Reg = parseRegister();
1688   if (Reg) {
1689     Operands.push_back(std::move(Reg));
1690     return MatchOperand_Success;
1691   }
1692 
1693   const AsmToken &Tok = Parser.getTok();
1694   if (Tok.getString() == "off") {
1695     Operands.push_back(AMDGPUOperand::CreateImm(this, 0, Tok.getLoc(),
1696                                                 AMDGPUOperand::ImmTyOff, false));
1697     Parser.Lex();
1698     return MatchOperand_Success;
1699   }
1700 
1701   return MatchOperand_NoMatch;
1702 }
1703 
1704 unsigned AMDGPUAsmParser::checkTargetMatchPredicate(MCInst &Inst) {
1705   uint64_t TSFlags = MII.get(Inst.getOpcode()).TSFlags;
1706 
1707   if ((getForcedEncodingSize() == 32 && (TSFlags & SIInstrFlags::VOP3)) ||
1708       (getForcedEncodingSize() == 64 && !(TSFlags & SIInstrFlags::VOP3)) ||
1709       (isForcedDPP() && !(TSFlags & SIInstrFlags::DPP)) ||
1710       (isForcedSDWA() && !(TSFlags & SIInstrFlags::SDWA)) )
1711     return Match_InvalidOperand;
1712 
1713   if ((TSFlags & SIInstrFlags::VOP3) &&
1714       (TSFlags & SIInstrFlags::VOPAsmPrefer32Bit) &&
1715       getForcedEncodingSize() != 64)
1716     return Match_PreferE32;
1717 
1718   if (Inst.getOpcode() == AMDGPU::V_MAC_F32_sdwa_vi ||
1719       Inst.getOpcode() == AMDGPU::V_MAC_F16_sdwa_vi) {
1720     // v_mac_f32/16 allow only dst_sel == DWORD;
1721     auto OpNum =
1722         AMDGPU::getNamedOperandIdx(Inst.getOpcode(), AMDGPU::OpName::dst_sel);
1723     const auto &Op = Inst.getOperand(OpNum);
1724     if (!Op.isImm() || Op.getImm() != AMDGPU::SDWA::SdwaSel::DWORD) {
1725       return Match_InvalidOperand;
1726     }
1727   }
1728 
1729   return Match_Success;
1730 }
1731 
1732 // What asm variants we should check
1733 ArrayRef<unsigned> AMDGPUAsmParser::getMatchedVariants() const {
1734   if (getForcedEncodingSize() == 32) {
1735     static const unsigned Variants[] = {AMDGPUAsmVariants::DEFAULT};
1736     return makeArrayRef(Variants);
1737   }
1738 
1739   if (isForcedVOP3()) {
1740     static const unsigned Variants[] = {AMDGPUAsmVariants::VOP3};
1741     return makeArrayRef(Variants);
1742   }
1743 
1744   if (isForcedSDWA()) {
1745     static const unsigned Variants[] = {AMDGPUAsmVariants::SDWA};
1746     return makeArrayRef(Variants);
1747   }
1748 
1749   if (isForcedDPP()) {
1750     static const unsigned Variants[] = {AMDGPUAsmVariants::DPP};
1751     return makeArrayRef(Variants);
1752   }
1753 
1754   static const unsigned Variants[] = {
1755     AMDGPUAsmVariants::DEFAULT, AMDGPUAsmVariants::VOP3,
1756     AMDGPUAsmVariants::SDWA, AMDGPUAsmVariants::DPP
1757   };
1758 
1759   return makeArrayRef(Variants);
1760 }
1761 
1762 bool AMDGPUAsmParser::MatchAndEmitInstruction(SMLoc IDLoc, unsigned &Opcode,
1763                                               OperandVector &Operands,
1764                                               MCStreamer &Out,
1765                                               uint64_t &ErrorInfo,
1766                                               bool MatchingInlineAsm) {
1767   MCInst Inst;
1768   unsigned Result = Match_Success;
1769   for (auto Variant : getMatchedVariants()) {
1770     uint64_t EI;
1771     auto R = MatchInstructionImpl(Operands, Inst, EI, MatchingInlineAsm,
1772                                   Variant);
1773     // We order match statuses from least to most specific. We use most specific
1774     // status as resulting
1775     // Match_MnemonicFail < Match_InvalidOperand < Match_MissingFeature < Match_PreferE32
1776     if ((R == Match_Success) ||
1777         (R == Match_PreferE32) ||
1778         (R == Match_MissingFeature && Result != Match_PreferE32) ||
1779         (R == Match_InvalidOperand && Result != Match_MissingFeature
1780                                    && Result != Match_PreferE32) ||
1781         (R == Match_MnemonicFail   && Result != Match_InvalidOperand
1782                                    && Result != Match_MissingFeature
1783                                    && Result != Match_PreferE32)) {
1784       Result = R;
1785       ErrorInfo = EI;
1786     }
1787     if (R == Match_Success)
1788       break;
1789   }
1790 
1791   switch (Result) {
1792   default: break;
1793   case Match_Success:
1794     Inst.setLoc(IDLoc);
1795     Out.EmitInstruction(Inst, getSTI());
1796     return false;
1797 
1798   case Match_MissingFeature:
1799     return Error(IDLoc, "instruction not supported on this GPU");
1800 
1801   case Match_MnemonicFail:
1802     return Error(IDLoc, "unrecognized instruction mnemonic");
1803 
1804   case Match_InvalidOperand: {
1805     SMLoc ErrorLoc = IDLoc;
1806     if (ErrorInfo != ~0ULL) {
1807       if (ErrorInfo >= Operands.size()) {
1808         return Error(IDLoc, "too few operands for instruction");
1809       }
1810       ErrorLoc = ((AMDGPUOperand &)*Operands[ErrorInfo]).getStartLoc();
1811       if (ErrorLoc == SMLoc())
1812         ErrorLoc = IDLoc;
1813     }
1814     return Error(ErrorLoc, "invalid operand for instruction");
1815   }
1816 
1817   case Match_PreferE32:
1818     return Error(IDLoc, "internal error: instruction without _e64 suffix "
1819                         "should be encoded as e32");
1820   }
1821   llvm_unreachable("Implement any new match types added!");
1822 }
1823 
1824 bool AMDGPUAsmParser::ParseAsAbsoluteExpression(uint32_t &Ret) {
1825   int64_t Tmp = -1;
1826   if (getLexer().isNot(AsmToken::Integer) && getLexer().isNot(AsmToken::Identifier)) {
1827     return true;
1828   }
1829   if (getParser().parseAbsoluteExpression(Tmp)) {
1830     return true;
1831   }
1832   Ret = static_cast<uint32_t>(Tmp);
1833   return false;
1834 }
1835 
1836 bool AMDGPUAsmParser::ParseDirectiveMajorMinor(uint32_t &Major,
1837                                                uint32_t &Minor) {
1838   if (ParseAsAbsoluteExpression(Major))
1839     return TokError("invalid major version");
1840 
1841   if (getLexer().isNot(AsmToken::Comma))
1842     return TokError("minor version number required, comma expected");
1843   Lex();
1844 
1845   if (ParseAsAbsoluteExpression(Minor))
1846     return TokError("invalid minor version");
1847 
1848   return false;
1849 }
1850 
1851 bool AMDGPUAsmParser::ParseDirectiveHSACodeObjectVersion() {
1852   uint32_t Major;
1853   uint32_t Minor;
1854 
1855   if (ParseDirectiveMajorMinor(Major, Minor))
1856     return true;
1857 
1858   getTargetStreamer().EmitDirectiveHSACodeObjectVersion(Major, Minor);
1859   return false;
1860 }
1861 
1862 bool AMDGPUAsmParser::ParseDirectiveHSACodeObjectISA() {
1863   uint32_t Major;
1864   uint32_t Minor;
1865   uint32_t Stepping;
1866   StringRef VendorName;
1867   StringRef ArchName;
1868 
1869   // If this directive has no arguments, then use the ISA version for the
1870   // targeted GPU.
1871   if (getLexer().is(AsmToken::EndOfStatement)) {
1872     AMDGPU::IsaInfo::IsaVersion ISA =
1873         AMDGPU::IsaInfo::getIsaVersion(getSTI().getFeatureBits());
1874     getTargetStreamer().EmitDirectiveHSACodeObjectISA(ISA.Major, ISA.Minor,
1875                                                       ISA.Stepping,
1876                                                       "AMD", "AMDGPU");
1877     return false;
1878   }
1879 
1880   if (ParseDirectiveMajorMinor(Major, Minor))
1881     return true;
1882 
1883   if (getLexer().isNot(AsmToken::Comma))
1884     return TokError("stepping version number required, comma expected");
1885   Lex();
1886 
1887   if (ParseAsAbsoluteExpression(Stepping))
1888     return TokError("invalid stepping version");
1889 
1890   if (getLexer().isNot(AsmToken::Comma))
1891     return TokError("vendor name required, comma expected");
1892   Lex();
1893 
1894   if (getLexer().isNot(AsmToken::String))
1895     return TokError("invalid vendor name");
1896 
1897   VendorName = getLexer().getTok().getStringContents();
1898   Lex();
1899 
1900   if (getLexer().isNot(AsmToken::Comma))
1901     return TokError("arch name required, comma expected");
1902   Lex();
1903 
1904   if (getLexer().isNot(AsmToken::String))
1905     return TokError("invalid arch name");
1906 
1907   ArchName = getLexer().getTok().getStringContents();
1908   Lex();
1909 
1910   getTargetStreamer().EmitDirectiveHSACodeObjectISA(Major, Minor, Stepping,
1911                                                     VendorName, ArchName);
1912   return false;
1913 }
1914 
1915 bool AMDGPUAsmParser::ParseDirectiveRuntimeMetadata() {
1916   std::string Metadata;
1917   raw_string_ostream MS(Metadata);
1918 
1919   getLexer().setSkipSpace(false);
1920 
1921   bool FoundEnd = false;
1922   while (!getLexer().is(AsmToken::Eof)) {
1923     while (getLexer().is(AsmToken::Space)) {
1924       MS << ' ';
1925       Lex();
1926     }
1927 
1928     if (getLexer().is(AsmToken::Identifier)) {
1929       StringRef ID = getLexer().getTok().getIdentifier();
1930       if (ID == ".end_amdgpu_runtime_metadata") {
1931         Lex();
1932         FoundEnd = true;
1933         break;
1934       }
1935     }
1936 
1937     MS << Parser.parseStringToEndOfStatement()
1938        << getContext().getAsmInfo()->getSeparatorString();
1939 
1940     Parser.eatToEndOfStatement();
1941   }
1942 
1943   getLexer().setSkipSpace(true);
1944 
1945   if (getLexer().is(AsmToken::Eof) && !FoundEnd)
1946     return TokError("expected directive .end_amdgpu_runtime_metadata not found");
1947 
1948   MS.flush();
1949 
1950   getTargetStreamer().EmitRuntimeMetadata(Metadata);
1951 
1952   return false;
1953 }
1954 
1955 bool AMDGPUAsmParser::ParseAMDKernelCodeTValue(StringRef ID,
1956                                                amd_kernel_code_t &Header) {
1957   SmallString<40> ErrStr;
1958   raw_svector_ostream Err(ErrStr);
1959   if (!parseAmdKernelCodeField(ID, getParser(), Header, Err)) {
1960     return TokError(Err.str());
1961   }
1962   Lex();
1963   return false;
1964 }
1965 
1966 bool AMDGPUAsmParser::ParseDirectiveAMDKernelCodeT() {
1967   amd_kernel_code_t Header;
1968   AMDGPU::initDefaultAMDKernelCodeT(Header, getSTI().getFeatureBits());
1969 
1970   while (true) {
1971     // Lex EndOfStatement.  This is in a while loop, because lexing a comment
1972     // will set the current token to EndOfStatement.
1973     while(getLexer().is(AsmToken::EndOfStatement))
1974       Lex();
1975 
1976     if (getLexer().isNot(AsmToken::Identifier))
1977       return TokError("expected value identifier or .end_amd_kernel_code_t");
1978 
1979     StringRef ID = getLexer().getTok().getIdentifier();
1980     Lex();
1981 
1982     if (ID == ".end_amd_kernel_code_t")
1983       break;
1984 
1985     if (ParseAMDKernelCodeTValue(ID, Header))
1986       return true;
1987   }
1988 
1989   getTargetStreamer().EmitAMDKernelCodeT(Header);
1990 
1991   return false;
1992 }
1993 
1994 bool AMDGPUAsmParser::ParseSectionDirectiveHSAText() {
1995   getParser().getStreamer().SwitchSection(
1996       AMDGPU::getHSATextSection(getContext()));
1997   return false;
1998 }
1999 
2000 bool AMDGPUAsmParser::ParseDirectiveAMDGPUHsaKernel() {
2001   if (getLexer().isNot(AsmToken::Identifier))
2002     return TokError("expected symbol name");
2003 
2004   StringRef KernelName = Parser.getTok().getString();
2005 
2006   getTargetStreamer().EmitAMDGPUSymbolType(KernelName,
2007                                            ELF::STT_AMDGPU_HSA_KERNEL);
2008   Lex();
2009   KernelScope.initialize(getContext());
2010   return false;
2011 }
2012 
2013 bool AMDGPUAsmParser::ParseDirectiveAMDGPUHsaModuleGlobal() {
2014   if (getLexer().isNot(AsmToken::Identifier))
2015     return TokError("expected symbol name");
2016 
2017   StringRef GlobalName = Parser.getTok().getIdentifier();
2018 
2019   getTargetStreamer().EmitAMDGPUHsaModuleScopeGlobal(GlobalName);
2020   Lex();
2021   return false;
2022 }
2023 
2024 bool AMDGPUAsmParser::ParseDirectiveAMDGPUHsaProgramGlobal() {
2025   if (getLexer().isNot(AsmToken::Identifier))
2026     return TokError("expected symbol name");
2027 
2028   StringRef GlobalName = Parser.getTok().getIdentifier();
2029 
2030   getTargetStreamer().EmitAMDGPUHsaProgramScopeGlobal(GlobalName);
2031   Lex();
2032   return false;
2033 }
2034 
2035 bool AMDGPUAsmParser::ParseSectionDirectiveHSADataGlobalAgent() {
2036   getParser().getStreamer().SwitchSection(
2037       AMDGPU::getHSADataGlobalAgentSection(getContext()));
2038   return false;
2039 }
2040 
2041 bool AMDGPUAsmParser::ParseSectionDirectiveHSADataGlobalProgram() {
2042   getParser().getStreamer().SwitchSection(
2043       AMDGPU::getHSADataGlobalProgramSection(getContext()));
2044   return false;
2045 }
2046 
2047 bool AMDGPUAsmParser::ParseSectionDirectiveHSARodataReadonlyAgent() {
2048   getParser().getStreamer().SwitchSection(
2049       AMDGPU::getHSARodataReadonlyAgentSection(getContext()));
2050   return false;
2051 }
2052 
2053 bool AMDGPUAsmParser::ParseDirective(AsmToken DirectiveID) {
2054   StringRef IDVal = DirectiveID.getString();
2055 
2056   if (IDVal == ".hsa_code_object_version")
2057     return ParseDirectiveHSACodeObjectVersion();
2058 
2059   if (IDVal == ".hsa_code_object_isa")
2060     return ParseDirectiveHSACodeObjectISA();
2061 
2062   if (IDVal == ".amdgpu_runtime_metadata")
2063     return ParseDirectiveRuntimeMetadata();
2064 
2065   if (IDVal == ".amd_kernel_code_t")
2066     return ParseDirectiveAMDKernelCodeT();
2067 
2068   if (IDVal == ".hsatext")
2069     return ParseSectionDirectiveHSAText();
2070 
2071   if (IDVal == ".amdgpu_hsa_kernel")
2072     return ParseDirectiveAMDGPUHsaKernel();
2073 
2074   if (IDVal == ".amdgpu_hsa_module_global")
2075     return ParseDirectiveAMDGPUHsaModuleGlobal();
2076 
2077   if (IDVal == ".amdgpu_hsa_program_global")
2078     return ParseDirectiveAMDGPUHsaProgramGlobal();
2079 
2080   if (IDVal == ".hsadata_global_agent")
2081     return ParseSectionDirectiveHSADataGlobalAgent();
2082 
2083   if (IDVal == ".hsadata_global_program")
2084     return ParseSectionDirectiveHSADataGlobalProgram();
2085 
2086   if (IDVal == ".hsarodata_readonly_agent")
2087     return ParseSectionDirectiveHSARodataReadonlyAgent();
2088 
2089   return true;
2090 }
2091 
2092 bool AMDGPUAsmParser::subtargetHasRegister(const MCRegisterInfo &MRI,
2093                                            unsigned RegNo) const {
2094   if (isCI())
2095     return true;
2096 
2097   if (isSI()) {
2098     // No flat_scr
2099     switch (RegNo) {
2100     case AMDGPU::FLAT_SCR:
2101     case AMDGPU::FLAT_SCR_LO:
2102     case AMDGPU::FLAT_SCR_HI:
2103       return false;
2104     default:
2105       return true;
2106     }
2107   }
2108 
2109   // VI only has 102 SGPRs, so make sure we aren't trying to use the 2 more that
2110   // SI/CI have.
2111   for (MCRegAliasIterator R(AMDGPU::SGPR102_SGPR103, &MRI, true);
2112        R.isValid(); ++R) {
2113     if (*R == RegNo)
2114       return false;
2115   }
2116 
2117   return true;
2118 }
2119 
2120 OperandMatchResultTy
2121 AMDGPUAsmParser::parseOperand(OperandVector &Operands, StringRef Mnemonic) {
2122   // Try to parse with a custom parser
2123   OperandMatchResultTy ResTy = MatchOperandParserImpl(Operands, Mnemonic);
2124 
2125   // If we successfully parsed the operand or if there as an error parsing,
2126   // we are done.
2127   //
2128   // If we are parsing after we reach EndOfStatement then this means we
2129   // are appending default values to the Operands list.  This is only done
2130   // by custom parser, so we shouldn't continue on to the generic parsing.
2131   if (ResTy == MatchOperand_Success || ResTy == MatchOperand_ParseFail ||
2132       getLexer().is(AsmToken::EndOfStatement))
2133     return ResTy;
2134 
2135   ResTy = parseRegOrImm(Operands);
2136 
2137   if (ResTy == MatchOperand_Success)
2138     return ResTy;
2139 
2140   if (getLexer().getKind() == AsmToken::Identifier) {
2141     // If this identifier is a symbol, we want to create an expression for it.
2142     // It is a little difficult to distinguish between a symbol name, and
2143     // an instruction flag like 'gds'.  In order to do this, we parse
2144     // all tokens as expressions and then treate the symbol name as the token
2145     // string when we want to interpret the operand as a token.
2146     const auto &Tok = Parser.getTok();
2147     SMLoc S = Tok.getLoc();
2148     const MCExpr *Expr = nullptr;
2149     if (!Parser.parseExpression(Expr)) {
2150       Operands.push_back(AMDGPUOperand::CreateExpr(this, Expr, S));
2151       return MatchOperand_Success;
2152     }
2153 
2154     Operands.push_back(AMDGPUOperand::CreateToken(this, Tok.getString(), Tok.getLoc()));
2155     Parser.Lex();
2156     return MatchOperand_Success;
2157   }
2158   return MatchOperand_NoMatch;
2159 }
2160 
2161 StringRef AMDGPUAsmParser::parseMnemonicSuffix(StringRef Name) {
2162   // Clear any forced encodings from the previous instruction.
2163   setForcedEncodingSize(0);
2164   setForcedDPP(false);
2165   setForcedSDWA(false);
2166 
2167   if (Name.endswith("_e64")) {
2168     setForcedEncodingSize(64);
2169     return Name.substr(0, Name.size() - 4);
2170   } else if (Name.endswith("_e32")) {
2171     setForcedEncodingSize(32);
2172     return Name.substr(0, Name.size() - 4);
2173   } else if (Name.endswith("_dpp")) {
2174     setForcedDPP(true);
2175     return Name.substr(0, Name.size() - 4);
2176   } else if (Name.endswith("_sdwa")) {
2177     setForcedSDWA(true);
2178     return Name.substr(0, Name.size() - 5);
2179   }
2180   return Name;
2181 }
2182 
2183 bool AMDGPUAsmParser::ParseInstruction(ParseInstructionInfo &Info,
2184                                        StringRef Name,
2185                                        SMLoc NameLoc, OperandVector &Operands) {
2186   // Add the instruction mnemonic
2187   Name = parseMnemonicSuffix(Name);
2188   Operands.push_back(AMDGPUOperand::CreateToken(this, Name, NameLoc));
2189 
2190   while (!getLexer().is(AsmToken::EndOfStatement)) {
2191     OperandMatchResultTy Res = parseOperand(Operands, Name);
2192 
2193     // Eat the comma or space if there is one.
2194     if (getLexer().is(AsmToken::Comma))
2195       Parser.Lex();
2196 
2197     switch (Res) {
2198       case MatchOperand_Success: break;
2199       case MatchOperand_ParseFail:
2200         Error(getLexer().getLoc(), "failed parsing operand.");
2201         while (!getLexer().is(AsmToken::EndOfStatement)) {
2202           Parser.Lex();
2203         }
2204         return true;
2205       case MatchOperand_NoMatch:
2206         Error(getLexer().getLoc(), "not a valid operand.");
2207         while (!getLexer().is(AsmToken::EndOfStatement)) {
2208           Parser.Lex();
2209         }
2210         return true;
2211     }
2212   }
2213 
2214   return false;
2215 }
2216 
2217 //===----------------------------------------------------------------------===//
2218 // Utility functions
2219 //===----------------------------------------------------------------------===//
2220 
2221 OperandMatchResultTy
2222 AMDGPUAsmParser::parseIntWithPrefix(const char *Prefix, int64_t &Int) {
2223   switch(getLexer().getKind()) {
2224     default: return MatchOperand_NoMatch;
2225     case AsmToken::Identifier: {
2226       StringRef Name = Parser.getTok().getString();
2227       if (!Name.equals(Prefix)) {
2228         return MatchOperand_NoMatch;
2229       }
2230 
2231       Parser.Lex();
2232       if (getLexer().isNot(AsmToken::Colon))
2233         return MatchOperand_ParseFail;
2234 
2235       Parser.Lex();
2236       if (getLexer().isNot(AsmToken::Integer))
2237         return MatchOperand_ParseFail;
2238 
2239       if (getParser().parseAbsoluteExpression(Int))
2240         return MatchOperand_ParseFail;
2241       break;
2242     }
2243   }
2244   return MatchOperand_Success;
2245 }
2246 
2247 OperandMatchResultTy
2248 AMDGPUAsmParser::parseIntWithPrefix(const char *Prefix, OperandVector &Operands,
2249                                     AMDGPUOperand::ImmTy ImmTy,
2250                                     bool (*ConvertResult)(int64_t&)) {
2251   SMLoc S = Parser.getTok().getLoc();
2252   int64_t Value = 0;
2253 
2254   OperandMatchResultTy Res = parseIntWithPrefix(Prefix, Value);
2255   if (Res != MatchOperand_Success)
2256     return Res;
2257 
2258   if (ConvertResult && !ConvertResult(Value)) {
2259     return MatchOperand_ParseFail;
2260   }
2261 
2262   Operands.push_back(AMDGPUOperand::CreateImm(this, Value, S, ImmTy));
2263   return MatchOperand_Success;
2264 }
2265 
2266 OperandMatchResultTy
2267 AMDGPUAsmParser::parseNamedBit(const char *Name, OperandVector &Operands,
2268                                AMDGPUOperand::ImmTy ImmTy) {
2269   int64_t Bit = 0;
2270   SMLoc S = Parser.getTok().getLoc();
2271 
2272   // We are at the end of the statement, and this is a default argument, so
2273   // use a default value.
2274   if (getLexer().isNot(AsmToken::EndOfStatement)) {
2275     switch(getLexer().getKind()) {
2276       case AsmToken::Identifier: {
2277         StringRef Tok = Parser.getTok().getString();
2278         if (Tok == Name) {
2279           Bit = 1;
2280           Parser.Lex();
2281         } else if (Tok.startswith("no") && Tok.endswith(Name)) {
2282           Bit = 0;
2283           Parser.Lex();
2284         } else {
2285           return MatchOperand_NoMatch;
2286         }
2287         break;
2288       }
2289       default:
2290         return MatchOperand_NoMatch;
2291     }
2292   }
2293 
2294   Operands.push_back(AMDGPUOperand::CreateImm(this, Bit, S, ImmTy));
2295   return MatchOperand_Success;
2296 }
2297 
2298 typedef std::map<enum AMDGPUOperand::ImmTy, unsigned> OptionalImmIndexMap;
2299 
2300 static void addOptionalImmOperand(MCInst& Inst, const OperandVector& Operands,
2301                                   OptionalImmIndexMap& OptionalIdx,
2302                                   AMDGPUOperand::ImmTy ImmT,
2303                                   int64_t Default = 0) {
2304   auto i = OptionalIdx.find(ImmT);
2305   if (i != OptionalIdx.end()) {
2306     unsigned Idx = i->second;
2307     ((AMDGPUOperand &)*Operands[Idx]).addImmOperands(Inst, 1);
2308   } else {
2309     Inst.addOperand(MCOperand::createImm(Default));
2310   }
2311 }
2312 
2313 OperandMatchResultTy
2314 AMDGPUAsmParser::parseStringWithPrefix(StringRef Prefix, StringRef &Value) {
2315   if (getLexer().isNot(AsmToken::Identifier)) {
2316     return MatchOperand_NoMatch;
2317   }
2318   StringRef Tok = Parser.getTok().getString();
2319   if (Tok != Prefix) {
2320     return MatchOperand_NoMatch;
2321   }
2322 
2323   Parser.Lex();
2324   if (getLexer().isNot(AsmToken::Colon)) {
2325     return MatchOperand_ParseFail;
2326   }
2327 
2328   Parser.Lex();
2329   if (getLexer().isNot(AsmToken::Identifier)) {
2330     return MatchOperand_ParseFail;
2331   }
2332 
2333   Value = Parser.getTok().getString();
2334   return MatchOperand_Success;
2335 }
2336 
2337 //===----------------------------------------------------------------------===//
2338 // ds
2339 //===----------------------------------------------------------------------===//
2340 
2341 void AMDGPUAsmParser::cvtDSOffset01(MCInst &Inst,
2342                                     const OperandVector &Operands) {
2343   OptionalImmIndexMap OptionalIdx;
2344 
2345   for (unsigned i = 1, e = Operands.size(); i != e; ++i) {
2346     AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[i]);
2347 
2348     // Add the register arguments
2349     if (Op.isReg()) {
2350       Op.addRegOperands(Inst, 1);
2351       continue;
2352     }
2353 
2354     // Handle optional arguments
2355     OptionalIdx[Op.getImmTy()] = i;
2356   }
2357 
2358   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyOffset0);
2359   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyOffset1);
2360   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyGDS);
2361 
2362   Inst.addOperand(MCOperand::createReg(AMDGPU::M0)); // m0
2363 }
2364 
2365 void AMDGPUAsmParser::cvtDSImpl(MCInst &Inst, const OperandVector &Operands,
2366                                 bool IsGdsHardcoded) {
2367   OptionalImmIndexMap OptionalIdx;
2368 
2369   for (unsigned i = 1, e = Operands.size(); i != e; ++i) {
2370     AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[i]);
2371 
2372     // Add the register arguments
2373     if (Op.isReg()) {
2374       Op.addRegOperands(Inst, 1);
2375       continue;
2376     }
2377 
2378     if (Op.isToken() && Op.getToken() == "gds") {
2379       IsGdsHardcoded = true;
2380       continue;
2381     }
2382 
2383     // Handle optional arguments
2384     OptionalIdx[Op.getImmTy()] = i;
2385   }
2386 
2387   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyOffset);
2388   if (!IsGdsHardcoded) {
2389     addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyGDS);
2390   }
2391   Inst.addOperand(MCOperand::createReg(AMDGPU::M0)); // m0
2392 }
2393 
2394 void AMDGPUAsmParser::cvtExp(MCInst &Inst, const OperandVector &Operands) {
2395   OptionalImmIndexMap OptionalIdx;
2396 
2397   unsigned EnMask = 0;
2398   int SrcIdx = 0;
2399 
2400   for (unsigned i = 1, e = Operands.size(); i != e; ++i) {
2401     AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[i]);
2402 
2403     // Add the register arguments
2404     if (Op.isReg()) {
2405       EnMask |= (1 << SrcIdx);
2406       Op.addRegOperands(Inst, 1);
2407       ++SrcIdx;
2408       continue;
2409     }
2410 
2411     if (Op.isOff()) {
2412       ++SrcIdx;
2413       Inst.addOperand(MCOperand::createReg(AMDGPU::NoRegister));
2414       continue;
2415     }
2416 
2417     if (Op.isImm() && Op.getImmTy() == AMDGPUOperand::ImmTyExpTgt) {
2418       Op.addImmOperands(Inst, 1);
2419       continue;
2420     }
2421 
2422     if (Op.isToken() && Op.getToken() == "done")
2423       continue;
2424 
2425     // Handle optional arguments
2426     OptionalIdx[Op.getImmTy()] = i;
2427   }
2428 
2429   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyExpVM);
2430   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyExpCompr);
2431 
2432   Inst.addOperand(MCOperand::createImm(EnMask));
2433 }
2434 
2435 //===----------------------------------------------------------------------===//
2436 // s_waitcnt
2437 //===----------------------------------------------------------------------===//
2438 
2439 bool AMDGPUAsmParser::parseCnt(int64_t &IntVal) {
2440   StringRef CntName = Parser.getTok().getString();
2441   int64_t CntVal;
2442 
2443   Parser.Lex();
2444   if (getLexer().isNot(AsmToken::LParen))
2445     return true;
2446 
2447   Parser.Lex();
2448   if (getLexer().isNot(AsmToken::Integer))
2449     return true;
2450 
2451   if (getParser().parseAbsoluteExpression(CntVal))
2452     return true;
2453 
2454   if (getLexer().isNot(AsmToken::RParen))
2455     return true;
2456 
2457   Parser.Lex();
2458   if (getLexer().is(AsmToken::Amp) || getLexer().is(AsmToken::Comma))
2459     Parser.Lex();
2460 
2461   AMDGPU::IsaInfo::IsaVersion ISA =
2462       AMDGPU::IsaInfo::getIsaVersion(getSTI().getFeatureBits());
2463   if (CntName == "vmcnt")
2464     IntVal = encodeVmcnt(ISA, IntVal, CntVal);
2465   else if (CntName == "expcnt")
2466     IntVal = encodeExpcnt(ISA, IntVal, CntVal);
2467   else if (CntName == "lgkmcnt")
2468     IntVal = encodeLgkmcnt(ISA, IntVal, CntVal);
2469   else
2470     return true;
2471 
2472   return false;
2473 }
2474 
2475 OperandMatchResultTy
2476 AMDGPUAsmParser::parseSWaitCntOps(OperandVector &Operands) {
2477   AMDGPU::IsaInfo::IsaVersion ISA =
2478       AMDGPU::IsaInfo::getIsaVersion(getSTI().getFeatureBits());
2479   int64_t Waitcnt = getWaitcntBitMask(ISA);
2480   SMLoc S = Parser.getTok().getLoc();
2481 
2482   switch(getLexer().getKind()) {
2483     default: return MatchOperand_ParseFail;
2484     case AsmToken::Integer:
2485       // The operand can be an integer value.
2486       if (getParser().parseAbsoluteExpression(Waitcnt))
2487         return MatchOperand_ParseFail;
2488       break;
2489 
2490     case AsmToken::Identifier:
2491       do {
2492         if (parseCnt(Waitcnt))
2493           return MatchOperand_ParseFail;
2494       } while(getLexer().isNot(AsmToken::EndOfStatement));
2495       break;
2496   }
2497   Operands.push_back(AMDGPUOperand::CreateImm(this, Waitcnt, S));
2498   return MatchOperand_Success;
2499 }
2500 
2501 bool AMDGPUAsmParser::parseHwregConstruct(OperandInfoTy &HwReg, int64_t &Offset,
2502                                           int64_t &Width) {
2503   using namespace llvm::AMDGPU::Hwreg;
2504 
2505   if (Parser.getTok().getString() != "hwreg")
2506     return true;
2507   Parser.Lex();
2508 
2509   if (getLexer().isNot(AsmToken::LParen))
2510     return true;
2511   Parser.Lex();
2512 
2513   if (getLexer().is(AsmToken::Identifier)) {
2514     HwReg.IsSymbolic = true;
2515     HwReg.Id = ID_UNKNOWN_;
2516     const StringRef tok = Parser.getTok().getString();
2517     for (int i = ID_SYMBOLIC_FIRST_; i < ID_SYMBOLIC_LAST_; ++i) {
2518       if (tok == IdSymbolic[i]) {
2519         HwReg.Id = i;
2520         break;
2521       }
2522     }
2523     Parser.Lex();
2524   } else {
2525     HwReg.IsSymbolic = false;
2526     if (getLexer().isNot(AsmToken::Integer))
2527       return true;
2528     if (getParser().parseAbsoluteExpression(HwReg.Id))
2529       return true;
2530   }
2531 
2532   if (getLexer().is(AsmToken::RParen)) {
2533     Parser.Lex();
2534     return false;
2535   }
2536 
2537   // optional params
2538   if (getLexer().isNot(AsmToken::Comma))
2539     return true;
2540   Parser.Lex();
2541 
2542   if (getLexer().isNot(AsmToken::Integer))
2543     return true;
2544   if (getParser().parseAbsoluteExpression(Offset))
2545     return true;
2546 
2547   if (getLexer().isNot(AsmToken::Comma))
2548     return true;
2549   Parser.Lex();
2550 
2551   if (getLexer().isNot(AsmToken::Integer))
2552     return true;
2553   if (getParser().parseAbsoluteExpression(Width))
2554     return true;
2555 
2556   if (getLexer().isNot(AsmToken::RParen))
2557     return true;
2558   Parser.Lex();
2559 
2560   return false;
2561 }
2562 
2563 OperandMatchResultTy AMDGPUAsmParser::parseHwreg(OperandVector &Operands) {
2564   using namespace llvm::AMDGPU::Hwreg;
2565 
2566   int64_t Imm16Val = 0;
2567   SMLoc S = Parser.getTok().getLoc();
2568 
2569   switch(getLexer().getKind()) {
2570     default: return MatchOperand_NoMatch;
2571     case AsmToken::Integer:
2572       // The operand can be an integer value.
2573       if (getParser().parseAbsoluteExpression(Imm16Val))
2574         return MatchOperand_NoMatch;
2575       if (Imm16Val < 0 || !isUInt<16>(Imm16Val)) {
2576         Error(S, "invalid immediate: only 16-bit values are legal");
2577         // Do not return error code, but create an imm operand anyway and proceed
2578         // to the next operand, if any. That avoids unneccessary error messages.
2579       }
2580       break;
2581 
2582     case AsmToken::Identifier: {
2583         OperandInfoTy HwReg(ID_UNKNOWN_);
2584         int64_t Offset = OFFSET_DEFAULT_;
2585         int64_t Width = WIDTH_M1_DEFAULT_ + 1;
2586         if (parseHwregConstruct(HwReg, Offset, Width))
2587           return MatchOperand_ParseFail;
2588         if (HwReg.Id < 0 || !isUInt<ID_WIDTH_>(HwReg.Id)) {
2589           if (HwReg.IsSymbolic)
2590             Error(S, "invalid symbolic name of hardware register");
2591           else
2592             Error(S, "invalid code of hardware register: only 6-bit values are legal");
2593         }
2594         if (Offset < 0 || !isUInt<OFFSET_WIDTH_>(Offset))
2595           Error(S, "invalid bit offset: only 5-bit values are legal");
2596         if ((Width-1) < 0 || !isUInt<WIDTH_M1_WIDTH_>(Width-1))
2597           Error(S, "invalid bitfield width: only values from 1 to 32 are legal");
2598         Imm16Val = (HwReg.Id << ID_SHIFT_) | (Offset << OFFSET_SHIFT_) | ((Width-1) << WIDTH_M1_SHIFT_);
2599       }
2600       break;
2601   }
2602   Operands.push_back(AMDGPUOperand::CreateImm(this, Imm16Val, S, AMDGPUOperand::ImmTyHwreg));
2603   return MatchOperand_Success;
2604 }
2605 
2606 bool AMDGPUOperand::isSWaitCnt() const {
2607   return isImm();
2608 }
2609 
2610 bool AMDGPUOperand::isHwreg() const {
2611   return isImmTy(ImmTyHwreg);
2612 }
2613 
2614 bool AMDGPUAsmParser::parseSendMsgConstruct(OperandInfoTy &Msg, OperandInfoTy &Operation, int64_t &StreamId) {
2615   using namespace llvm::AMDGPU::SendMsg;
2616 
2617   if (Parser.getTok().getString() != "sendmsg")
2618     return true;
2619   Parser.Lex();
2620 
2621   if (getLexer().isNot(AsmToken::LParen))
2622     return true;
2623   Parser.Lex();
2624 
2625   if (getLexer().is(AsmToken::Identifier)) {
2626     Msg.IsSymbolic = true;
2627     Msg.Id = ID_UNKNOWN_;
2628     const std::string tok = Parser.getTok().getString();
2629     for (int i = ID_GAPS_FIRST_; i < ID_GAPS_LAST_; ++i) {
2630       switch(i) {
2631         default: continue; // Omit gaps.
2632         case ID_INTERRUPT: case ID_GS: case ID_GS_DONE:  case ID_SYSMSG: break;
2633       }
2634       if (tok == IdSymbolic[i]) {
2635         Msg.Id = i;
2636         break;
2637       }
2638     }
2639     Parser.Lex();
2640   } else {
2641     Msg.IsSymbolic = false;
2642     if (getLexer().isNot(AsmToken::Integer))
2643       return true;
2644     if (getParser().parseAbsoluteExpression(Msg.Id))
2645       return true;
2646     if (getLexer().is(AsmToken::Integer))
2647       if (getParser().parseAbsoluteExpression(Msg.Id))
2648         Msg.Id = ID_UNKNOWN_;
2649   }
2650   if (Msg.Id == ID_UNKNOWN_) // Don't know how to parse the rest.
2651     return false;
2652 
2653   if (!(Msg.Id == ID_GS || Msg.Id == ID_GS_DONE || Msg.Id == ID_SYSMSG)) {
2654     if (getLexer().isNot(AsmToken::RParen))
2655       return true;
2656     Parser.Lex();
2657     return false;
2658   }
2659 
2660   if (getLexer().isNot(AsmToken::Comma))
2661     return true;
2662   Parser.Lex();
2663 
2664   assert(Msg.Id == ID_GS || Msg.Id == ID_GS_DONE || Msg.Id == ID_SYSMSG);
2665   Operation.Id = ID_UNKNOWN_;
2666   if (getLexer().is(AsmToken::Identifier)) {
2667     Operation.IsSymbolic = true;
2668     const char* const *S = (Msg.Id == ID_SYSMSG) ? OpSysSymbolic : OpGsSymbolic;
2669     const int F = (Msg.Id == ID_SYSMSG) ? OP_SYS_FIRST_ : OP_GS_FIRST_;
2670     const int L = (Msg.Id == ID_SYSMSG) ? OP_SYS_LAST_ : OP_GS_LAST_;
2671     const StringRef Tok = Parser.getTok().getString();
2672     for (int i = F; i < L; ++i) {
2673       if (Tok == S[i]) {
2674         Operation.Id = i;
2675         break;
2676       }
2677     }
2678     Parser.Lex();
2679   } else {
2680     Operation.IsSymbolic = false;
2681     if (getLexer().isNot(AsmToken::Integer))
2682       return true;
2683     if (getParser().parseAbsoluteExpression(Operation.Id))
2684       return true;
2685   }
2686 
2687   if ((Msg.Id == ID_GS || Msg.Id == ID_GS_DONE) && Operation.Id != OP_GS_NOP) {
2688     // Stream id is optional.
2689     if (getLexer().is(AsmToken::RParen)) {
2690       Parser.Lex();
2691       return false;
2692     }
2693 
2694     if (getLexer().isNot(AsmToken::Comma))
2695       return true;
2696     Parser.Lex();
2697 
2698     if (getLexer().isNot(AsmToken::Integer))
2699       return true;
2700     if (getParser().parseAbsoluteExpression(StreamId))
2701       return true;
2702   }
2703 
2704   if (getLexer().isNot(AsmToken::RParen))
2705     return true;
2706   Parser.Lex();
2707   return false;
2708 }
2709 
2710 OperandMatchResultTy AMDGPUAsmParser::parseInterpSlot(OperandVector &Operands) {
2711   if (getLexer().getKind() != AsmToken::Identifier)
2712     return MatchOperand_NoMatch;
2713 
2714   StringRef Str = Parser.getTok().getString();
2715   int Slot = StringSwitch<int>(Str)
2716     .Case("p10", 0)
2717     .Case("p20", 1)
2718     .Case("p0", 2)
2719     .Default(-1);
2720 
2721   SMLoc S = Parser.getTok().getLoc();
2722   if (Slot == -1)
2723     return MatchOperand_ParseFail;
2724 
2725   Parser.Lex();
2726   Operands.push_back(AMDGPUOperand::CreateImm(this, Slot, S,
2727                                               AMDGPUOperand::ImmTyInterpSlot));
2728   return MatchOperand_Success;
2729 }
2730 
2731 OperandMatchResultTy AMDGPUAsmParser::parseInterpAttr(OperandVector &Operands) {
2732   if (getLexer().getKind() != AsmToken::Identifier)
2733     return MatchOperand_NoMatch;
2734 
2735   StringRef Str = Parser.getTok().getString();
2736   if (!Str.startswith("attr"))
2737     return MatchOperand_NoMatch;
2738 
2739   StringRef Chan = Str.take_back(2);
2740   int AttrChan = StringSwitch<int>(Chan)
2741     .Case(".x", 0)
2742     .Case(".y", 1)
2743     .Case(".z", 2)
2744     .Case(".w", 3)
2745     .Default(-1);
2746   if (AttrChan == -1)
2747     return MatchOperand_ParseFail;
2748 
2749   Str = Str.drop_back(2).drop_front(4);
2750 
2751   uint8_t Attr;
2752   if (Str.getAsInteger(10, Attr))
2753     return MatchOperand_ParseFail;
2754 
2755   SMLoc S = Parser.getTok().getLoc();
2756   Parser.Lex();
2757   if (Attr > 63) {
2758     Error(S, "out of bounds attr");
2759     return MatchOperand_Success;
2760   }
2761 
2762   SMLoc SChan = SMLoc::getFromPointer(Chan.data());
2763 
2764   Operands.push_back(AMDGPUOperand::CreateImm(this, Attr, S,
2765                                               AMDGPUOperand::ImmTyInterpAttr));
2766   Operands.push_back(AMDGPUOperand::CreateImm(this, AttrChan, SChan,
2767                                               AMDGPUOperand::ImmTyAttrChan));
2768   return MatchOperand_Success;
2769 }
2770 
2771 void AMDGPUAsmParser::errorExpTgt() {
2772   Error(Parser.getTok().getLoc(), "invalid exp target");
2773 }
2774 
2775 OperandMatchResultTy AMDGPUAsmParser::parseExpTgtImpl(StringRef Str,
2776                                                       uint8_t &Val) {
2777   if (Str == "null") {
2778     Val = 9;
2779     return MatchOperand_Success;
2780   }
2781 
2782   if (Str.startswith("mrt")) {
2783     Str = Str.drop_front(3);
2784     if (Str == "z") { // == mrtz
2785       Val = 8;
2786       return MatchOperand_Success;
2787     }
2788 
2789     if (Str.getAsInteger(10, Val))
2790       return MatchOperand_ParseFail;
2791 
2792     if (Val > 7)
2793       errorExpTgt();
2794 
2795     return MatchOperand_Success;
2796   }
2797 
2798   if (Str.startswith("pos")) {
2799     Str = Str.drop_front(3);
2800     if (Str.getAsInteger(10, Val))
2801       return MatchOperand_ParseFail;
2802 
2803     if (Val > 3)
2804       errorExpTgt();
2805 
2806     Val += 12;
2807     return MatchOperand_Success;
2808   }
2809 
2810   if (Str.startswith("param")) {
2811     Str = Str.drop_front(5);
2812     if (Str.getAsInteger(10, Val))
2813       return MatchOperand_ParseFail;
2814 
2815     if (Val >= 32)
2816       errorExpTgt();
2817 
2818     Val += 32;
2819     return MatchOperand_Success;
2820   }
2821 
2822   if (Str.startswith("invalid_target_")) {
2823     Str = Str.drop_front(15);
2824     if (Str.getAsInteger(10, Val))
2825       return MatchOperand_ParseFail;
2826 
2827     errorExpTgt();
2828     return MatchOperand_Success;
2829   }
2830 
2831   return MatchOperand_NoMatch;
2832 }
2833 
2834 OperandMatchResultTy AMDGPUAsmParser::parseExpTgt(OperandVector &Operands) {
2835   uint8_t Val;
2836   StringRef Str = Parser.getTok().getString();
2837 
2838   auto Res = parseExpTgtImpl(Str, Val);
2839   if (Res != MatchOperand_Success)
2840     return Res;
2841 
2842   SMLoc S = Parser.getTok().getLoc();
2843   Parser.Lex();
2844 
2845   Operands.push_back(AMDGPUOperand::CreateImm(this, Val, S,
2846                                               AMDGPUOperand::ImmTyExpTgt));
2847   return MatchOperand_Success;
2848 }
2849 
2850 OperandMatchResultTy
2851 AMDGPUAsmParser::parseSendMsgOp(OperandVector &Operands) {
2852   using namespace llvm::AMDGPU::SendMsg;
2853 
2854   int64_t Imm16Val = 0;
2855   SMLoc S = Parser.getTok().getLoc();
2856 
2857   switch(getLexer().getKind()) {
2858   default:
2859     return MatchOperand_NoMatch;
2860   case AsmToken::Integer:
2861     // The operand can be an integer value.
2862     if (getParser().parseAbsoluteExpression(Imm16Val))
2863       return MatchOperand_NoMatch;
2864     if (Imm16Val < 0 || !isUInt<16>(Imm16Val)) {
2865       Error(S, "invalid immediate: only 16-bit values are legal");
2866       // Do not return error code, but create an imm operand anyway and proceed
2867       // to the next operand, if any. That avoids unneccessary error messages.
2868     }
2869     break;
2870   case AsmToken::Identifier: {
2871       OperandInfoTy Msg(ID_UNKNOWN_);
2872       OperandInfoTy Operation(OP_UNKNOWN_);
2873       int64_t StreamId = STREAM_ID_DEFAULT_;
2874       if (parseSendMsgConstruct(Msg, Operation, StreamId))
2875         return MatchOperand_ParseFail;
2876       do {
2877         // Validate and encode message ID.
2878         if (! ((ID_INTERRUPT <= Msg.Id && Msg.Id <= ID_GS_DONE)
2879                 || Msg.Id == ID_SYSMSG)) {
2880           if (Msg.IsSymbolic)
2881             Error(S, "invalid/unsupported symbolic name of message");
2882           else
2883             Error(S, "invalid/unsupported code of message");
2884           break;
2885         }
2886         Imm16Val = (Msg.Id << ID_SHIFT_);
2887         // Validate and encode operation ID.
2888         if (Msg.Id == ID_GS || Msg.Id == ID_GS_DONE) {
2889           if (! (OP_GS_FIRST_ <= Operation.Id && Operation.Id < OP_GS_LAST_)) {
2890             if (Operation.IsSymbolic)
2891               Error(S, "invalid symbolic name of GS_OP");
2892             else
2893               Error(S, "invalid code of GS_OP: only 2-bit values are legal");
2894             break;
2895           }
2896           if (Operation.Id == OP_GS_NOP
2897               && Msg.Id != ID_GS_DONE) {
2898             Error(S, "invalid GS_OP: NOP is for GS_DONE only");
2899             break;
2900           }
2901           Imm16Val |= (Operation.Id << OP_SHIFT_);
2902         }
2903         if (Msg.Id == ID_SYSMSG) {
2904           if (! (OP_SYS_FIRST_ <= Operation.Id && Operation.Id < OP_SYS_LAST_)) {
2905             if (Operation.IsSymbolic)
2906               Error(S, "invalid/unsupported symbolic name of SYSMSG_OP");
2907             else
2908               Error(S, "invalid/unsupported code of SYSMSG_OP");
2909             break;
2910           }
2911           Imm16Val |= (Operation.Id << OP_SHIFT_);
2912         }
2913         // Validate and encode stream ID.
2914         if ((Msg.Id == ID_GS || Msg.Id == ID_GS_DONE) && Operation.Id != OP_GS_NOP) {
2915           if (! (STREAM_ID_FIRST_ <= StreamId && StreamId < STREAM_ID_LAST_)) {
2916             Error(S, "invalid stream id: only 2-bit values are legal");
2917             break;
2918           }
2919           Imm16Val |= (StreamId << STREAM_ID_SHIFT_);
2920         }
2921       } while (false);
2922     }
2923     break;
2924   }
2925   Operands.push_back(AMDGPUOperand::CreateImm(this, Imm16Val, S, AMDGPUOperand::ImmTySendMsg));
2926   return MatchOperand_Success;
2927 }
2928 
2929 bool AMDGPUOperand::isSendMsg() const {
2930   return isImmTy(ImmTySendMsg);
2931 }
2932 
2933 //===----------------------------------------------------------------------===//
2934 // sopp branch targets
2935 //===----------------------------------------------------------------------===//
2936 
2937 OperandMatchResultTy
2938 AMDGPUAsmParser::parseSOppBrTarget(OperandVector &Operands) {
2939   SMLoc S = Parser.getTok().getLoc();
2940 
2941   switch (getLexer().getKind()) {
2942     default: return MatchOperand_ParseFail;
2943     case AsmToken::Integer: {
2944       int64_t Imm;
2945       if (getParser().parseAbsoluteExpression(Imm))
2946         return MatchOperand_ParseFail;
2947       Operands.push_back(AMDGPUOperand::CreateImm(this, Imm, S));
2948       return MatchOperand_Success;
2949     }
2950 
2951     case AsmToken::Identifier:
2952       Operands.push_back(AMDGPUOperand::CreateExpr(this,
2953           MCSymbolRefExpr::create(getContext().getOrCreateSymbol(
2954                                   Parser.getTok().getString()), getContext()), S));
2955       Parser.Lex();
2956       return MatchOperand_Success;
2957   }
2958 }
2959 
2960 //===----------------------------------------------------------------------===//
2961 // mubuf
2962 //===----------------------------------------------------------------------===//
2963 
2964 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultGLC() const {
2965   return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTyGLC);
2966 }
2967 
2968 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultSLC() const {
2969   return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTySLC);
2970 }
2971 
2972 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultTFE() const {
2973   return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTyTFE);
2974 }
2975 
2976 void AMDGPUAsmParser::cvtMubufImpl(MCInst &Inst,
2977                                const OperandVector &Operands,
2978                                bool IsAtomic, bool IsAtomicReturn) {
2979   OptionalImmIndexMap OptionalIdx;
2980   assert(IsAtomicReturn ? IsAtomic : true);
2981 
2982   for (unsigned i = 1, e = Operands.size(); i != e; ++i) {
2983     AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[i]);
2984 
2985     // Add the register arguments
2986     if (Op.isReg()) {
2987       Op.addRegOperands(Inst, 1);
2988       continue;
2989     }
2990 
2991     // Handle the case where soffset is an immediate
2992     if (Op.isImm() && Op.getImmTy() == AMDGPUOperand::ImmTyNone) {
2993       Op.addImmOperands(Inst, 1);
2994       continue;
2995     }
2996 
2997     // Handle tokens like 'offen' which are sometimes hard-coded into the
2998     // asm string.  There are no MCInst operands for these.
2999     if (Op.isToken()) {
3000       continue;
3001     }
3002     assert(Op.isImm());
3003 
3004     // Handle optional arguments
3005     OptionalIdx[Op.getImmTy()] = i;
3006   }
3007 
3008   // Copy $vdata_in operand and insert as $vdata for MUBUF_Atomic RTN insns.
3009   if (IsAtomicReturn) {
3010     MCInst::iterator I = Inst.begin(); // $vdata_in is always at the beginning.
3011     Inst.insert(I, *I);
3012   }
3013 
3014   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyOffset);
3015   if (!IsAtomic) { // glc is hard-coded.
3016     addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyGLC);
3017   }
3018   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySLC);
3019   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyTFE);
3020 }
3021 
3022 //===----------------------------------------------------------------------===//
3023 // mimg
3024 //===----------------------------------------------------------------------===//
3025 
3026 void AMDGPUAsmParser::cvtMIMG(MCInst &Inst, const OperandVector &Operands) {
3027   unsigned I = 1;
3028   const MCInstrDesc &Desc = MII.get(Inst.getOpcode());
3029   for (unsigned J = 0; J < Desc.getNumDefs(); ++J) {
3030     ((AMDGPUOperand &)*Operands[I++]).addRegOperands(Inst, 1);
3031   }
3032 
3033   OptionalImmIndexMap OptionalIdx;
3034 
3035   for (unsigned E = Operands.size(); I != E; ++I) {
3036     AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[I]);
3037 
3038     // Add the register arguments
3039     if (Op.isRegOrImm()) {
3040       Op.addRegOrImmOperands(Inst, 1);
3041       continue;
3042     } else if (Op.isImmModifier()) {
3043       OptionalIdx[Op.getImmTy()] = I;
3044     } else {
3045       llvm_unreachable("unexpected operand type");
3046     }
3047   }
3048 
3049   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyDMask);
3050   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyUNorm);
3051   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyGLC);
3052   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyDA);
3053   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyR128);
3054   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyTFE);
3055   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyLWE);
3056   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySLC);
3057 }
3058 
3059 void AMDGPUAsmParser::cvtMIMGAtomic(MCInst &Inst, const OperandVector &Operands) {
3060   unsigned I = 1;
3061   const MCInstrDesc &Desc = MII.get(Inst.getOpcode());
3062   for (unsigned J = 0; J < Desc.getNumDefs(); ++J) {
3063     ((AMDGPUOperand &)*Operands[I++]).addRegOperands(Inst, 1);
3064   }
3065 
3066   // Add src, same as dst
3067   ((AMDGPUOperand &)*Operands[I]).addRegOperands(Inst, 1);
3068 
3069   OptionalImmIndexMap OptionalIdx;
3070 
3071   for (unsigned E = Operands.size(); I != E; ++I) {
3072     AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[I]);
3073 
3074     // Add the register arguments
3075     if (Op.isRegOrImm()) {
3076       Op.addRegOrImmOperands(Inst, 1);
3077       continue;
3078     } else if (Op.isImmModifier()) {
3079       OptionalIdx[Op.getImmTy()] = I;
3080     } else {
3081       llvm_unreachable("unexpected operand type");
3082     }
3083   }
3084 
3085   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyDMask);
3086   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyUNorm);
3087   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyGLC);
3088   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyDA);
3089   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyR128);
3090   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyTFE);
3091   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyLWE);
3092   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySLC);
3093 }
3094 
3095 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultDMask() const {
3096   return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTyDMask);
3097 }
3098 
3099 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultUNorm() const {
3100   return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTyUNorm);
3101 }
3102 
3103 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultDA() const {
3104   return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTyDA);
3105 }
3106 
3107 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultR128() const {
3108   return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTyR128);
3109 }
3110 
3111 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultLWE() const {
3112   return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTyLWE);
3113 }
3114 
3115 //===----------------------------------------------------------------------===//
3116 // smrd
3117 //===----------------------------------------------------------------------===//
3118 
3119 bool AMDGPUOperand::isSMRDOffset8() const {
3120   return isImm() && isUInt<8>(getImm());
3121 }
3122 
3123 bool AMDGPUOperand::isSMRDOffset20() const {
3124   return isImm() && isUInt<20>(getImm());
3125 }
3126 
3127 bool AMDGPUOperand::isSMRDLiteralOffset() const {
3128   // 32-bit literals are only supported on CI and we only want to use them
3129   // when the offset is > 8-bits.
3130   return isImm() && !isUInt<8>(getImm()) && isUInt<32>(getImm());
3131 }
3132 
3133 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultSMRDOffset8() const {
3134   return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTyOffset);
3135 }
3136 
3137 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultSMRDOffset20() const {
3138   return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTyOffset);
3139 }
3140 
3141 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultSMRDLiteralOffset() const {
3142   return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTyOffset);
3143 }
3144 
3145 //===----------------------------------------------------------------------===//
3146 // vop3
3147 //===----------------------------------------------------------------------===//
3148 
3149 static bool ConvertOmodMul(int64_t &Mul) {
3150   if (Mul != 1 && Mul != 2 && Mul != 4)
3151     return false;
3152 
3153   Mul >>= 1;
3154   return true;
3155 }
3156 
3157 static bool ConvertOmodDiv(int64_t &Div) {
3158   if (Div == 1) {
3159     Div = 0;
3160     return true;
3161   }
3162 
3163   if (Div == 2) {
3164     Div = 3;
3165     return true;
3166   }
3167 
3168   return false;
3169 }
3170 
3171 static bool ConvertBoundCtrl(int64_t &BoundCtrl) {
3172   if (BoundCtrl == 0) {
3173     BoundCtrl = 1;
3174     return true;
3175   }
3176 
3177   if (BoundCtrl == -1) {
3178     BoundCtrl = 0;
3179     return true;
3180   }
3181 
3182   return false;
3183 }
3184 
3185 // Note: the order in this table matches the order of operands in AsmString.
3186 static const OptionalOperand AMDGPUOptionalOperandTable[] = {
3187   {"offen",   AMDGPUOperand::ImmTyOffen, true, nullptr},
3188   {"idxen",   AMDGPUOperand::ImmTyIdxen, true, nullptr},
3189   {"addr64",  AMDGPUOperand::ImmTyAddr64, true, nullptr},
3190   {"offset0", AMDGPUOperand::ImmTyOffset0, false, nullptr},
3191   {"offset1", AMDGPUOperand::ImmTyOffset1, false, nullptr},
3192   {"gds",     AMDGPUOperand::ImmTyGDS, true, nullptr},
3193   {"offset",  AMDGPUOperand::ImmTyOffset, false, nullptr},
3194   {"glc",     AMDGPUOperand::ImmTyGLC, true, nullptr},
3195   {"slc",     AMDGPUOperand::ImmTySLC, true, nullptr},
3196   {"tfe",     AMDGPUOperand::ImmTyTFE, true, nullptr},
3197   {"clamp",   AMDGPUOperand::ImmTyClampSI, true, nullptr},
3198   {"omod",    AMDGPUOperand::ImmTyOModSI, false, ConvertOmodMul},
3199   {"unorm",   AMDGPUOperand::ImmTyUNorm, true, nullptr},
3200   {"da",      AMDGPUOperand::ImmTyDA,    true, nullptr},
3201   {"r128",    AMDGPUOperand::ImmTyR128,  true, nullptr},
3202   {"lwe",     AMDGPUOperand::ImmTyLWE,   true, nullptr},
3203   {"dmask",   AMDGPUOperand::ImmTyDMask, false, nullptr},
3204   {"row_mask",   AMDGPUOperand::ImmTyDppRowMask, false, nullptr},
3205   {"bank_mask",  AMDGPUOperand::ImmTyDppBankMask, false, nullptr},
3206   {"bound_ctrl", AMDGPUOperand::ImmTyDppBoundCtrl, false, ConvertBoundCtrl},
3207   {"dst_sel",    AMDGPUOperand::ImmTySdwaDstSel, false, nullptr},
3208   {"src0_sel",   AMDGPUOperand::ImmTySdwaSrc0Sel, false, nullptr},
3209   {"src1_sel",   AMDGPUOperand::ImmTySdwaSrc1Sel, false, nullptr},
3210   {"dst_unused", AMDGPUOperand::ImmTySdwaDstUnused, false, nullptr},
3211   {"vm", AMDGPUOperand::ImmTyExpVM, true, nullptr},
3212 };
3213 
3214 OperandMatchResultTy AMDGPUAsmParser::parseOptionalOperand(OperandVector &Operands) {
3215   OperandMatchResultTy res;
3216   for (const OptionalOperand &Op : AMDGPUOptionalOperandTable) {
3217     // try to parse any optional operand here
3218     if (Op.IsBit) {
3219       res = parseNamedBit(Op.Name, Operands, Op.Type);
3220     } else if (Op.Type == AMDGPUOperand::ImmTyOModSI) {
3221       res = parseOModOperand(Operands);
3222     } else if (Op.Type == AMDGPUOperand::ImmTySdwaDstSel ||
3223                Op.Type == AMDGPUOperand::ImmTySdwaSrc0Sel ||
3224                Op.Type == AMDGPUOperand::ImmTySdwaSrc1Sel) {
3225       res = parseSDWASel(Operands, Op.Name, Op.Type);
3226     } else if (Op.Type == AMDGPUOperand::ImmTySdwaDstUnused) {
3227       res = parseSDWADstUnused(Operands);
3228     } else {
3229       res = parseIntWithPrefix(Op.Name, Operands, Op.Type, Op.ConvertResult);
3230     }
3231     if (res != MatchOperand_NoMatch) {
3232       return res;
3233     }
3234   }
3235   return MatchOperand_NoMatch;
3236 }
3237 
3238 OperandMatchResultTy AMDGPUAsmParser::parseOModOperand(OperandVector &Operands) {
3239   StringRef Name = Parser.getTok().getString();
3240   if (Name == "mul") {
3241     return parseIntWithPrefix("mul", Operands,
3242                               AMDGPUOperand::ImmTyOModSI, ConvertOmodMul);
3243   }
3244 
3245   if (Name == "div") {
3246     return parseIntWithPrefix("div", Operands,
3247                               AMDGPUOperand::ImmTyOModSI, ConvertOmodDiv);
3248   }
3249 
3250   return MatchOperand_NoMatch;
3251 }
3252 
3253 void AMDGPUAsmParser::cvtId(MCInst &Inst, const OperandVector &Operands) {
3254   unsigned I = 1;
3255   const MCInstrDesc &Desc = MII.get(Inst.getOpcode());
3256   for (unsigned J = 0; J < Desc.getNumDefs(); ++J) {
3257     ((AMDGPUOperand &)*Operands[I++]).addRegOperands(Inst, 1);
3258   }
3259   for (unsigned E = Operands.size(); I != E; ++I)
3260     ((AMDGPUOperand &)*Operands[I]).addRegOrImmOperands(Inst, 1);
3261 }
3262 
3263 void AMDGPUAsmParser::cvtVOP3_2_mod(MCInst &Inst, const OperandVector &Operands) {
3264   uint64_t TSFlags = MII.get(Inst.getOpcode()).TSFlags;
3265   if (TSFlags & SIInstrFlags::VOP3) {
3266     cvtVOP3(Inst, Operands);
3267   } else {
3268     cvtId(Inst, Operands);
3269   }
3270 }
3271 
3272 static bool isRegOrImmWithInputMods(const MCInstrDesc &Desc, unsigned OpNum) {
3273       // 1. This operand is input modifiers
3274   return Desc.OpInfo[OpNum].OperandType == AMDGPU::OPERAND_INPUT_MODS
3275       // 2. This is not last operand
3276       && Desc.NumOperands > (OpNum + 1)
3277       // 3. Next operand is register class
3278       && Desc.OpInfo[OpNum + 1].RegClass != -1
3279       // 4. Next register is not tied to any other operand
3280       && Desc.getOperandConstraint(OpNum + 1, MCOI::OperandConstraint::TIED_TO) == -1;
3281 }
3282 
3283 void AMDGPUAsmParser::cvtVOP3(MCInst &Inst, const OperandVector &Operands) {
3284   OptionalImmIndexMap OptionalIdx;
3285   unsigned I = 1;
3286   const MCInstrDesc &Desc = MII.get(Inst.getOpcode());
3287   for (unsigned J = 0; J < Desc.getNumDefs(); ++J) {
3288     ((AMDGPUOperand &)*Operands[I++]).addRegOperands(Inst, 1);
3289   }
3290 
3291   for (unsigned E = Operands.size(); I != E; ++I) {
3292     AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[I]);
3293     if (isRegOrImmWithInputMods(Desc, Inst.getNumOperands())) {
3294       Op.addRegOrImmWithFPInputModsOperands(Inst, 2);
3295     } else if (Op.isImm()) {
3296       OptionalIdx[Op.getImmTy()] = I;
3297     } else {
3298       llvm_unreachable("unhandled operand type");
3299     }
3300   }
3301 
3302   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyClampSI);
3303   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyOModSI);
3304 
3305   // special case v_mac_{f16, f32}:
3306   // it has src2 register operand that is tied to dst operand
3307   // we don't allow modifiers for this operand in assembler so src2_modifiers
3308   // should be 0
3309   if (Inst.getOpcode() == AMDGPU::V_MAC_F32_e64_si ||
3310       Inst.getOpcode() == AMDGPU::V_MAC_F32_e64_vi ||
3311       Inst.getOpcode() == AMDGPU::V_MAC_F16_e64_vi) {
3312     auto it = Inst.begin();
3313     std::advance(
3314       it,
3315       AMDGPU::getNamedOperandIdx(Inst.getOpcode() == AMDGPU::V_MAC_F16_e64_vi ?
3316                                      AMDGPU::V_MAC_F16_e64 :
3317                                      AMDGPU::V_MAC_F32_e64,
3318                                  AMDGPU::OpName::src2_modifiers));
3319     it = Inst.insert(it, MCOperand::createImm(0)); // no modifiers for src2
3320     ++it;
3321     Inst.insert(it, Inst.getOperand(0)); // src2 = dst
3322   }
3323 }
3324 
3325 //===----------------------------------------------------------------------===//
3326 // dpp
3327 //===----------------------------------------------------------------------===//
3328 
3329 bool AMDGPUOperand::isDPPCtrl() const {
3330   bool result = isImm() && getImmTy() == ImmTyDppCtrl && isUInt<9>(getImm());
3331   if (result) {
3332     int64_t Imm = getImm();
3333     return ((Imm >= 0x000) && (Imm <= 0x0ff)) ||
3334            ((Imm >= 0x101) && (Imm <= 0x10f)) ||
3335            ((Imm >= 0x111) && (Imm <= 0x11f)) ||
3336            ((Imm >= 0x121) && (Imm <= 0x12f)) ||
3337            (Imm == 0x130) ||
3338            (Imm == 0x134) ||
3339            (Imm == 0x138) ||
3340            (Imm == 0x13c) ||
3341            (Imm == 0x140) ||
3342            (Imm == 0x141) ||
3343            (Imm == 0x142) ||
3344            (Imm == 0x143);
3345   }
3346   return false;
3347 }
3348 
3349 bool AMDGPUOperand::isGPRIdxMode() const {
3350   return isImm() && isUInt<4>(getImm());
3351 }
3352 
3353 OperandMatchResultTy
3354 AMDGPUAsmParser::parseDPPCtrl(OperandVector &Operands) {
3355   SMLoc S = Parser.getTok().getLoc();
3356   StringRef Prefix;
3357   int64_t Int;
3358 
3359   if (getLexer().getKind() == AsmToken::Identifier) {
3360     Prefix = Parser.getTok().getString();
3361   } else {
3362     return MatchOperand_NoMatch;
3363   }
3364 
3365   if (Prefix == "row_mirror") {
3366     Int = 0x140;
3367     Parser.Lex();
3368   } else if (Prefix == "row_half_mirror") {
3369     Int = 0x141;
3370     Parser.Lex();
3371   } else {
3372     // Check to prevent parseDPPCtrlOps from eating invalid tokens
3373     if (Prefix != "quad_perm"
3374         && Prefix != "row_shl"
3375         && Prefix != "row_shr"
3376         && Prefix != "row_ror"
3377         && Prefix != "wave_shl"
3378         && Prefix != "wave_rol"
3379         && Prefix != "wave_shr"
3380         && Prefix != "wave_ror"
3381         && Prefix != "row_bcast") {
3382       return MatchOperand_NoMatch;
3383     }
3384 
3385     Parser.Lex();
3386     if (getLexer().isNot(AsmToken::Colon))
3387       return MatchOperand_ParseFail;
3388 
3389     if (Prefix == "quad_perm") {
3390       // quad_perm:[%d,%d,%d,%d]
3391       Parser.Lex();
3392       if (getLexer().isNot(AsmToken::LBrac))
3393         return MatchOperand_ParseFail;
3394       Parser.Lex();
3395 
3396       if (getParser().parseAbsoluteExpression(Int) || !(0 <= Int && Int <=3))
3397         return MatchOperand_ParseFail;
3398 
3399       for (int i = 0; i < 3; ++i) {
3400         if (getLexer().isNot(AsmToken::Comma))
3401           return MatchOperand_ParseFail;
3402         Parser.Lex();
3403 
3404         int64_t Temp;
3405         if (getParser().parseAbsoluteExpression(Temp) || !(0 <= Temp && Temp <=3))
3406           return MatchOperand_ParseFail;
3407         const int shift = i*2 + 2;
3408         Int += (Temp << shift);
3409       }
3410 
3411       if (getLexer().isNot(AsmToken::RBrac))
3412         return MatchOperand_ParseFail;
3413       Parser.Lex();
3414 
3415     } else {
3416       // sel:%d
3417       Parser.Lex();
3418       if (getParser().parseAbsoluteExpression(Int))
3419         return MatchOperand_ParseFail;
3420 
3421       if (Prefix == "row_shl" && 1 <= Int && Int <= 15) {
3422         Int |= 0x100;
3423       } else if (Prefix == "row_shr" && 1 <= Int && Int <= 15) {
3424         Int |= 0x110;
3425       } else if (Prefix == "row_ror" && 1 <= Int && Int <= 15) {
3426         Int |= 0x120;
3427       } else if (Prefix == "wave_shl" && 1 == Int) {
3428         Int = 0x130;
3429       } else if (Prefix == "wave_rol" && 1 == Int) {
3430         Int = 0x134;
3431       } else if (Prefix == "wave_shr" && 1 == Int) {
3432         Int = 0x138;
3433       } else if (Prefix == "wave_ror" && 1 == Int) {
3434         Int = 0x13C;
3435       } else if (Prefix == "row_bcast") {
3436         if (Int == 15) {
3437           Int = 0x142;
3438         } else if (Int == 31) {
3439           Int = 0x143;
3440         } else {
3441           return MatchOperand_ParseFail;
3442         }
3443       } else {
3444         return MatchOperand_ParseFail;
3445       }
3446     }
3447   }
3448 
3449   Operands.push_back(AMDGPUOperand::CreateImm(this, Int, S, AMDGPUOperand::ImmTyDppCtrl));
3450   return MatchOperand_Success;
3451 }
3452 
3453 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultRowMask() const {
3454   return AMDGPUOperand::CreateImm(this, 0xf, SMLoc(), AMDGPUOperand::ImmTyDppRowMask);
3455 }
3456 
3457 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultBankMask() const {
3458   return AMDGPUOperand::CreateImm(this, 0xf, SMLoc(), AMDGPUOperand::ImmTyDppBankMask);
3459 }
3460 
3461 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultBoundCtrl() const {
3462   return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTyDppBoundCtrl);
3463 }
3464 
3465 void AMDGPUAsmParser::cvtDPP(MCInst &Inst, const OperandVector &Operands) {
3466   OptionalImmIndexMap OptionalIdx;
3467 
3468   unsigned I = 1;
3469   const MCInstrDesc &Desc = MII.get(Inst.getOpcode());
3470   for (unsigned J = 0; J < Desc.getNumDefs(); ++J) {
3471     ((AMDGPUOperand &)*Operands[I++]).addRegOperands(Inst, 1);
3472   }
3473 
3474   for (unsigned E = Operands.size(); I != E; ++I) {
3475     AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[I]);
3476     // Add the register arguments
3477     if (Op.isReg() && Op.Reg.RegNo == AMDGPU::VCC) {
3478       // VOP2b (v_add_u32, v_sub_u32 ...) dpp use "vcc" token.
3479       // Skip it.
3480       continue;
3481     } if (isRegOrImmWithInputMods(Desc, Inst.getNumOperands())) {
3482       Op.addRegWithFPInputModsOperands(Inst, 2);
3483     } else if (Op.isDPPCtrl()) {
3484       Op.addImmOperands(Inst, 1);
3485     } else if (Op.isImm()) {
3486       // Handle optional arguments
3487       OptionalIdx[Op.getImmTy()] = I;
3488     } else {
3489       llvm_unreachable("Invalid operand type");
3490     }
3491   }
3492 
3493   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyDppRowMask, 0xf);
3494   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyDppBankMask, 0xf);
3495   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyDppBoundCtrl);
3496 
3497   // special case v_mac_{f16, f32}:
3498   // it has src2 register operand that is tied to dst operand
3499   if (Inst.getOpcode() == AMDGPU::V_MAC_F32_dpp ||
3500       Inst.getOpcode() == AMDGPU::V_MAC_F16_dpp) {
3501     auto it = Inst.begin();
3502     std::advance(
3503         it, AMDGPU::getNamedOperandIdx(Inst.getOpcode(), AMDGPU::OpName::src2));
3504     Inst.insert(it, Inst.getOperand(0)); // src2 = dst
3505   }
3506 }
3507 
3508 //===----------------------------------------------------------------------===//
3509 // sdwa
3510 //===----------------------------------------------------------------------===//
3511 
3512 OperandMatchResultTy
3513 AMDGPUAsmParser::parseSDWASel(OperandVector &Operands, StringRef Prefix,
3514                               AMDGPUOperand::ImmTy Type) {
3515   using namespace llvm::AMDGPU::SDWA;
3516 
3517   SMLoc S = Parser.getTok().getLoc();
3518   StringRef Value;
3519   OperandMatchResultTy res;
3520 
3521   res = parseStringWithPrefix(Prefix, Value);
3522   if (res != MatchOperand_Success) {
3523     return res;
3524   }
3525 
3526   int64_t Int;
3527   Int = StringSwitch<int64_t>(Value)
3528         .Case("BYTE_0", SdwaSel::BYTE_0)
3529         .Case("BYTE_1", SdwaSel::BYTE_1)
3530         .Case("BYTE_2", SdwaSel::BYTE_2)
3531         .Case("BYTE_3", SdwaSel::BYTE_3)
3532         .Case("WORD_0", SdwaSel::WORD_0)
3533         .Case("WORD_1", SdwaSel::WORD_1)
3534         .Case("DWORD", SdwaSel::DWORD)
3535         .Default(0xffffffff);
3536   Parser.Lex(); // eat last token
3537 
3538   if (Int == 0xffffffff) {
3539     return MatchOperand_ParseFail;
3540   }
3541 
3542   Operands.push_back(AMDGPUOperand::CreateImm(this, Int, S, Type));
3543   return MatchOperand_Success;
3544 }
3545 
3546 OperandMatchResultTy
3547 AMDGPUAsmParser::parseSDWADstUnused(OperandVector &Operands) {
3548   using namespace llvm::AMDGPU::SDWA;
3549 
3550   SMLoc S = Parser.getTok().getLoc();
3551   StringRef Value;
3552   OperandMatchResultTy res;
3553 
3554   res = parseStringWithPrefix("dst_unused", Value);
3555   if (res != MatchOperand_Success) {
3556     return res;
3557   }
3558 
3559   int64_t Int;
3560   Int = StringSwitch<int64_t>(Value)
3561         .Case("UNUSED_PAD", DstUnused::UNUSED_PAD)
3562         .Case("UNUSED_SEXT", DstUnused::UNUSED_SEXT)
3563         .Case("UNUSED_PRESERVE", DstUnused::UNUSED_PRESERVE)
3564         .Default(0xffffffff);
3565   Parser.Lex(); // eat last token
3566 
3567   if (Int == 0xffffffff) {
3568     return MatchOperand_ParseFail;
3569   }
3570 
3571   Operands.push_back(AMDGPUOperand::CreateImm(this, Int, S, AMDGPUOperand::ImmTySdwaDstUnused));
3572   return MatchOperand_Success;
3573 }
3574 
3575 void AMDGPUAsmParser::cvtSdwaVOP1(MCInst &Inst, const OperandVector &Operands) {
3576   cvtSDWA(Inst, Operands, SIInstrFlags::VOP1);
3577 }
3578 
3579 void AMDGPUAsmParser::cvtSdwaVOP2(MCInst &Inst, const OperandVector &Operands) {
3580   cvtSDWA(Inst, Operands, SIInstrFlags::VOP2);
3581 }
3582 
3583 void AMDGPUAsmParser::cvtSdwaVOPC(MCInst &Inst, const OperandVector &Operands) {
3584   cvtSDWA(Inst, Operands, SIInstrFlags::VOPC);
3585 }
3586 
3587 void AMDGPUAsmParser::cvtSDWA(MCInst &Inst, const OperandVector &Operands,
3588                               uint64_t BasicInstType) {
3589   using namespace llvm::AMDGPU::SDWA;
3590   OptionalImmIndexMap OptionalIdx;
3591 
3592   unsigned I = 1;
3593   const MCInstrDesc &Desc = MII.get(Inst.getOpcode());
3594   for (unsigned J = 0; J < Desc.getNumDefs(); ++J) {
3595     ((AMDGPUOperand &)*Operands[I++]).addRegOperands(Inst, 1);
3596   }
3597 
3598   for (unsigned E = Operands.size(); I != E; ++I) {
3599     AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[I]);
3600     // Add the register arguments
3601     if ((BasicInstType == SIInstrFlags::VOPC ||
3602          BasicInstType == SIInstrFlags::VOP2)&&
3603         Op.isReg() &&
3604         Op.Reg.RegNo == AMDGPU::VCC) {
3605       // VOPC and VOP2b (v_add_u32, v_sub_u32 ...) sdwa use "vcc" token as dst.
3606       // Skip it.
3607       continue;
3608     } else if (isRegOrImmWithInputMods(Desc, Inst.getNumOperands())) {
3609       Op.addRegWithInputModsOperands(Inst, 2);
3610     } else if (Op.isImm()) {
3611       // Handle optional arguments
3612       OptionalIdx[Op.getImmTy()] = I;
3613     } else {
3614       llvm_unreachable("Invalid operand type");
3615     }
3616   }
3617 
3618   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyClampSI, 0);
3619 
3620   if (Inst.getOpcode() != AMDGPU::V_NOP_sdwa_vi) {
3621     // V_NOP_sdwa_vi has no optional sdwa arguments
3622     switch (BasicInstType) {
3623     case SIInstrFlags::VOP1:
3624       addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySdwaDstSel, SdwaSel::DWORD);
3625       addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySdwaDstUnused, DstUnused::UNUSED_PRESERVE);
3626       addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySdwaSrc0Sel, SdwaSel::DWORD);
3627       break;
3628 
3629     case SIInstrFlags::VOP2:
3630       addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySdwaDstSel, SdwaSel::DWORD);
3631       addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySdwaDstUnused, DstUnused::UNUSED_PRESERVE);
3632       addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySdwaSrc0Sel, SdwaSel::DWORD);
3633       addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySdwaSrc1Sel, SdwaSel::DWORD);
3634       break;
3635 
3636     case SIInstrFlags::VOPC:
3637       addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySdwaSrc0Sel, SdwaSel::DWORD);
3638       addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySdwaSrc1Sel, SdwaSel::DWORD);
3639       break;
3640 
3641     default:
3642       llvm_unreachable("Invalid instruction type. Only VOP1, VOP2 and VOPC allowed");
3643     }
3644   }
3645 
3646   // special case v_mac_{f16, f32}:
3647   // it has src2 register operand that is tied to dst operand
3648   if (Inst.getOpcode() == AMDGPU::V_MAC_F32_sdwa_vi ||
3649       Inst.getOpcode() == AMDGPU::V_MAC_F16_sdwa_vi)  {
3650     auto it = Inst.begin();
3651     std::advance(
3652         it, AMDGPU::getNamedOperandIdx(Inst.getOpcode(), AMDGPU::OpName::src2));
3653     Inst.insert(it, Inst.getOperand(0)); // src2 = dst
3654   }
3655 
3656 }
3657 
3658 /// Force static initialization.
3659 extern "C" void LLVMInitializeAMDGPUAsmParser() {
3660   RegisterMCAsmParser<AMDGPUAsmParser> A(getTheAMDGPUTarget());
3661   RegisterMCAsmParser<AMDGPUAsmParser> B(getTheGCNTarget());
3662 }
3663 
3664 #define GET_REGISTER_MATCHER
3665 #define GET_MATCHER_IMPLEMENTATION
3666 #include "AMDGPUGenAsmMatcher.inc"
3667 
3668 // This fuction should be defined after auto-generated include so that we have
3669 // MatchClassKind enum defined
3670 unsigned AMDGPUAsmParser::validateTargetOperandClass(MCParsedAsmOperand &Op,
3671                                                      unsigned Kind) {
3672   // Tokens like "glc" would be parsed as immediate operands in ParseOperand().
3673   // But MatchInstructionImpl() expects to meet token and fails to validate
3674   // operand. This method checks if we are given immediate operand but expect to
3675   // get corresponding token.
3676   AMDGPUOperand &Operand = (AMDGPUOperand&)Op;
3677   switch (Kind) {
3678   case MCK_addr64:
3679     return Operand.isAddr64() ? Match_Success : Match_InvalidOperand;
3680   case MCK_gds:
3681     return Operand.isGDS() ? Match_Success : Match_InvalidOperand;
3682   case MCK_glc:
3683     return Operand.isGLC() ? Match_Success : Match_InvalidOperand;
3684   case MCK_idxen:
3685     return Operand.isIdxen() ? Match_Success : Match_InvalidOperand;
3686   case MCK_offen:
3687     return Operand.isOffen() ? Match_Success : Match_InvalidOperand;
3688   case MCK_SSrcB32:
3689     // When operands have expression values, they will return true for isToken,
3690     // because it is not possible to distinguish between a token and an
3691     // expression at parse time. MatchInstructionImpl() will always try to
3692     // match an operand as a token, when isToken returns true, and when the
3693     // name of the expression is not a valid token, the match will fail,
3694     // so we need to handle it here.
3695     return Operand.isSSrcB32() ? Match_Success : Match_InvalidOperand;
3696   case MCK_SSrcF32:
3697     return Operand.isSSrcF32() ? Match_Success : Match_InvalidOperand;
3698   case MCK_SoppBrTarget:
3699     return Operand.isSoppBrTarget() ? Match_Success : Match_InvalidOperand;
3700   case MCK_VReg32OrOff:
3701     return Operand.isVReg32OrOff() ? Match_Success : Match_InvalidOperand;
3702   case MCK_InterpSlot:
3703     return Operand.isInterpSlot() ? Match_Success : Match_InvalidOperand;
3704   case MCK_Attr:
3705     return Operand.isInterpAttr() ? Match_Success : Match_InvalidOperand;
3706   case MCK_AttrChan:
3707     return Operand.isAttrChan() ? Match_Success : Match_InvalidOperand;
3708   default:
3709     return Match_InvalidOperand;
3710   }
3711 }
3712