1 //===-- AMDGPUAsmParser.cpp - Parse SI asm to MCInst instructions ---------===//
2 //
3 //                     The LLVM Compiler Infrastructure
4 //
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
7 //
8 //===----------------------------------------------------------------------===//
9 
10 #include "AMDKernelCodeT.h"
11 #include "MCTargetDesc/AMDGPUMCTargetDesc.h"
12 #include "MCTargetDesc/AMDGPUTargetStreamer.h"
13 #include "SIDefines.h"
14 #include "Utils/AMDGPUBaseInfo.h"
15 #include "Utils/AMDKernelCodeTUtils.h"
16 #include "Utils/AMDGPUAsmUtils.h"
17 #include "llvm/ADT/APFloat.h"
18 #include "llvm/ADT/APInt.h"
19 #include "llvm/ADT/ArrayRef.h"
20 #include "llvm/ADT/SmallBitVector.h"
21 #include "llvm/ADT/SmallString.h"
22 #include "llvm/ADT/STLExtras.h"
23 #include "llvm/ADT/StringRef.h"
24 #include "llvm/ADT/StringSwitch.h"
25 #include "llvm/ADT/Twine.h"
26 #include "llvm/CodeGen/MachineValueType.h"
27 #include "llvm/MC/MCAsmInfo.h"
28 #include "llvm/MC/MCContext.h"
29 #include "llvm/MC/MCExpr.h"
30 #include "llvm/MC/MCInst.h"
31 #include "llvm/MC/MCInstrDesc.h"
32 #include "llvm/MC/MCInstrInfo.h"
33 #include "llvm/MC/MCParser/MCAsmLexer.h"
34 #include "llvm/MC/MCParser/MCAsmParser.h"
35 #include "llvm/MC/MCParser/MCAsmParserExtension.h"
36 #include "llvm/MC/MCParser/MCParsedAsmOperand.h"
37 #include "llvm/MC/MCParser/MCTargetAsmParser.h"
38 #include "llvm/MC/MCRegisterInfo.h"
39 #include "llvm/MC/MCStreamer.h"
40 #include "llvm/MC/MCSubtargetInfo.h"
41 #include "llvm/MC/MCSymbol.h"
42 #include "llvm/Support/Casting.h"
43 #include "llvm/Support/ELF.h"
44 #include "llvm/Support/ErrorHandling.h"
45 #include "llvm/Support/MathExtras.h"
46 #include "llvm/Support/raw_ostream.h"
47 #include "llvm/Support/SMLoc.h"
48 #include "llvm/Support/TargetRegistry.h"
49 #include <algorithm>
50 #include <cassert>
51 #include <cstdint>
52 #include <cstring>
53 #include <iterator>
54 #include <map>
55 #include <memory>
56 #include <string>
57 
58 using namespace llvm;
59 using namespace llvm::AMDGPU;
60 
61 namespace {
62 
63 class AMDGPUAsmParser;
64 
65 enum RegisterKind { IS_UNKNOWN, IS_VGPR, IS_SGPR, IS_TTMP, IS_SPECIAL };
66 
67 //===----------------------------------------------------------------------===//
68 // Operand
69 //===----------------------------------------------------------------------===//
70 
71 class AMDGPUOperand : public MCParsedAsmOperand {
72   enum KindTy {
73     Token,
74     Immediate,
75     Register,
76     Expression
77   } Kind;
78 
79   SMLoc StartLoc, EndLoc;
80   const AMDGPUAsmParser *AsmParser;
81 
82 public:
83   AMDGPUOperand(enum KindTy Kind_, const AMDGPUAsmParser *AsmParser_)
84     : MCParsedAsmOperand(), Kind(Kind_), AsmParser(AsmParser_) {}
85 
86   typedef std::unique_ptr<AMDGPUOperand> Ptr;
87 
88   struct Modifiers {
89     bool Abs = false;
90     bool Neg = false;
91     bool Sext = false;
92 
93     bool hasFPModifiers() const { return Abs || Neg; }
94     bool hasIntModifiers() const { return Sext; }
95     bool hasModifiers() const { return hasFPModifiers() || hasIntModifiers(); }
96 
97     int64_t getFPModifiersOperand() const {
98       int64_t Operand = 0;
99       Operand |= Abs ? SISrcMods::ABS : 0;
100       Operand |= Neg ? SISrcMods::NEG : 0;
101       return Operand;
102     }
103 
104     int64_t getIntModifiersOperand() const {
105       int64_t Operand = 0;
106       Operand |= Sext ? SISrcMods::SEXT : 0;
107       return Operand;
108     }
109 
110     int64_t getModifiersOperand() const {
111       assert(!(hasFPModifiers() && hasIntModifiers())
112            && "fp and int modifiers should not be used simultaneously");
113       if (hasFPModifiers()) {
114         return getFPModifiersOperand();
115       } else if (hasIntModifiers()) {
116         return getIntModifiersOperand();
117       } else {
118         return 0;
119       }
120     }
121 
122     friend raw_ostream &operator <<(raw_ostream &OS, AMDGPUOperand::Modifiers Mods);
123   };
124 
125   enum ImmTy {
126     ImmTyNone,
127     ImmTyGDS,
128     ImmTyOffen,
129     ImmTyIdxen,
130     ImmTyAddr64,
131     ImmTyOffset,
132     ImmTyOffset0,
133     ImmTyOffset1,
134     ImmTyGLC,
135     ImmTySLC,
136     ImmTyTFE,
137     ImmTyClampSI,
138     ImmTyOModSI,
139     ImmTyDppCtrl,
140     ImmTyDppRowMask,
141     ImmTyDppBankMask,
142     ImmTyDppBoundCtrl,
143     ImmTySdwaDstSel,
144     ImmTySdwaSrc0Sel,
145     ImmTySdwaSrc1Sel,
146     ImmTySdwaDstUnused,
147     ImmTyDMask,
148     ImmTyUNorm,
149     ImmTyDA,
150     ImmTyR128,
151     ImmTyLWE,
152     ImmTyExpTgt,
153     ImmTyExpCompr,
154     ImmTyExpVM,
155     ImmTyHwreg,
156     ImmTyOff,
157     ImmTySendMsg,
158     ImmTyInterpSlot,
159     ImmTyInterpAttr,
160     ImmTyAttrChan
161   };
162 
163   struct TokOp {
164     const char *Data;
165     unsigned Length;
166   };
167 
168   struct ImmOp {
169     int64_t Val;
170     ImmTy Type;
171     bool IsFPImm;
172     Modifiers Mods;
173   };
174 
175   struct RegOp {
176     unsigned RegNo;
177     bool IsForcedVOP3;
178     Modifiers Mods;
179   };
180 
181   union {
182     TokOp Tok;
183     ImmOp Imm;
184     RegOp Reg;
185     const MCExpr *Expr;
186   };
187 
188   bool isToken() const override {
189     if (Kind == Token)
190       return true;
191 
192     if (Kind != Expression || !Expr)
193       return false;
194 
195     // When parsing operands, we can't always tell if something was meant to be
196     // a token, like 'gds', or an expression that references a global variable.
197     // In this case, we assume the string is an expression, and if we need to
198     // interpret is a token, then we treat the symbol name as the token.
199     return isa<MCSymbolRefExpr>(Expr);
200   }
201 
202   bool isImm() const override {
203     return Kind == Immediate;
204   }
205 
206   bool isInlinableImm(MVT type) const;
207   bool isLiteralImm(MVT type) const;
208 
209   bool isRegKind() const {
210     return Kind == Register;
211   }
212 
213   bool isReg() const override {
214     return isRegKind() && !hasModifiers();
215   }
216 
217   bool isRegOrImmWithInputMods(MVT type) const {
218     return isRegKind() || isInlinableImm(type);
219   }
220 
221   bool isRegOrImmWithInt16InputMods() const {
222     return isRegOrImmWithInputMods(MVT::i16);
223   }
224 
225   bool isRegOrImmWithInt32InputMods() const {
226     return isRegOrImmWithInputMods(MVT::i32);
227   }
228 
229   bool isRegOrImmWithInt64InputMods() const {
230     return isRegOrImmWithInputMods(MVT::i64);
231   }
232 
233   bool isRegOrImmWithFP16InputMods() const {
234     return isRegOrImmWithInputMods(MVT::f16);
235   }
236 
237   bool isRegOrImmWithFP32InputMods() const {
238     return isRegOrImmWithInputMods(MVT::f32);
239   }
240 
241   bool isRegOrImmWithFP64InputMods() const {
242     return isRegOrImmWithInputMods(MVT::f64);
243   }
244 
245   bool isVReg() const {
246     return isRegClass(AMDGPU::VGPR_32RegClassID) ||
247            isRegClass(AMDGPU::VReg_64RegClassID) ||
248            isRegClass(AMDGPU::VReg_96RegClassID) ||
249            isRegClass(AMDGPU::VReg_128RegClassID) ||
250            isRegClass(AMDGPU::VReg_256RegClassID) ||
251            isRegClass(AMDGPU::VReg_512RegClassID);
252   }
253 
254   bool isVReg32OrOff() const {
255     return isOff() || isRegClass(AMDGPU::VGPR_32RegClassID);
256   }
257 
258   bool isImmTy(ImmTy ImmT) const {
259     return isImm() && Imm.Type == ImmT;
260   }
261 
262   bool isImmModifier() const {
263     return isImm() && Imm.Type != ImmTyNone;
264   }
265 
266   bool isClampSI() const { return isImmTy(ImmTyClampSI); }
267   bool isOModSI() const { return isImmTy(ImmTyOModSI); }
268   bool isDMask() const { return isImmTy(ImmTyDMask); }
269   bool isUNorm() const { return isImmTy(ImmTyUNorm); }
270   bool isDA() const { return isImmTy(ImmTyDA); }
271   bool isR128() const { return isImmTy(ImmTyUNorm); }
272   bool isLWE() const { return isImmTy(ImmTyLWE); }
273   bool isOff() const { return isImmTy(ImmTyOff); }
274   bool isExpTgt() const { return isImmTy(ImmTyExpTgt); }
275   bool isExpVM() const { return isImmTy(ImmTyExpVM); }
276   bool isExpCompr() const { return isImmTy(ImmTyExpCompr); }
277   bool isOffen() const { return isImmTy(ImmTyOffen); }
278   bool isIdxen() const { return isImmTy(ImmTyIdxen); }
279   bool isAddr64() const { return isImmTy(ImmTyAddr64); }
280   bool isOffset() const { return isImmTy(ImmTyOffset) && isUInt<16>(getImm()); }
281   bool isOffset0() const { return isImmTy(ImmTyOffset0) && isUInt<16>(getImm()); }
282   bool isOffset1() const { return isImmTy(ImmTyOffset1) && isUInt<8>(getImm()); }
283   bool isGDS() const { return isImmTy(ImmTyGDS); }
284   bool isGLC() const { return isImmTy(ImmTyGLC); }
285   bool isSLC() const { return isImmTy(ImmTySLC); }
286   bool isTFE() const { return isImmTy(ImmTyTFE); }
287   bool isBankMask() const { return isImmTy(ImmTyDppBankMask); }
288   bool isRowMask() const { return isImmTy(ImmTyDppRowMask); }
289   bool isBoundCtrl() const { return isImmTy(ImmTyDppBoundCtrl); }
290   bool isSDWADstSel() const { return isImmTy(ImmTySdwaDstSel); }
291   bool isSDWASrc0Sel() const { return isImmTy(ImmTySdwaSrc0Sel); }
292   bool isSDWASrc1Sel() const { return isImmTy(ImmTySdwaSrc1Sel); }
293   bool isSDWADstUnused() const { return isImmTy(ImmTySdwaDstUnused); }
294   bool isInterpSlot() const { return isImmTy(ImmTyInterpSlot); }
295   bool isInterpAttr() const { return isImmTy(ImmTyInterpAttr); }
296   bool isAttrChan() const { return isImmTy(ImmTyAttrChan); }
297 
298   bool isMod() const {
299     return isClampSI() || isOModSI();
300   }
301 
302   bool isRegOrImm() const {
303     return isReg() || isImm();
304   }
305 
306   bool isRegClass(unsigned RCID) const;
307 
308   bool isRegOrInlineNoMods(unsigned RCID, MVT type) const {
309     return (isRegClass(RCID) || isInlinableImm(type)) && !hasModifiers();
310   }
311 
312   bool isSCSrcB16() const {
313     return isRegOrInlineNoMods(AMDGPU::SReg_32RegClassID, MVT::i16);
314   }
315 
316   bool isSCSrcB32() const {
317     return isRegOrInlineNoMods(AMDGPU::SReg_32RegClassID, MVT::i32);
318   }
319 
320   bool isSCSrcB64() const {
321     return isRegOrInlineNoMods(AMDGPU::SReg_64RegClassID, MVT::i64);
322   }
323 
324   bool isSCSrcF16() const {
325     return isRegOrInlineNoMods(AMDGPU::SReg_32RegClassID, MVT::f16);
326   }
327 
328   bool isSCSrcF32() const {
329     return isRegOrInlineNoMods(AMDGPU::SReg_32RegClassID, MVT::f32);
330   }
331 
332   bool isSCSrcF64() const {
333     return isRegOrInlineNoMods(AMDGPU::SReg_64RegClassID, MVT::f64);
334   }
335 
336   bool isSSrcB32() const {
337     return isSCSrcB32() || isLiteralImm(MVT::i32) || isExpr();
338   }
339 
340   bool isSSrcB16() const {
341     return isSCSrcB16() || isLiteralImm(MVT::i16);
342   }
343 
344   bool isSSrcB64() const {
345     // TODO: Find out how SALU supports extension of 32-bit literals to 64 bits.
346     // See isVSrc64().
347     return isSCSrcB64() || isLiteralImm(MVT::i64);
348   }
349 
350   bool isSSrcF32() const {
351     return isSCSrcB32() || isLiteralImm(MVT::f32) || isExpr();
352   }
353 
354   bool isSSrcF64() const {
355     return isSCSrcB64() || isLiteralImm(MVT::f64);
356   }
357 
358   bool isSSrcF16() const {
359     return isSCSrcB16() || isLiteralImm(MVT::f16);
360   }
361 
362   bool isVCSrcB32() const {
363     return isRegOrInlineNoMods(AMDGPU::VS_32RegClassID, MVT::i32);
364   }
365 
366   bool isVCSrcB64() const {
367     return isRegOrInlineNoMods(AMDGPU::VS_64RegClassID, MVT::i64);
368   }
369 
370   bool isVCSrcB16() const {
371     return isRegOrInlineNoMods(AMDGPU::VS_32RegClassID, MVT::i16);
372   }
373 
374   bool isVCSrcF32() const {
375     return isRegOrInlineNoMods(AMDGPU::VS_32RegClassID, MVT::f32);
376   }
377 
378   bool isVCSrcF64() const {
379     return isRegOrInlineNoMods(AMDGPU::VS_64RegClassID, MVT::f64);
380   }
381 
382   bool isVCSrcF16() const {
383     return isRegOrInlineNoMods(AMDGPU::VS_32RegClassID, MVT::f16);
384   }
385 
386   bool isVSrcB32() const {
387     return isVCSrcF32() || isLiteralImm(MVT::i32);
388   }
389 
390   bool isVSrcB64() const {
391     return isVCSrcF64() || isLiteralImm(MVT::i64);
392   }
393 
394   bool isVSrcB16() const {
395     return isVCSrcF16() || isLiteralImm(MVT::i16);
396   }
397 
398   bool isVSrcF32() const {
399     return isVCSrcF32() || isLiteralImm(MVT::f32);
400   }
401 
402   bool isVSrcF64() const {
403     return isVCSrcF64() || isLiteralImm(MVT::f64);
404   }
405 
406   bool isVSrcF16() const {
407     return isVCSrcF16() || isLiteralImm(MVT::f16);
408   }
409 
410   bool isKImmFP32() const {
411     return isLiteralImm(MVT::f32);
412   }
413 
414   bool isKImmFP16() const {
415     return isLiteralImm(MVT::f16);
416   }
417 
418   bool isMem() const override {
419     return false;
420   }
421 
422   bool isExpr() const {
423     return Kind == Expression;
424   }
425 
426   bool isSoppBrTarget() const {
427     return isExpr() || isImm();
428   }
429 
430   bool isSWaitCnt() const;
431   bool isHwreg() const;
432   bool isSendMsg() const;
433   bool isSMRDOffset8() const;
434   bool isSMRDOffset20() const;
435   bool isSMRDLiteralOffset() const;
436   bool isDPPCtrl() const;
437   bool isGPRIdxMode() const;
438 
439   StringRef getExpressionAsToken() const {
440     assert(isExpr());
441     const MCSymbolRefExpr *S = cast<MCSymbolRefExpr>(Expr);
442     return S->getSymbol().getName();
443   }
444 
445   StringRef getToken() const {
446     assert(isToken());
447 
448     if (Kind == Expression)
449       return getExpressionAsToken();
450 
451     return StringRef(Tok.Data, Tok.Length);
452   }
453 
454   int64_t getImm() const {
455     assert(isImm());
456     return Imm.Val;
457   }
458 
459   enum ImmTy getImmTy() const {
460     assert(isImm());
461     return Imm.Type;
462   }
463 
464   unsigned getReg() const override {
465     return Reg.RegNo;
466   }
467 
468   SMLoc getStartLoc() const override {
469     return StartLoc;
470   }
471 
472   SMLoc getEndLoc() const override {
473     return EndLoc;
474   }
475 
476   Modifiers getModifiers() const {
477     assert(isRegKind() || isImmTy(ImmTyNone));
478     return isRegKind() ? Reg.Mods : Imm.Mods;
479   }
480 
481   void setModifiers(Modifiers Mods) {
482     assert(isRegKind() || isImmTy(ImmTyNone));
483     if (isRegKind())
484       Reg.Mods = Mods;
485     else
486       Imm.Mods = Mods;
487   }
488 
489   bool hasModifiers() const {
490     return getModifiers().hasModifiers();
491   }
492 
493   bool hasFPModifiers() const {
494     return getModifiers().hasFPModifiers();
495   }
496 
497   bool hasIntModifiers() const {
498     return getModifiers().hasIntModifiers();
499   }
500 
501   void addImmOperands(MCInst &Inst, unsigned N, bool ApplyModifiers = true) const;
502 
503   void addLiteralImmOperand(MCInst &Inst, int64_t Val) const;
504 
505   template <unsigned Bitwidth>
506   void addKImmFPOperands(MCInst &Inst, unsigned N) const;
507 
508   void addKImmFP16Operands(MCInst &Inst, unsigned N) const {
509     addKImmFPOperands<16>(Inst, N);
510   }
511 
512   void addKImmFP32Operands(MCInst &Inst, unsigned N) const {
513     addKImmFPOperands<32>(Inst, N);
514   }
515 
516   void addRegOperands(MCInst &Inst, unsigned N) const;
517 
518   void addRegOrImmOperands(MCInst &Inst, unsigned N) const {
519     if (isRegKind())
520       addRegOperands(Inst, N);
521     else if (isExpr())
522       Inst.addOperand(MCOperand::createExpr(Expr));
523     else
524       addImmOperands(Inst, N);
525   }
526 
527   void addRegOrImmWithInputModsOperands(MCInst &Inst, unsigned N) const {
528     Modifiers Mods = getModifiers();
529     Inst.addOperand(MCOperand::createImm(Mods.getModifiersOperand()));
530     if (isRegKind()) {
531       addRegOperands(Inst, N);
532     } else {
533       addImmOperands(Inst, N, false);
534     }
535   }
536 
537   void addRegOrImmWithFPInputModsOperands(MCInst &Inst, unsigned N) const {
538     assert(!hasIntModifiers());
539     addRegOrImmWithInputModsOperands(Inst, N);
540   }
541 
542   void addRegOrImmWithIntInputModsOperands(MCInst &Inst, unsigned N) const {
543     assert(!hasFPModifiers());
544     addRegOrImmWithInputModsOperands(Inst, N);
545   }
546 
547   void addRegWithInputModsOperands(MCInst &Inst, unsigned N) const {
548     Modifiers Mods = getModifiers();
549     Inst.addOperand(MCOperand::createImm(Mods.getModifiersOperand()));
550     assert(isRegKind());
551     addRegOperands(Inst, N);
552   }
553 
554   void addRegWithFPInputModsOperands(MCInst &Inst, unsigned N) const {
555     assert(!hasIntModifiers());
556     addRegWithInputModsOperands(Inst, N);
557   }
558 
559   void addRegWithIntInputModsOperands(MCInst &Inst, unsigned N) const {
560     assert(!hasFPModifiers());
561     addRegWithInputModsOperands(Inst, N);
562   }
563 
564   void addSoppBrTargetOperands(MCInst &Inst, unsigned N) const {
565     if (isImm())
566       addImmOperands(Inst, N);
567     else {
568       assert(isExpr());
569       Inst.addOperand(MCOperand::createExpr(Expr));
570     }
571   }
572 
573   static void printImmTy(raw_ostream& OS, ImmTy Type) {
574     switch (Type) {
575     case ImmTyNone: OS << "None"; break;
576     case ImmTyGDS: OS << "GDS"; break;
577     case ImmTyOffen: OS << "Offen"; break;
578     case ImmTyIdxen: OS << "Idxen"; break;
579     case ImmTyAddr64: OS << "Addr64"; break;
580     case ImmTyOffset: OS << "Offset"; break;
581     case ImmTyOffset0: OS << "Offset0"; break;
582     case ImmTyOffset1: OS << "Offset1"; break;
583     case ImmTyGLC: OS << "GLC"; break;
584     case ImmTySLC: OS << "SLC"; break;
585     case ImmTyTFE: OS << "TFE"; break;
586     case ImmTyClampSI: OS << "ClampSI"; break;
587     case ImmTyOModSI: OS << "OModSI"; break;
588     case ImmTyDppCtrl: OS << "DppCtrl"; break;
589     case ImmTyDppRowMask: OS << "DppRowMask"; break;
590     case ImmTyDppBankMask: OS << "DppBankMask"; break;
591     case ImmTyDppBoundCtrl: OS << "DppBoundCtrl"; break;
592     case ImmTySdwaDstSel: OS << "SdwaDstSel"; break;
593     case ImmTySdwaSrc0Sel: OS << "SdwaSrc0Sel"; break;
594     case ImmTySdwaSrc1Sel: OS << "SdwaSrc1Sel"; break;
595     case ImmTySdwaDstUnused: OS << "SdwaDstUnused"; break;
596     case ImmTyDMask: OS << "DMask"; break;
597     case ImmTyUNorm: OS << "UNorm"; break;
598     case ImmTyDA: OS << "DA"; break;
599     case ImmTyR128: OS << "R128"; break;
600     case ImmTyLWE: OS << "LWE"; break;
601     case ImmTyOff: OS << "Off"; break;
602     case ImmTyExpTgt: OS << "ExpTgt"; break;
603     case ImmTyExpCompr: OS << "ExpCompr"; break;
604     case ImmTyExpVM: OS << "ExpVM"; break;
605     case ImmTyHwreg: OS << "Hwreg"; break;
606     case ImmTySendMsg: OS << "SendMsg"; break;
607     case ImmTyInterpSlot: OS << "InterpSlot"; break;
608     case ImmTyInterpAttr: OS << "InterpAttr"; break;
609     case ImmTyAttrChan: OS << "AttrChan"; break;
610     }
611   }
612 
613   void print(raw_ostream &OS) const override {
614     switch (Kind) {
615     case Register:
616       OS << "<register " << getReg() << " mods: " << Reg.Mods << '>';
617       break;
618     case Immediate:
619       OS << '<' << getImm();
620       if (getImmTy() != ImmTyNone) {
621         OS << " type: "; printImmTy(OS, getImmTy());
622       }
623       OS << " mods: " << Imm.Mods << '>';
624       break;
625     case Token:
626       OS << '\'' << getToken() << '\'';
627       break;
628     case Expression:
629       OS << "<expr " << *Expr << '>';
630       break;
631     }
632   }
633 
634   static AMDGPUOperand::Ptr CreateImm(const AMDGPUAsmParser *AsmParser,
635                                       int64_t Val, SMLoc Loc,
636                                       enum ImmTy Type = ImmTyNone,
637                                       bool IsFPImm = false) {
638     auto Op = llvm::make_unique<AMDGPUOperand>(Immediate, AsmParser);
639     Op->Imm.Val = Val;
640     Op->Imm.IsFPImm = IsFPImm;
641     Op->Imm.Type = Type;
642     Op->Imm.Mods = Modifiers();
643     Op->StartLoc = Loc;
644     Op->EndLoc = Loc;
645     return Op;
646   }
647 
648   static AMDGPUOperand::Ptr CreateToken(const AMDGPUAsmParser *AsmParser,
649                                         StringRef Str, SMLoc Loc,
650                                         bool HasExplicitEncodingSize = true) {
651     auto Res = llvm::make_unique<AMDGPUOperand>(Token, AsmParser);
652     Res->Tok.Data = Str.data();
653     Res->Tok.Length = Str.size();
654     Res->StartLoc = Loc;
655     Res->EndLoc = Loc;
656     return Res;
657   }
658 
659   static AMDGPUOperand::Ptr CreateReg(const AMDGPUAsmParser *AsmParser,
660                                       unsigned RegNo, SMLoc S,
661                                       SMLoc E,
662                                       bool ForceVOP3) {
663     auto Op = llvm::make_unique<AMDGPUOperand>(Register, AsmParser);
664     Op->Reg.RegNo = RegNo;
665     Op->Reg.Mods = Modifiers();
666     Op->Reg.IsForcedVOP3 = ForceVOP3;
667     Op->StartLoc = S;
668     Op->EndLoc = E;
669     return Op;
670   }
671 
672   static AMDGPUOperand::Ptr CreateExpr(const AMDGPUAsmParser *AsmParser,
673                                        const class MCExpr *Expr, SMLoc S) {
674     auto Op = llvm::make_unique<AMDGPUOperand>(Expression, AsmParser);
675     Op->Expr = Expr;
676     Op->StartLoc = S;
677     Op->EndLoc = S;
678     return Op;
679   }
680 };
681 
682 raw_ostream &operator <<(raw_ostream &OS, AMDGPUOperand::Modifiers Mods) {
683   OS << "abs:" << Mods.Abs << " neg: " << Mods.Neg << " sext:" << Mods.Sext;
684   return OS;
685 }
686 
687 //===----------------------------------------------------------------------===//
688 // AsmParser
689 //===----------------------------------------------------------------------===//
690 
691 // Holds info related to the current kernel, e.g. count of SGPRs used.
692 // Kernel scope begins at .amdgpu_hsa_kernel directive, ends at next
693 // .amdgpu_hsa_kernel or at EOF.
694 class KernelScopeInfo {
695   int SgprIndexUnusedMin = -1;
696   int VgprIndexUnusedMin = -1;
697   MCContext *Ctx = nullptr;
698 
699   void usesSgprAt(int i) {
700     if (i >= SgprIndexUnusedMin) {
701       SgprIndexUnusedMin = ++i;
702       if (Ctx) {
703         MCSymbol * const Sym = Ctx->getOrCreateSymbol(Twine(".kernel.sgpr_count"));
704         Sym->setVariableValue(MCConstantExpr::create(SgprIndexUnusedMin, *Ctx));
705       }
706     }
707   }
708 
709   void usesVgprAt(int i) {
710     if (i >= VgprIndexUnusedMin) {
711       VgprIndexUnusedMin = ++i;
712       if (Ctx) {
713         MCSymbol * const Sym = Ctx->getOrCreateSymbol(Twine(".kernel.vgpr_count"));
714         Sym->setVariableValue(MCConstantExpr::create(VgprIndexUnusedMin, *Ctx));
715       }
716     }
717   }
718 
719 public:
720   KernelScopeInfo() = default;
721 
722   void initialize(MCContext &Context) {
723     Ctx = &Context;
724     usesSgprAt(SgprIndexUnusedMin = -1);
725     usesVgprAt(VgprIndexUnusedMin = -1);
726   }
727 
728   void usesRegister(RegisterKind RegKind, unsigned DwordRegIndex, unsigned RegWidth) {
729     switch (RegKind) {
730       case IS_SGPR: usesSgprAt(DwordRegIndex + RegWidth - 1); break;
731       case IS_VGPR: usesVgprAt(DwordRegIndex + RegWidth - 1); break;
732       default: break;
733     }
734   }
735 };
736 
737 class AMDGPUAsmParser : public MCTargetAsmParser {
738   const MCInstrInfo &MII;
739   MCAsmParser &Parser;
740 
741   unsigned ForcedEncodingSize = 0;
742   bool ForcedDPP = false;
743   bool ForcedSDWA = false;
744   KernelScopeInfo KernelScope;
745 
746   /// @name Auto-generated Match Functions
747   /// {
748 
749 #define GET_ASSEMBLER_HEADER
750 #include "AMDGPUGenAsmMatcher.inc"
751 
752   /// }
753 
754 private:
755   bool ParseAsAbsoluteExpression(uint32_t &Ret);
756   bool ParseDirectiveMajorMinor(uint32_t &Major, uint32_t &Minor);
757   bool ParseDirectiveHSACodeObjectVersion();
758   bool ParseDirectiveHSACodeObjectISA();
759   bool ParseDirectiveRuntimeMetadata();
760   bool ParseAMDKernelCodeTValue(StringRef ID, amd_kernel_code_t &Header);
761   bool ParseDirectiveAMDKernelCodeT();
762   bool ParseSectionDirectiveHSAText();
763   bool subtargetHasRegister(const MCRegisterInfo &MRI, unsigned RegNo) const;
764   bool ParseDirectiveAMDGPUHsaKernel();
765   bool ParseDirectiveAMDGPUHsaModuleGlobal();
766   bool ParseDirectiveAMDGPUHsaProgramGlobal();
767   bool ParseSectionDirectiveHSADataGlobalAgent();
768   bool ParseSectionDirectiveHSADataGlobalProgram();
769   bool ParseSectionDirectiveHSARodataReadonlyAgent();
770   bool AddNextRegisterToList(unsigned& Reg, unsigned& RegWidth, RegisterKind RegKind, unsigned Reg1, unsigned RegNum);
771   bool ParseAMDGPURegister(RegisterKind& RegKind, unsigned& Reg, unsigned& RegNum, unsigned& RegWidth, unsigned *DwordRegIndex);
772   void cvtMubufImpl(MCInst &Inst, const OperandVector &Operands, bool IsAtomic, bool IsAtomicReturn);
773 
774 public:
775   enum AMDGPUMatchResultTy {
776     Match_PreferE32 = FIRST_TARGET_MATCH_RESULT_TY
777   };
778 
779   AMDGPUAsmParser(const MCSubtargetInfo &STI, MCAsmParser &_Parser,
780                const MCInstrInfo &MII,
781                const MCTargetOptions &Options)
782       : MCTargetAsmParser(Options, STI), MII(MII), Parser(_Parser) {
783     MCAsmParserExtension::Initialize(Parser);
784 
785     if (getSTI().getFeatureBits().none()) {
786       // Set default features.
787       copySTI().ToggleFeature("SOUTHERN_ISLANDS");
788     }
789 
790     setAvailableFeatures(ComputeAvailableFeatures(getSTI().getFeatureBits()));
791 
792     {
793       // TODO: make those pre-defined variables read-only.
794       // Currently there is none suitable machinery in the core llvm-mc for this.
795       // MCSymbol::isRedefinable is intended for another purpose, and
796       // AsmParser::parseDirectiveSet() cannot be specialized for specific target.
797       AMDGPU::IsaVersion Isa = AMDGPU::getIsaVersion(getSTI().getFeatureBits());
798       MCContext &Ctx = getContext();
799       MCSymbol *Sym = Ctx.getOrCreateSymbol(Twine(".option.machine_version_major"));
800       Sym->setVariableValue(MCConstantExpr::create(Isa.Major, Ctx));
801       Sym = Ctx.getOrCreateSymbol(Twine(".option.machine_version_minor"));
802       Sym->setVariableValue(MCConstantExpr::create(Isa.Minor, Ctx));
803       Sym = Ctx.getOrCreateSymbol(Twine(".option.machine_version_stepping"));
804       Sym->setVariableValue(MCConstantExpr::create(Isa.Stepping, Ctx));
805     }
806     KernelScope.initialize(getContext());
807   }
808 
809   bool isSI() const {
810     return AMDGPU::isSI(getSTI());
811   }
812 
813   bool isCI() const {
814     return AMDGPU::isCI(getSTI());
815   }
816 
817   bool isVI() const {
818     return AMDGPU::isVI(getSTI());
819   }
820 
821   bool hasInv2PiInlineImm() const {
822     return getSTI().getFeatureBits()[AMDGPU::FeatureInv2PiInlineImm];
823   }
824 
825   bool hasSGPR102_SGPR103() const {
826     return !isVI();
827   }
828 
829   AMDGPUTargetStreamer &getTargetStreamer() {
830     MCTargetStreamer &TS = *getParser().getStreamer().getTargetStreamer();
831     return static_cast<AMDGPUTargetStreamer &>(TS);
832   }
833 
834   const MCRegisterInfo *getMRI() const {
835     // We need this const_cast because for some reason getContext() is not const
836     // in MCAsmParser.
837     return const_cast<AMDGPUAsmParser*>(this)->getContext().getRegisterInfo();
838   }
839 
840   const MCInstrInfo *getMII() const {
841     return &MII;
842   }
843 
844   void setForcedEncodingSize(unsigned Size) { ForcedEncodingSize = Size; }
845   void setForcedDPP(bool ForceDPP_) { ForcedDPP = ForceDPP_; }
846   void setForcedSDWA(bool ForceSDWA_) { ForcedSDWA = ForceSDWA_; }
847 
848   unsigned getForcedEncodingSize() const { return ForcedEncodingSize; }
849   bool isForcedVOP3() const { return ForcedEncodingSize == 64; }
850   bool isForcedDPP() const { return ForcedDPP; }
851   bool isForcedSDWA() const { return ForcedSDWA; }
852   ArrayRef<unsigned> getMatchedVariants() const;
853 
854   std::unique_ptr<AMDGPUOperand> parseRegister();
855   bool ParseRegister(unsigned &RegNo, SMLoc &StartLoc, SMLoc &EndLoc) override;
856   unsigned checkTargetMatchPredicate(MCInst &Inst) override;
857   unsigned validateTargetOperandClass(MCParsedAsmOperand &Op,
858                                       unsigned Kind) override;
859   bool MatchAndEmitInstruction(SMLoc IDLoc, unsigned &Opcode,
860                                OperandVector &Operands, MCStreamer &Out,
861                                uint64_t &ErrorInfo,
862                                bool MatchingInlineAsm) override;
863   bool ParseDirective(AsmToken DirectiveID) override;
864   OperandMatchResultTy parseOperand(OperandVector &Operands, StringRef Mnemonic);
865   StringRef parseMnemonicSuffix(StringRef Name);
866   bool ParseInstruction(ParseInstructionInfo &Info, StringRef Name,
867                         SMLoc NameLoc, OperandVector &Operands) override;
868   //bool ProcessInstruction(MCInst &Inst);
869 
870   OperandMatchResultTy parseIntWithPrefix(const char *Prefix, int64_t &Int);
871   OperandMatchResultTy
872   parseIntWithPrefix(const char *Prefix, OperandVector &Operands,
873                      enum AMDGPUOperand::ImmTy ImmTy = AMDGPUOperand::ImmTyNone,
874                      bool (*ConvertResult)(int64_t &) = nullptr);
875   OperandMatchResultTy
876   parseNamedBit(const char *Name, OperandVector &Operands,
877                 enum AMDGPUOperand::ImmTy ImmTy = AMDGPUOperand::ImmTyNone);
878   OperandMatchResultTy parseStringWithPrefix(StringRef Prefix,
879                                              StringRef &Value);
880 
881   OperandMatchResultTy parseImm(OperandVector &Operands);
882   OperandMatchResultTy parseReg(OperandVector &Operands);
883   OperandMatchResultTy parseRegOrImm(OperandVector &Operands);
884   OperandMatchResultTy parseRegOrImmWithFPInputMods(OperandVector &Operands, bool AllowImm = true);
885   OperandMatchResultTy parseRegOrImmWithIntInputMods(OperandVector &Operands, bool AllowImm = true);
886   OperandMatchResultTy parseRegWithFPInputMods(OperandVector &Operands);
887   OperandMatchResultTy parseRegWithIntInputMods(OperandVector &Operands);
888   OperandMatchResultTy parseVReg32OrOff(OperandVector &Operands);
889 
890   void cvtDSOffset01(MCInst &Inst, const OperandVector &Operands);
891   void cvtDS(MCInst &Inst, const OperandVector &Operands);
892   void cvtExp(MCInst &Inst, const OperandVector &Operands);
893 
894   bool parseCnt(int64_t &IntVal);
895   OperandMatchResultTy parseSWaitCntOps(OperandVector &Operands);
896   OperandMatchResultTy parseHwreg(OperandVector &Operands);
897 
898 private:
899   struct OperandInfoTy {
900     int64_t Id;
901     bool IsSymbolic;
902     OperandInfoTy(int64_t Id_) : Id(Id_), IsSymbolic(false) { }
903   };
904 
905   bool parseSendMsgConstruct(OperandInfoTy &Msg, OperandInfoTy &Operation, int64_t &StreamId);
906   bool parseHwregConstruct(OperandInfoTy &HwReg, int64_t &Offset, int64_t &Width);
907 
908   void errorExpTgt();
909   OperandMatchResultTy parseExpTgtImpl(StringRef Str, uint8_t &Val);
910 
911 public:
912   OperandMatchResultTy parseOptionalOperand(OperandVector &Operands);
913 
914   OperandMatchResultTy parseExpTgt(OperandVector &Operands);
915   OperandMatchResultTy parseSendMsgOp(OperandVector &Operands);
916   OperandMatchResultTy parseInterpSlot(OperandVector &Operands);
917   OperandMatchResultTy parseInterpAttr(OperandVector &Operands);
918   OperandMatchResultTy parseSOppBrTarget(OperandVector &Operands);
919 
920   void cvtMubuf(MCInst &Inst, const OperandVector &Operands) { cvtMubufImpl(Inst, Operands, false, false); }
921   void cvtMubufAtomic(MCInst &Inst, const OperandVector &Operands) { cvtMubufImpl(Inst, Operands, true, false); }
922   void cvtMubufAtomicReturn(MCInst &Inst, const OperandVector &Operands) { cvtMubufImpl(Inst, Operands, true, true); }
923   AMDGPUOperand::Ptr defaultGLC() const;
924   AMDGPUOperand::Ptr defaultSLC() const;
925   AMDGPUOperand::Ptr defaultTFE() const;
926 
927   AMDGPUOperand::Ptr defaultDMask() const;
928   AMDGPUOperand::Ptr defaultUNorm() const;
929   AMDGPUOperand::Ptr defaultDA() const;
930   AMDGPUOperand::Ptr defaultR128() const;
931   AMDGPUOperand::Ptr defaultLWE() const;
932   AMDGPUOperand::Ptr defaultSMRDOffset8() const;
933   AMDGPUOperand::Ptr defaultSMRDOffset20() const;
934   AMDGPUOperand::Ptr defaultSMRDLiteralOffset() const;
935 
936   OperandMatchResultTy parseOModOperand(OperandVector &Operands);
937 
938   void cvtId(MCInst &Inst, const OperandVector &Operands);
939   void cvtVOP3_2_mod(MCInst &Inst, const OperandVector &Operands);
940   void cvtVOP3(MCInst &Inst, const OperandVector &Operands);
941 
942   void cvtMIMG(MCInst &Inst, const OperandVector &Operands);
943   void cvtMIMGAtomic(MCInst &Inst, const OperandVector &Operands);
944 
945   OperandMatchResultTy parseDPPCtrl(OperandVector &Operands);
946   AMDGPUOperand::Ptr defaultRowMask() const;
947   AMDGPUOperand::Ptr defaultBankMask() const;
948   AMDGPUOperand::Ptr defaultBoundCtrl() const;
949   void cvtDPP(MCInst &Inst, const OperandVector &Operands);
950 
951   OperandMatchResultTy parseSDWASel(OperandVector &Operands, StringRef Prefix,
952                                     AMDGPUOperand::ImmTy Type);
953   OperandMatchResultTy parseSDWADstUnused(OperandVector &Operands);
954   void cvtSdwaVOP1(MCInst &Inst, const OperandVector &Operands);
955   void cvtSdwaVOP2(MCInst &Inst, const OperandVector &Operands);
956   void cvtSdwaVOPC(MCInst &Inst, const OperandVector &Operands);
957   void cvtSDWA(MCInst &Inst, const OperandVector &Operands,
958                uint64_t BasicInstType);
959 };
960 
961 struct OptionalOperand {
962   const char *Name;
963   AMDGPUOperand::ImmTy Type;
964   bool IsBit;
965   bool (*ConvertResult)(int64_t&);
966 };
967 
968 } // end anonymous namespace
969 
970 // May be called with integer type with equivalent bitwidth.
971 static const fltSemantics *getFltSemantics(unsigned Size) {
972   switch (Size) {
973   case 4:
974     return &APFloat::IEEEsingle();
975   case 8:
976     return &APFloat::IEEEdouble();
977   case 2:
978     return &APFloat::IEEEhalf();
979   default:
980     llvm_unreachable("unsupported fp type");
981   }
982 }
983 
984 static const fltSemantics *getFltSemantics(MVT VT) {
985   return getFltSemantics(VT.getSizeInBits() / 8);
986 }
987 
988 //===----------------------------------------------------------------------===//
989 // Operand
990 //===----------------------------------------------------------------------===//
991 
992 static bool canLosslesslyConvertToFPType(APFloat &FPLiteral, MVT VT) {
993   bool Lost;
994 
995   // Convert literal to single precision
996   APFloat::opStatus Status = FPLiteral.convert(*getFltSemantics(VT),
997                                                APFloat::rmNearestTiesToEven,
998                                                &Lost);
999   // We allow precision lost but not overflow or underflow
1000   if (Status != APFloat::opOK &&
1001       Lost &&
1002       ((Status & APFloat::opOverflow)  != 0 ||
1003        (Status & APFloat::opUnderflow) != 0)) {
1004     return false;
1005   }
1006 
1007   return true;
1008 }
1009 
1010 bool AMDGPUOperand::isInlinableImm(MVT type) const {
1011   if (!isImmTy(ImmTyNone)) {
1012     // Only plain immediates are inlinable (e.g. "clamp" attribute is not)
1013     return false;
1014   }
1015   // TODO: We should avoid using host float here. It would be better to
1016   // check the float bit values which is what a few other places do.
1017   // We've had bot failures before due to weird NaN support on mips hosts.
1018 
1019   APInt Literal(64, Imm.Val);
1020 
1021   if (Imm.IsFPImm) { // We got fp literal token
1022     if (type == MVT::f64 || type == MVT::i64) { // Expected 64-bit operand
1023       return AMDGPU::isInlinableLiteral64(Imm.Val,
1024                                           AsmParser->hasInv2PiInlineImm());
1025     }
1026 
1027     APFloat FPLiteral(APFloat::IEEEdouble(), APInt(64, Imm.Val));
1028     if (!canLosslesslyConvertToFPType(FPLiteral, type))
1029       return false;
1030 
1031     if (type.getScalarSizeInBits() == 16) {
1032       return AMDGPU::isInlinableLiteral16(
1033         static_cast<int32_t>(FPLiteral.bitcastToAPInt().getZExtValue()),
1034         AsmParser->hasInv2PiInlineImm());
1035     }
1036 
1037     // Check if single precision literal is inlinable
1038     return AMDGPU::isInlinableLiteral32(
1039       static_cast<int32_t>(FPLiteral.bitcastToAPInt().getZExtValue()),
1040       AsmParser->hasInv2PiInlineImm());
1041   }
1042 
1043   // We got int literal token.
1044   if (type == MVT::f64 || type == MVT::i64) { // Expected 64-bit operand
1045     return AMDGPU::isInlinableLiteral64(Imm.Val,
1046                                         AsmParser->hasInv2PiInlineImm());
1047   }
1048 
1049   if (type.getScalarSizeInBits() == 16) {
1050     return AMDGPU::isInlinableLiteral16(
1051       static_cast<int16_t>(Literal.getLoBits(16).getSExtValue()),
1052       AsmParser->hasInv2PiInlineImm());
1053   }
1054 
1055   return AMDGPU::isInlinableLiteral32(
1056     static_cast<int32_t>(Literal.getLoBits(32).getZExtValue()),
1057     AsmParser->hasInv2PiInlineImm());
1058 }
1059 
1060 bool AMDGPUOperand::isLiteralImm(MVT type) const {
1061   // Check that this imediate can be added as literal
1062   if (!isImmTy(ImmTyNone)) {
1063     return false;
1064   }
1065 
1066   if (!Imm.IsFPImm) {
1067     // We got int literal token.
1068 
1069     unsigned Size = type.getSizeInBits();
1070     if (Size == 64)
1071       Size = 32;
1072 
1073     // FIXME: 64-bit operands can zero extend, sign extend, or pad zeroes for FP
1074     // types.
1075     return isUIntN(Size, Imm.Val) || isIntN(Size, Imm.Val);
1076   }
1077 
1078   // We got fp literal token
1079   if (type == MVT::f64) { // Expected 64-bit fp operand
1080     // We would set low 64-bits of literal to zeroes but we accept this literals
1081     return true;
1082   }
1083 
1084   if (type == MVT::i64) { // Expected 64-bit int operand
1085     // We don't allow fp literals in 64-bit integer instructions. It is
1086     // unclear how we should encode them.
1087     return false;
1088   }
1089 
1090   APFloat FPLiteral(APFloat::IEEEdouble(), APInt(64, Imm.Val));
1091   return canLosslesslyConvertToFPType(FPLiteral, type);
1092 }
1093 
1094 bool AMDGPUOperand::isRegClass(unsigned RCID) const {
1095   return isRegKind() && AsmParser->getMRI()->getRegClass(RCID).contains(getReg());
1096 }
1097 
1098 void AMDGPUOperand::addImmOperands(MCInst &Inst, unsigned N, bool ApplyModifiers) const {
1099   int64_t Val = Imm.Val;
1100   if (isImmTy(ImmTyNone) && ApplyModifiers && Imm.Mods.hasFPModifiers() && Imm.Mods.Neg) {
1101     // Apply modifiers to immediate value. Only negate can get here
1102     if (Imm.IsFPImm) {
1103       APFloat F(BitsToDouble(Val));
1104       F.changeSign();
1105       Val = F.bitcastToAPInt().getZExtValue();
1106     } else {
1107       Val = -Val;
1108     }
1109   }
1110 
1111   if (AMDGPU::isSISrcOperand(AsmParser->getMII()->get(Inst.getOpcode()),
1112                              Inst.getNumOperands())) {
1113     addLiteralImmOperand(Inst, Val);
1114   } else {
1115     Inst.addOperand(MCOperand::createImm(Val));
1116   }
1117 }
1118 
1119 void AMDGPUOperand::addLiteralImmOperand(MCInst &Inst, int64_t Val) const {
1120   const auto& InstDesc = AsmParser->getMII()->get(Inst.getOpcode());
1121   auto OpNum = Inst.getNumOperands();
1122   // Check that this operand accepts literals
1123   assert(AMDGPU::isSISrcOperand(InstDesc, OpNum));
1124 
1125   auto OpSize = AMDGPU::getOperandSize(InstDesc, OpNum); // expected operand size
1126 
1127   if (Imm.IsFPImm) { // We got fp literal token
1128     APInt Literal(64, Val);
1129 
1130     switch (OpSize) {
1131     case 8:
1132       if (AMDGPU::isInlinableLiteral64(Literal.getZExtValue(),
1133                                        AsmParser->hasInv2PiInlineImm())) {
1134         Inst.addOperand(MCOperand::createImm(Literal.getZExtValue()));
1135         return;
1136       }
1137 
1138       // Non-inlineable
1139       if (AMDGPU::isSISrcFPOperand(InstDesc, OpNum)) { // Expected 64-bit fp operand
1140         // For fp operands we check if low 32 bits are zeros
1141         if (Literal.getLoBits(32) != 0) {
1142           const_cast<AMDGPUAsmParser *>(AsmParser)->Warning(Inst.getLoc(),
1143           "Can't encode literal as exact 64-bit floating-point operand. "
1144           "Low 32-bits will be set to zero");
1145         }
1146 
1147         Inst.addOperand(MCOperand::createImm(Literal.lshr(32).getZExtValue()));
1148         return;
1149       }
1150 
1151       // We don't allow fp literals in 64-bit integer instructions. It is
1152       // unclear how we should encode them. This case should be checked earlier
1153       // in predicate methods (isLiteralImm())
1154       llvm_unreachable("fp literal in 64-bit integer instruction.");
1155 
1156     case 4:
1157     case 2: {
1158       bool lost;
1159       APFloat FPLiteral(APFloat::IEEEdouble(), Literal);
1160       // Convert literal to single precision
1161       FPLiteral.convert(*getFltSemantics(OpSize),
1162                         APFloat::rmNearestTiesToEven, &lost);
1163       // We allow precision lost but not overflow or underflow. This should be
1164       // checked earlier in isLiteralImm()
1165       Inst.addOperand(MCOperand::createImm(FPLiteral.bitcastToAPInt().getZExtValue()));
1166       return;
1167     }
1168     default:
1169       llvm_unreachable("invalid operand size");
1170     }
1171 
1172     return;
1173   }
1174 
1175    // We got int literal token.
1176   // Only sign extend inline immediates.
1177   // FIXME: No errors on truncation
1178   switch (OpSize) {
1179   case 4:
1180     if (isInt<32>(Val) &&
1181         AMDGPU::isInlinableLiteral32(static_cast<int32_t>(Val),
1182                                      AsmParser->hasInv2PiInlineImm())) {
1183       Inst.addOperand(MCOperand::createImm(Val));
1184       return;
1185     }
1186 
1187     Inst.addOperand(MCOperand::createImm(Val & 0xffffffff));
1188     return;
1189 
1190   case 8:
1191     if (AMDGPU::isInlinableLiteral64(Val,
1192                                      AsmParser->hasInv2PiInlineImm())) {
1193       Inst.addOperand(MCOperand::createImm(Val));
1194       return;
1195     }
1196 
1197     Inst.addOperand(MCOperand::createImm(Lo_32(Val)));
1198     return;
1199 
1200   case 2:
1201     if (isInt<16>(Val) &&
1202         AMDGPU::isInlinableLiteral16(static_cast<int16_t>(Val),
1203                                      AsmParser->hasInv2PiInlineImm())) {
1204       Inst.addOperand(MCOperand::createImm(Val));
1205       return;
1206     }
1207 
1208     Inst.addOperand(MCOperand::createImm(Val & 0xffff));
1209     return;
1210 
1211   default:
1212     llvm_unreachable("invalid operand size");
1213   }
1214 }
1215 
1216 template <unsigned Bitwidth>
1217 void AMDGPUOperand::addKImmFPOperands(MCInst &Inst, unsigned N) const {
1218   APInt Literal(64, Imm.Val);
1219 
1220   if (!Imm.IsFPImm) {
1221     // We got int literal token.
1222     Inst.addOperand(MCOperand::createImm(Literal.getLoBits(Bitwidth).getZExtValue()));
1223     return;
1224   }
1225 
1226   bool Lost;
1227   APFloat FPLiteral(APFloat::IEEEdouble(), Literal);
1228   FPLiteral.convert(*getFltSemantics(Bitwidth / 8),
1229                     APFloat::rmNearestTiesToEven, &Lost);
1230   Inst.addOperand(MCOperand::createImm(FPLiteral.bitcastToAPInt().getZExtValue()));
1231 }
1232 
1233 void AMDGPUOperand::addRegOperands(MCInst &Inst, unsigned N) const {
1234   Inst.addOperand(MCOperand::createReg(AMDGPU::getMCReg(getReg(), AsmParser->getSTI())));
1235 }
1236 
1237 //===----------------------------------------------------------------------===//
1238 // AsmParser
1239 //===----------------------------------------------------------------------===//
1240 
1241 static int getRegClass(RegisterKind Is, unsigned RegWidth) {
1242   if (Is == IS_VGPR) {
1243     switch (RegWidth) {
1244       default: return -1;
1245       case 1: return AMDGPU::VGPR_32RegClassID;
1246       case 2: return AMDGPU::VReg_64RegClassID;
1247       case 3: return AMDGPU::VReg_96RegClassID;
1248       case 4: return AMDGPU::VReg_128RegClassID;
1249       case 8: return AMDGPU::VReg_256RegClassID;
1250       case 16: return AMDGPU::VReg_512RegClassID;
1251     }
1252   } else if (Is == IS_TTMP) {
1253     switch (RegWidth) {
1254       default: return -1;
1255       case 1: return AMDGPU::TTMP_32RegClassID;
1256       case 2: return AMDGPU::TTMP_64RegClassID;
1257       case 4: return AMDGPU::TTMP_128RegClassID;
1258     }
1259   } else if (Is == IS_SGPR) {
1260     switch (RegWidth) {
1261       default: return -1;
1262       case 1: return AMDGPU::SGPR_32RegClassID;
1263       case 2: return AMDGPU::SGPR_64RegClassID;
1264       case 4: return AMDGPU::SGPR_128RegClassID;
1265       case 8: return AMDGPU::SReg_256RegClassID;
1266       case 16: return AMDGPU::SReg_512RegClassID;
1267     }
1268   }
1269   return -1;
1270 }
1271 
1272 static unsigned getSpecialRegForName(StringRef RegName) {
1273   return StringSwitch<unsigned>(RegName)
1274     .Case("exec", AMDGPU::EXEC)
1275     .Case("vcc", AMDGPU::VCC)
1276     .Case("flat_scratch", AMDGPU::FLAT_SCR)
1277     .Case("m0", AMDGPU::M0)
1278     .Case("scc", AMDGPU::SCC)
1279     .Case("tba", AMDGPU::TBA)
1280     .Case("tma", AMDGPU::TMA)
1281     .Case("flat_scratch_lo", AMDGPU::FLAT_SCR_LO)
1282     .Case("flat_scratch_hi", AMDGPU::FLAT_SCR_HI)
1283     .Case("vcc_lo", AMDGPU::VCC_LO)
1284     .Case("vcc_hi", AMDGPU::VCC_HI)
1285     .Case("exec_lo", AMDGPU::EXEC_LO)
1286     .Case("exec_hi", AMDGPU::EXEC_HI)
1287     .Case("tma_lo", AMDGPU::TMA_LO)
1288     .Case("tma_hi", AMDGPU::TMA_HI)
1289     .Case("tba_lo", AMDGPU::TBA_LO)
1290     .Case("tba_hi", AMDGPU::TBA_HI)
1291     .Default(0);
1292 }
1293 
1294 bool AMDGPUAsmParser::ParseRegister(unsigned &RegNo, SMLoc &StartLoc,
1295                                     SMLoc &EndLoc) {
1296   auto R = parseRegister();
1297   if (!R) return true;
1298   assert(R->isReg());
1299   RegNo = R->getReg();
1300   StartLoc = R->getStartLoc();
1301   EndLoc = R->getEndLoc();
1302   return false;
1303 }
1304 
1305 bool AMDGPUAsmParser::AddNextRegisterToList(unsigned &Reg, unsigned &RegWidth,
1306                                             RegisterKind RegKind, unsigned Reg1,
1307                                             unsigned RegNum) {
1308   switch (RegKind) {
1309   case IS_SPECIAL:
1310     if (Reg == AMDGPU::EXEC_LO && Reg1 == AMDGPU::EXEC_HI) {
1311       Reg = AMDGPU::EXEC;
1312       RegWidth = 2;
1313       return true;
1314     }
1315     if (Reg == AMDGPU::FLAT_SCR_LO && Reg1 == AMDGPU::FLAT_SCR_HI) {
1316       Reg = AMDGPU::FLAT_SCR;
1317       RegWidth = 2;
1318       return true;
1319     }
1320     if (Reg == AMDGPU::VCC_LO && Reg1 == AMDGPU::VCC_HI) {
1321       Reg = AMDGPU::VCC;
1322       RegWidth = 2;
1323       return true;
1324     }
1325     if (Reg == AMDGPU::TBA_LO && Reg1 == AMDGPU::TBA_HI) {
1326       Reg = AMDGPU::TBA;
1327       RegWidth = 2;
1328       return true;
1329     }
1330     if (Reg == AMDGPU::TMA_LO && Reg1 == AMDGPU::TMA_HI) {
1331       Reg = AMDGPU::TMA;
1332       RegWidth = 2;
1333       return true;
1334     }
1335     return false;
1336   case IS_VGPR:
1337   case IS_SGPR:
1338   case IS_TTMP:
1339     if (Reg1 != Reg + RegWidth) {
1340       return false;
1341     }
1342     RegWidth++;
1343     return true;
1344   default:
1345     llvm_unreachable("unexpected register kind");
1346   }
1347 }
1348 
1349 bool AMDGPUAsmParser::ParseAMDGPURegister(RegisterKind &RegKind, unsigned &Reg,
1350                                           unsigned &RegNum, unsigned &RegWidth,
1351                                           unsigned *DwordRegIndex) {
1352   if (DwordRegIndex) { *DwordRegIndex = 0; }
1353   const MCRegisterInfo *TRI = getContext().getRegisterInfo();
1354   if (getLexer().is(AsmToken::Identifier)) {
1355     StringRef RegName = Parser.getTok().getString();
1356     if ((Reg = getSpecialRegForName(RegName))) {
1357       Parser.Lex();
1358       RegKind = IS_SPECIAL;
1359     } else {
1360       unsigned RegNumIndex = 0;
1361       if (RegName[0] == 'v') {
1362         RegNumIndex = 1;
1363         RegKind = IS_VGPR;
1364       } else if (RegName[0] == 's') {
1365         RegNumIndex = 1;
1366         RegKind = IS_SGPR;
1367       } else if (RegName.startswith("ttmp")) {
1368         RegNumIndex = strlen("ttmp");
1369         RegKind = IS_TTMP;
1370       } else {
1371         return false;
1372       }
1373       if (RegName.size() > RegNumIndex) {
1374         // Single 32-bit register: vXX.
1375         if (RegName.substr(RegNumIndex).getAsInteger(10, RegNum))
1376           return false;
1377         Parser.Lex();
1378         RegWidth = 1;
1379       } else {
1380         // Range of registers: v[XX:YY]. ":YY" is optional.
1381         Parser.Lex();
1382         int64_t RegLo, RegHi;
1383         if (getLexer().isNot(AsmToken::LBrac))
1384           return false;
1385         Parser.Lex();
1386 
1387         if (getParser().parseAbsoluteExpression(RegLo))
1388           return false;
1389 
1390         const bool isRBrace = getLexer().is(AsmToken::RBrac);
1391         if (!isRBrace && getLexer().isNot(AsmToken::Colon))
1392           return false;
1393         Parser.Lex();
1394 
1395         if (isRBrace) {
1396           RegHi = RegLo;
1397         } else {
1398           if (getParser().parseAbsoluteExpression(RegHi))
1399             return false;
1400 
1401           if (getLexer().isNot(AsmToken::RBrac))
1402             return false;
1403           Parser.Lex();
1404         }
1405         RegNum = (unsigned) RegLo;
1406         RegWidth = (RegHi - RegLo) + 1;
1407       }
1408     }
1409   } else if (getLexer().is(AsmToken::LBrac)) {
1410     // List of consecutive registers: [s0,s1,s2,s3]
1411     Parser.Lex();
1412     if (!ParseAMDGPURegister(RegKind, Reg, RegNum, RegWidth, nullptr))
1413       return false;
1414     if (RegWidth != 1)
1415       return false;
1416     RegisterKind RegKind1;
1417     unsigned Reg1, RegNum1, RegWidth1;
1418     do {
1419       if (getLexer().is(AsmToken::Comma)) {
1420         Parser.Lex();
1421       } else if (getLexer().is(AsmToken::RBrac)) {
1422         Parser.Lex();
1423         break;
1424       } else if (ParseAMDGPURegister(RegKind1, Reg1, RegNum1, RegWidth1, nullptr)) {
1425         if (RegWidth1 != 1) {
1426           return false;
1427         }
1428         if (RegKind1 != RegKind) {
1429           return false;
1430         }
1431         if (!AddNextRegisterToList(Reg, RegWidth, RegKind1, Reg1, RegNum1)) {
1432           return false;
1433         }
1434       } else {
1435         return false;
1436       }
1437     } while (true);
1438   } else {
1439     return false;
1440   }
1441   switch (RegKind) {
1442   case IS_SPECIAL:
1443     RegNum = 0;
1444     RegWidth = 1;
1445     break;
1446   case IS_VGPR:
1447   case IS_SGPR:
1448   case IS_TTMP:
1449   {
1450     unsigned Size = 1;
1451     if (RegKind == IS_SGPR || RegKind == IS_TTMP) {
1452       // SGPR and TTMP registers must be aligned. Max required alignment is 4 dwords.
1453       Size = std::min(RegWidth, 4u);
1454     }
1455     if (RegNum % Size != 0)
1456       return false;
1457     if (DwordRegIndex) { *DwordRegIndex = RegNum; }
1458     RegNum = RegNum / Size;
1459     int RCID = getRegClass(RegKind, RegWidth);
1460     if (RCID == -1)
1461       return false;
1462     const MCRegisterClass RC = TRI->getRegClass(RCID);
1463     if (RegNum >= RC.getNumRegs())
1464       return false;
1465     Reg = RC.getRegister(RegNum);
1466     break;
1467   }
1468 
1469   default:
1470     llvm_unreachable("unexpected register kind");
1471   }
1472 
1473   if (!subtargetHasRegister(*TRI, Reg))
1474     return false;
1475   return true;
1476 }
1477 
1478 std::unique_ptr<AMDGPUOperand> AMDGPUAsmParser::parseRegister() {
1479   const auto &Tok = Parser.getTok();
1480   SMLoc StartLoc = Tok.getLoc();
1481   SMLoc EndLoc = Tok.getEndLoc();
1482   RegisterKind RegKind;
1483   unsigned Reg, RegNum, RegWidth, DwordRegIndex;
1484 
1485   if (!ParseAMDGPURegister(RegKind, Reg, RegNum, RegWidth, &DwordRegIndex)) {
1486     return nullptr;
1487   }
1488   KernelScope.usesRegister(RegKind, DwordRegIndex, RegWidth);
1489   return AMDGPUOperand::CreateReg(this, Reg, StartLoc, EndLoc, false);
1490 }
1491 
1492 OperandMatchResultTy
1493 AMDGPUAsmParser::parseImm(OperandVector &Operands) {
1494   // TODO: add syntactic sugar for 1/(2*PI)
1495   bool Minus = false;
1496   if (getLexer().getKind() == AsmToken::Minus) {
1497     Minus = true;
1498     Parser.Lex();
1499   }
1500 
1501   SMLoc S = Parser.getTok().getLoc();
1502   switch(getLexer().getKind()) {
1503   case AsmToken::Integer: {
1504     int64_t IntVal;
1505     if (getParser().parseAbsoluteExpression(IntVal))
1506       return MatchOperand_ParseFail;
1507     if (Minus)
1508       IntVal *= -1;
1509     Operands.push_back(AMDGPUOperand::CreateImm(this, IntVal, S));
1510     return MatchOperand_Success;
1511   }
1512   case AsmToken::Real: {
1513     int64_t IntVal;
1514     if (getParser().parseAbsoluteExpression(IntVal))
1515       return MatchOperand_ParseFail;
1516 
1517     APFloat F(BitsToDouble(IntVal));
1518     if (Minus)
1519       F.changeSign();
1520     Operands.push_back(
1521         AMDGPUOperand::CreateImm(this, F.bitcastToAPInt().getZExtValue(), S,
1522                                  AMDGPUOperand::ImmTyNone, true));
1523     return MatchOperand_Success;
1524   }
1525   default:
1526     return Minus ? MatchOperand_ParseFail : MatchOperand_NoMatch;
1527   }
1528 }
1529 
1530 OperandMatchResultTy
1531 AMDGPUAsmParser::parseReg(OperandVector &Operands) {
1532   if (auto R = parseRegister()) {
1533     assert(R->isReg());
1534     R->Reg.IsForcedVOP3 = isForcedVOP3();
1535     Operands.push_back(std::move(R));
1536     return MatchOperand_Success;
1537   }
1538   return MatchOperand_NoMatch;
1539 }
1540 
1541 OperandMatchResultTy
1542 AMDGPUAsmParser::parseRegOrImm(OperandVector &Operands) {
1543   auto res = parseImm(Operands);
1544   if (res != MatchOperand_NoMatch) {
1545     return res;
1546   }
1547 
1548   return parseReg(Operands);
1549 }
1550 
1551 OperandMatchResultTy
1552 AMDGPUAsmParser::parseRegOrImmWithFPInputMods(OperandVector &Operands,
1553                                               bool AllowImm) {
1554   // XXX: During parsing we can't determine if minus sign means
1555   // negate-modifier or negative immediate value.
1556   // By default we suppose it is modifier.
1557   bool Negate = false, Abs = false, Abs2 = false;
1558 
1559   if (getLexer().getKind()== AsmToken::Minus) {
1560     Parser.Lex();
1561     Negate = true;
1562   }
1563 
1564   if (getLexer().getKind() == AsmToken::Identifier &&
1565       Parser.getTok().getString() == "abs") {
1566     Parser.Lex();
1567     Abs2 = true;
1568     if (getLexer().isNot(AsmToken::LParen)) {
1569       Error(Parser.getTok().getLoc(), "expected left paren after abs");
1570       return MatchOperand_ParseFail;
1571     }
1572     Parser.Lex();
1573   }
1574 
1575   if (getLexer().getKind() == AsmToken::Pipe) {
1576     if (Abs2) {
1577       Error(Parser.getTok().getLoc(), "expected register or immediate");
1578       return MatchOperand_ParseFail;
1579     }
1580     Parser.Lex();
1581     Abs = true;
1582   }
1583 
1584   OperandMatchResultTy Res;
1585   if (AllowImm) {
1586     Res = parseRegOrImm(Operands);
1587   } else {
1588     Res = parseReg(Operands);
1589   }
1590   if (Res != MatchOperand_Success) {
1591     return Res;
1592   }
1593 
1594   AMDGPUOperand::Modifiers Mods;
1595   if (Negate) {
1596     Mods.Neg = true;
1597   }
1598   if (Abs) {
1599     if (getLexer().getKind() != AsmToken::Pipe) {
1600       Error(Parser.getTok().getLoc(), "expected vertical bar");
1601       return MatchOperand_ParseFail;
1602     }
1603     Parser.Lex();
1604     Mods.Abs = true;
1605   }
1606   if (Abs2) {
1607     if (getLexer().isNot(AsmToken::RParen)) {
1608       Error(Parser.getTok().getLoc(), "expected closing parentheses");
1609       return MatchOperand_ParseFail;
1610     }
1611     Parser.Lex();
1612     Mods.Abs = true;
1613   }
1614 
1615   if (Mods.hasFPModifiers()) {
1616     AMDGPUOperand &Op = static_cast<AMDGPUOperand &>(*Operands.back());
1617     Op.setModifiers(Mods);
1618   }
1619   return MatchOperand_Success;
1620 }
1621 
1622 OperandMatchResultTy
1623 AMDGPUAsmParser::parseRegOrImmWithIntInputMods(OperandVector &Operands,
1624                                                bool AllowImm) {
1625   bool Sext = false;
1626 
1627   if (getLexer().getKind() == AsmToken::Identifier &&
1628       Parser.getTok().getString() == "sext") {
1629     Parser.Lex();
1630     Sext = true;
1631     if (getLexer().isNot(AsmToken::LParen)) {
1632       Error(Parser.getTok().getLoc(), "expected left paren after sext");
1633       return MatchOperand_ParseFail;
1634     }
1635     Parser.Lex();
1636   }
1637 
1638   OperandMatchResultTy Res;
1639   if (AllowImm) {
1640     Res = parseRegOrImm(Operands);
1641   } else {
1642     Res = parseReg(Operands);
1643   }
1644   if (Res != MatchOperand_Success) {
1645     return Res;
1646   }
1647 
1648   AMDGPUOperand::Modifiers Mods;
1649   if (Sext) {
1650     if (getLexer().isNot(AsmToken::RParen)) {
1651       Error(Parser.getTok().getLoc(), "expected closing parentheses");
1652       return MatchOperand_ParseFail;
1653     }
1654     Parser.Lex();
1655     Mods.Sext = true;
1656   }
1657 
1658   if (Mods.hasIntModifiers()) {
1659     AMDGPUOperand &Op = static_cast<AMDGPUOperand &>(*Operands.back());
1660     Op.setModifiers(Mods);
1661   }
1662 
1663   return MatchOperand_Success;
1664 }
1665 
1666 OperandMatchResultTy
1667 AMDGPUAsmParser::parseRegWithFPInputMods(OperandVector &Operands) {
1668   return parseRegOrImmWithFPInputMods(Operands, false);
1669 }
1670 
1671 OperandMatchResultTy
1672 AMDGPUAsmParser::parseRegWithIntInputMods(OperandVector &Operands) {
1673   return parseRegOrImmWithIntInputMods(Operands, false);
1674 }
1675 
1676 OperandMatchResultTy AMDGPUAsmParser::parseVReg32OrOff(OperandVector &Operands) {
1677   std::unique_ptr<AMDGPUOperand> Reg = parseRegister();
1678   if (Reg) {
1679     Operands.push_back(std::move(Reg));
1680     return MatchOperand_Success;
1681   }
1682 
1683   const AsmToken &Tok = Parser.getTok();
1684   if (Tok.getString() == "off") {
1685     Operands.push_back(AMDGPUOperand::CreateImm(this, 0, Tok.getLoc(),
1686                                                 AMDGPUOperand::ImmTyOff, false));
1687     Parser.Lex();
1688     return MatchOperand_Success;
1689   }
1690 
1691   return MatchOperand_NoMatch;
1692 }
1693 
1694 unsigned AMDGPUAsmParser::checkTargetMatchPredicate(MCInst &Inst) {
1695   uint64_t TSFlags = MII.get(Inst.getOpcode()).TSFlags;
1696 
1697   if ((getForcedEncodingSize() == 32 && (TSFlags & SIInstrFlags::VOP3)) ||
1698       (getForcedEncodingSize() == 64 && !(TSFlags & SIInstrFlags::VOP3)) ||
1699       (isForcedDPP() && !(TSFlags & SIInstrFlags::DPP)) ||
1700       (isForcedSDWA() && !(TSFlags & SIInstrFlags::SDWA)) )
1701     return Match_InvalidOperand;
1702 
1703   if ((TSFlags & SIInstrFlags::VOP3) &&
1704       (TSFlags & SIInstrFlags::VOPAsmPrefer32Bit) &&
1705       getForcedEncodingSize() != 64)
1706     return Match_PreferE32;
1707 
1708   if (Inst.getOpcode() == AMDGPU::V_MAC_F32_sdwa_vi ||
1709       Inst.getOpcode() == AMDGPU::V_MAC_F16_sdwa_vi) {
1710     // v_mac_f32/16 allow only dst_sel == DWORD;
1711     auto OpNum =
1712         AMDGPU::getNamedOperandIdx(Inst.getOpcode(), AMDGPU::OpName::dst_sel);
1713     const auto &Op = Inst.getOperand(OpNum);
1714     if (!Op.isImm() || Op.getImm() != AMDGPU::SDWA::SdwaSel::DWORD) {
1715       return Match_InvalidOperand;
1716     }
1717   }
1718 
1719   return Match_Success;
1720 }
1721 
1722 // What asm variants we should check
1723 ArrayRef<unsigned> AMDGPUAsmParser::getMatchedVariants() const {
1724   if (getForcedEncodingSize() == 32) {
1725     static const unsigned Variants[] = {AMDGPUAsmVariants::DEFAULT};
1726     return makeArrayRef(Variants);
1727   }
1728 
1729   if (isForcedVOP3()) {
1730     static const unsigned Variants[] = {AMDGPUAsmVariants::VOP3};
1731     return makeArrayRef(Variants);
1732   }
1733 
1734   if (isForcedSDWA()) {
1735     static const unsigned Variants[] = {AMDGPUAsmVariants::SDWA};
1736     return makeArrayRef(Variants);
1737   }
1738 
1739   if (isForcedDPP()) {
1740     static const unsigned Variants[] = {AMDGPUAsmVariants::DPP};
1741     return makeArrayRef(Variants);
1742   }
1743 
1744   static const unsigned Variants[] = {
1745     AMDGPUAsmVariants::DEFAULT, AMDGPUAsmVariants::VOP3,
1746     AMDGPUAsmVariants::SDWA, AMDGPUAsmVariants::DPP
1747   };
1748 
1749   return makeArrayRef(Variants);
1750 }
1751 
1752 bool AMDGPUAsmParser::MatchAndEmitInstruction(SMLoc IDLoc, unsigned &Opcode,
1753                                               OperandVector &Operands,
1754                                               MCStreamer &Out,
1755                                               uint64_t &ErrorInfo,
1756                                               bool MatchingInlineAsm) {
1757   MCInst Inst;
1758   unsigned Result = Match_Success;
1759   for (auto Variant : getMatchedVariants()) {
1760     uint64_t EI;
1761     auto R = MatchInstructionImpl(Operands, Inst, EI, MatchingInlineAsm,
1762                                   Variant);
1763     // We order match statuses from least to most specific. We use most specific
1764     // status as resulting
1765     // Match_MnemonicFail < Match_InvalidOperand < Match_MissingFeature < Match_PreferE32
1766     if ((R == Match_Success) ||
1767         (R == Match_PreferE32) ||
1768         (R == Match_MissingFeature && Result != Match_PreferE32) ||
1769         (R == Match_InvalidOperand && Result != Match_MissingFeature
1770                                    && Result != Match_PreferE32) ||
1771         (R == Match_MnemonicFail   && Result != Match_InvalidOperand
1772                                    && Result != Match_MissingFeature
1773                                    && Result != Match_PreferE32)) {
1774       Result = R;
1775       ErrorInfo = EI;
1776     }
1777     if (R == Match_Success)
1778       break;
1779   }
1780 
1781   switch (Result) {
1782   default: break;
1783   case Match_Success:
1784     Inst.setLoc(IDLoc);
1785     Out.EmitInstruction(Inst, getSTI());
1786     return false;
1787 
1788   case Match_MissingFeature:
1789     return Error(IDLoc, "instruction not supported on this GPU");
1790 
1791   case Match_MnemonicFail:
1792     return Error(IDLoc, "unrecognized instruction mnemonic");
1793 
1794   case Match_InvalidOperand: {
1795     SMLoc ErrorLoc = IDLoc;
1796     if (ErrorInfo != ~0ULL) {
1797       if (ErrorInfo >= Operands.size()) {
1798         return Error(IDLoc, "too few operands for instruction");
1799       }
1800       ErrorLoc = ((AMDGPUOperand &)*Operands[ErrorInfo]).getStartLoc();
1801       if (ErrorLoc == SMLoc())
1802         ErrorLoc = IDLoc;
1803     }
1804     return Error(ErrorLoc, "invalid operand for instruction");
1805   }
1806 
1807   case Match_PreferE32:
1808     return Error(IDLoc, "internal error: instruction without _e64 suffix "
1809                         "should be encoded as e32");
1810   }
1811   llvm_unreachable("Implement any new match types added!");
1812 }
1813 
1814 bool AMDGPUAsmParser::ParseAsAbsoluteExpression(uint32_t &Ret) {
1815   int64_t Tmp = -1;
1816   if (getLexer().isNot(AsmToken::Integer) && getLexer().isNot(AsmToken::Identifier)) {
1817     return true;
1818   }
1819   if (getParser().parseAbsoluteExpression(Tmp)) {
1820     return true;
1821   }
1822   Ret = static_cast<uint32_t>(Tmp);
1823   return false;
1824 }
1825 
1826 bool AMDGPUAsmParser::ParseDirectiveMajorMinor(uint32_t &Major,
1827                                                uint32_t &Minor) {
1828   if (ParseAsAbsoluteExpression(Major))
1829     return TokError("invalid major version");
1830 
1831   if (getLexer().isNot(AsmToken::Comma))
1832     return TokError("minor version number required, comma expected");
1833   Lex();
1834 
1835   if (ParseAsAbsoluteExpression(Minor))
1836     return TokError("invalid minor version");
1837 
1838   return false;
1839 }
1840 
1841 bool AMDGPUAsmParser::ParseDirectiveHSACodeObjectVersion() {
1842   uint32_t Major;
1843   uint32_t Minor;
1844 
1845   if (ParseDirectiveMajorMinor(Major, Minor))
1846     return true;
1847 
1848   getTargetStreamer().EmitDirectiveHSACodeObjectVersion(Major, Minor);
1849   return false;
1850 }
1851 
1852 bool AMDGPUAsmParser::ParseDirectiveHSACodeObjectISA() {
1853   uint32_t Major;
1854   uint32_t Minor;
1855   uint32_t Stepping;
1856   StringRef VendorName;
1857   StringRef ArchName;
1858 
1859   // If this directive has no arguments, then use the ISA version for the
1860   // targeted GPU.
1861   if (getLexer().is(AsmToken::EndOfStatement)) {
1862     AMDGPU::IsaVersion Isa = AMDGPU::getIsaVersion(getSTI().getFeatureBits());
1863     getTargetStreamer().EmitDirectiveHSACodeObjectISA(Isa.Major, Isa.Minor,
1864                                                       Isa.Stepping,
1865                                                       "AMD", "AMDGPU");
1866     return false;
1867   }
1868 
1869   if (ParseDirectiveMajorMinor(Major, Minor))
1870     return true;
1871 
1872   if (getLexer().isNot(AsmToken::Comma))
1873     return TokError("stepping version number required, comma expected");
1874   Lex();
1875 
1876   if (ParseAsAbsoluteExpression(Stepping))
1877     return TokError("invalid stepping version");
1878 
1879   if (getLexer().isNot(AsmToken::Comma))
1880     return TokError("vendor name required, comma expected");
1881   Lex();
1882 
1883   if (getLexer().isNot(AsmToken::String))
1884     return TokError("invalid vendor name");
1885 
1886   VendorName = getLexer().getTok().getStringContents();
1887   Lex();
1888 
1889   if (getLexer().isNot(AsmToken::Comma))
1890     return TokError("arch name required, comma expected");
1891   Lex();
1892 
1893   if (getLexer().isNot(AsmToken::String))
1894     return TokError("invalid arch name");
1895 
1896   ArchName = getLexer().getTok().getStringContents();
1897   Lex();
1898 
1899   getTargetStreamer().EmitDirectiveHSACodeObjectISA(Major, Minor, Stepping,
1900                                                     VendorName, ArchName);
1901   return false;
1902 }
1903 
1904 bool AMDGPUAsmParser::ParseDirectiveRuntimeMetadata() {
1905   std::string Metadata;
1906   raw_string_ostream MS(Metadata);
1907 
1908   getLexer().setSkipSpace(false);
1909 
1910   bool FoundEnd = false;
1911   while (!getLexer().is(AsmToken::Eof)) {
1912     while (getLexer().is(AsmToken::Space)) {
1913       MS << ' ';
1914       Lex();
1915     }
1916 
1917     if (getLexer().is(AsmToken::Identifier)) {
1918       StringRef ID = getLexer().getTok().getIdentifier();
1919       if (ID == ".end_amdgpu_runtime_metadata") {
1920         Lex();
1921         FoundEnd = true;
1922         break;
1923       }
1924     }
1925 
1926     MS << Parser.parseStringToEndOfStatement()
1927        << getContext().getAsmInfo()->getSeparatorString();
1928 
1929     Parser.eatToEndOfStatement();
1930   }
1931 
1932   getLexer().setSkipSpace(true);
1933 
1934   if (getLexer().is(AsmToken::Eof) && !FoundEnd)
1935     return TokError("expected directive .end_amdgpu_runtime_metadata not found");
1936 
1937   MS.flush();
1938 
1939   getTargetStreamer().EmitRuntimeMetadata(Metadata);
1940 
1941   return false;
1942 }
1943 
1944 bool AMDGPUAsmParser::ParseAMDKernelCodeTValue(StringRef ID,
1945                                                amd_kernel_code_t &Header) {
1946   SmallString<40> ErrStr;
1947   raw_svector_ostream Err(ErrStr);
1948   if (!parseAmdKernelCodeField(ID, getParser(), Header, Err)) {
1949     return TokError(Err.str());
1950   }
1951   Lex();
1952   return false;
1953 }
1954 
1955 bool AMDGPUAsmParser::ParseDirectiveAMDKernelCodeT() {
1956   amd_kernel_code_t Header;
1957   AMDGPU::initDefaultAMDKernelCodeT(Header, getSTI().getFeatureBits());
1958 
1959   while (true) {
1960     // Lex EndOfStatement.  This is in a while loop, because lexing a comment
1961     // will set the current token to EndOfStatement.
1962     while(getLexer().is(AsmToken::EndOfStatement))
1963       Lex();
1964 
1965     if (getLexer().isNot(AsmToken::Identifier))
1966       return TokError("expected value identifier or .end_amd_kernel_code_t");
1967 
1968     StringRef ID = getLexer().getTok().getIdentifier();
1969     Lex();
1970 
1971     if (ID == ".end_amd_kernel_code_t")
1972       break;
1973 
1974     if (ParseAMDKernelCodeTValue(ID, Header))
1975       return true;
1976   }
1977 
1978   getTargetStreamer().EmitAMDKernelCodeT(Header);
1979 
1980   return false;
1981 }
1982 
1983 bool AMDGPUAsmParser::ParseSectionDirectiveHSAText() {
1984   getParser().getStreamer().SwitchSection(
1985       AMDGPU::getHSATextSection(getContext()));
1986   return false;
1987 }
1988 
1989 bool AMDGPUAsmParser::ParseDirectiveAMDGPUHsaKernel() {
1990   if (getLexer().isNot(AsmToken::Identifier))
1991     return TokError("expected symbol name");
1992 
1993   StringRef KernelName = Parser.getTok().getString();
1994 
1995   getTargetStreamer().EmitAMDGPUSymbolType(KernelName,
1996                                            ELF::STT_AMDGPU_HSA_KERNEL);
1997   Lex();
1998   KernelScope.initialize(getContext());
1999   return false;
2000 }
2001 
2002 bool AMDGPUAsmParser::ParseDirectiveAMDGPUHsaModuleGlobal() {
2003   if (getLexer().isNot(AsmToken::Identifier))
2004     return TokError("expected symbol name");
2005 
2006   StringRef GlobalName = Parser.getTok().getIdentifier();
2007 
2008   getTargetStreamer().EmitAMDGPUHsaModuleScopeGlobal(GlobalName);
2009   Lex();
2010   return false;
2011 }
2012 
2013 bool AMDGPUAsmParser::ParseDirectiveAMDGPUHsaProgramGlobal() {
2014   if (getLexer().isNot(AsmToken::Identifier))
2015     return TokError("expected symbol name");
2016 
2017   StringRef GlobalName = Parser.getTok().getIdentifier();
2018 
2019   getTargetStreamer().EmitAMDGPUHsaProgramScopeGlobal(GlobalName);
2020   Lex();
2021   return false;
2022 }
2023 
2024 bool AMDGPUAsmParser::ParseSectionDirectiveHSADataGlobalAgent() {
2025   getParser().getStreamer().SwitchSection(
2026       AMDGPU::getHSADataGlobalAgentSection(getContext()));
2027   return false;
2028 }
2029 
2030 bool AMDGPUAsmParser::ParseSectionDirectiveHSADataGlobalProgram() {
2031   getParser().getStreamer().SwitchSection(
2032       AMDGPU::getHSADataGlobalProgramSection(getContext()));
2033   return false;
2034 }
2035 
2036 bool AMDGPUAsmParser::ParseSectionDirectiveHSARodataReadonlyAgent() {
2037   getParser().getStreamer().SwitchSection(
2038       AMDGPU::getHSARodataReadonlyAgentSection(getContext()));
2039   return false;
2040 }
2041 
2042 bool AMDGPUAsmParser::ParseDirective(AsmToken DirectiveID) {
2043   StringRef IDVal = DirectiveID.getString();
2044 
2045   if (IDVal == ".hsa_code_object_version")
2046     return ParseDirectiveHSACodeObjectVersion();
2047 
2048   if (IDVal == ".hsa_code_object_isa")
2049     return ParseDirectiveHSACodeObjectISA();
2050 
2051   if (IDVal == ".amdgpu_runtime_metadata")
2052     return ParseDirectiveRuntimeMetadata();
2053 
2054   if (IDVal == ".amd_kernel_code_t")
2055     return ParseDirectiveAMDKernelCodeT();
2056 
2057   if (IDVal == ".hsatext")
2058     return ParseSectionDirectiveHSAText();
2059 
2060   if (IDVal == ".amdgpu_hsa_kernel")
2061     return ParseDirectiveAMDGPUHsaKernel();
2062 
2063   if (IDVal == ".amdgpu_hsa_module_global")
2064     return ParseDirectiveAMDGPUHsaModuleGlobal();
2065 
2066   if (IDVal == ".amdgpu_hsa_program_global")
2067     return ParseDirectiveAMDGPUHsaProgramGlobal();
2068 
2069   if (IDVal == ".hsadata_global_agent")
2070     return ParseSectionDirectiveHSADataGlobalAgent();
2071 
2072   if (IDVal == ".hsadata_global_program")
2073     return ParseSectionDirectiveHSADataGlobalProgram();
2074 
2075   if (IDVal == ".hsarodata_readonly_agent")
2076     return ParseSectionDirectiveHSARodataReadonlyAgent();
2077 
2078   return true;
2079 }
2080 
2081 bool AMDGPUAsmParser::subtargetHasRegister(const MCRegisterInfo &MRI,
2082                                            unsigned RegNo) const {
2083   if (isCI())
2084     return true;
2085 
2086   if (isSI()) {
2087     // No flat_scr
2088     switch (RegNo) {
2089     case AMDGPU::FLAT_SCR:
2090     case AMDGPU::FLAT_SCR_LO:
2091     case AMDGPU::FLAT_SCR_HI:
2092       return false;
2093     default:
2094       return true;
2095     }
2096   }
2097 
2098   // VI only has 102 SGPRs, so make sure we aren't trying to use the 2 more that
2099   // SI/CI have.
2100   for (MCRegAliasIterator R(AMDGPU::SGPR102_SGPR103, &MRI, true);
2101        R.isValid(); ++R) {
2102     if (*R == RegNo)
2103       return false;
2104   }
2105 
2106   return true;
2107 }
2108 
2109 OperandMatchResultTy
2110 AMDGPUAsmParser::parseOperand(OperandVector &Operands, StringRef Mnemonic) {
2111   // Try to parse with a custom parser
2112   OperandMatchResultTy ResTy = MatchOperandParserImpl(Operands, Mnemonic);
2113 
2114   // If we successfully parsed the operand or if there as an error parsing,
2115   // we are done.
2116   //
2117   // If we are parsing after we reach EndOfStatement then this means we
2118   // are appending default values to the Operands list.  This is only done
2119   // by custom parser, so we shouldn't continue on to the generic parsing.
2120   if (ResTy == MatchOperand_Success || ResTy == MatchOperand_ParseFail ||
2121       getLexer().is(AsmToken::EndOfStatement))
2122     return ResTy;
2123 
2124   ResTy = parseRegOrImm(Operands);
2125 
2126   if (ResTy == MatchOperand_Success)
2127     return ResTy;
2128 
2129   if (getLexer().getKind() == AsmToken::Identifier) {
2130     // If this identifier is a symbol, we want to create an expression for it.
2131     // It is a little difficult to distinguish between a symbol name, and
2132     // an instruction flag like 'gds'.  In order to do this, we parse
2133     // all tokens as expressions and then treate the symbol name as the token
2134     // string when we want to interpret the operand as a token.
2135     const auto &Tok = Parser.getTok();
2136     SMLoc S = Tok.getLoc();
2137     const MCExpr *Expr = nullptr;
2138     if (!Parser.parseExpression(Expr)) {
2139       Operands.push_back(AMDGPUOperand::CreateExpr(this, Expr, S));
2140       return MatchOperand_Success;
2141     }
2142 
2143     Operands.push_back(AMDGPUOperand::CreateToken(this, Tok.getString(), Tok.getLoc()));
2144     Parser.Lex();
2145     return MatchOperand_Success;
2146   }
2147   return MatchOperand_NoMatch;
2148 }
2149 
2150 StringRef AMDGPUAsmParser::parseMnemonicSuffix(StringRef Name) {
2151   // Clear any forced encodings from the previous instruction.
2152   setForcedEncodingSize(0);
2153   setForcedDPP(false);
2154   setForcedSDWA(false);
2155 
2156   if (Name.endswith("_e64")) {
2157     setForcedEncodingSize(64);
2158     return Name.substr(0, Name.size() - 4);
2159   } else if (Name.endswith("_e32")) {
2160     setForcedEncodingSize(32);
2161     return Name.substr(0, Name.size() - 4);
2162   } else if (Name.endswith("_dpp")) {
2163     setForcedDPP(true);
2164     return Name.substr(0, Name.size() - 4);
2165   } else if (Name.endswith("_sdwa")) {
2166     setForcedSDWA(true);
2167     return Name.substr(0, Name.size() - 5);
2168   }
2169   return Name;
2170 }
2171 
2172 bool AMDGPUAsmParser::ParseInstruction(ParseInstructionInfo &Info,
2173                                        StringRef Name,
2174                                        SMLoc NameLoc, OperandVector &Operands) {
2175   // Add the instruction mnemonic
2176   Name = parseMnemonicSuffix(Name);
2177   Operands.push_back(AMDGPUOperand::CreateToken(this, Name, NameLoc));
2178 
2179   while (!getLexer().is(AsmToken::EndOfStatement)) {
2180     OperandMatchResultTy Res = parseOperand(Operands, Name);
2181 
2182     // Eat the comma or space if there is one.
2183     if (getLexer().is(AsmToken::Comma))
2184       Parser.Lex();
2185 
2186     switch (Res) {
2187       case MatchOperand_Success: break;
2188       case MatchOperand_ParseFail:
2189         Error(getLexer().getLoc(), "failed parsing operand.");
2190         while (!getLexer().is(AsmToken::EndOfStatement)) {
2191           Parser.Lex();
2192         }
2193         return true;
2194       case MatchOperand_NoMatch:
2195         Error(getLexer().getLoc(), "not a valid operand.");
2196         while (!getLexer().is(AsmToken::EndOfStatement)) {
2197           Parser.Lex();
2198         }
2199         return true;
2200     }
2201   }
2202 
2203   return false;
2204 }
2205 
2206 //===----------------------------------------------------------------------===//
2207 // Utility functions
2208 //===----------------------------------------------------------------------===//
2209 
2210 OperandMatchResultTy
2211 AMDGPUAsmParser::parseIntWithPrefix(const char *Prefix, int64_t &Int) {
2212   switch(getLexer().getKind()) {
2213     default: return MatchOperand_NoMatch;
2214     case AsmToken::Identifier: {
2215       StringRef Name = Parser.getTok().getString();
2216       if (!Name.equals(Prefix)) {
2217         return MatchOperand_NoMatch;
2218       }
2219 
2220       Parser.Lex();
2221       if (getLexer().isNot(AsmToken::Colon))
2222         return MatchOperand_ParseFail;
2223 
2224       Parser.Lex();
2225       if (getLexer().isNot(AsmToken::Integer))
2226         return MatchOperand_ParseFail;
2227 
2228       if (getParser().parseAbsoluteExpression(Int))
2229         return MatchOperand_ParseFail;
2230       break;
2231     }
2232   }
2233   return MatchOperand_Success;
2234 }
2235 
2236 OperandMatchResultTy
2237 AMDGPUAsmParser::parseIntWithPrefix(const char *Prefix, OperandVector &Operands,
2238                                     enum AMDGPUOperand::ImmTy ImmTy,
2239                                     bool (*ConvertResult)(int64_t&)) {
2240   SMLoc S = Parser.getTok().getLoc();
2241   int64_t Value = 0;
2242 
2243   OperandMatchResultTy Res = parseIntWithPrefix(Prefix, Value);
2244   if (Res != MatchOperand_Success)
2245     return Res;
2246 
2247   if (ConvertResult && !ConvertResult(Value)) {
2248     return MatchOperand_ParseFail;
2249   }
2250 
2251   Operands.push_back(AMDGPUOperand::CreateImm(this, Value, S, ImmTy));
2252   return MatchOperand_Success;
2253 }
2254 
2255 OperandMatchResultTy
2256 AMDGPUAsmParser::parseNamedBit(const char *Name, OperandVector &Operands,
2257                                enum AMDGPUOperand::ImmTy ImmTy) {
2258   int64_t Bit = 0;
2259   SMLoc S = Parser.getTok().getLoc();
2260 
2261   // We are at the end of the statement, and this is a default argument, so
2262   // use a default value.
2263   if (getLexer().isNot(AsmToken::EndOfStatement)) {
2264     switch(getLexer().getKind()) {
2265       case AsmToken::Identifier: {
2266         StringRef Tok = Parser.getTok().getString();
2267         if (Tok == Name) {
2268           Bit = 1;
2269           Parser.Lex();
2270         } else if (Tok.startswith("no") && Tok.endswith(Name)) {
2271           Bit = 0;
2272           Parser.Lex();
2273         } else {
2274           return MatchOperand_NoMatch;
2275         }
2276         break;
2277       }
2278       default:
2279         return MatchOperand_NoMatch;
2280     }
2281   }
2282 
2283   Operands.push_back(AMDGPUOperand::CreateImm(this, Bit, S, ImmTy));
2284   return MatchOperand_Success;
2285 }
2286 
2287 typedef std::map<enum AMDGPUOperand::ImmTy, unsigned> OptionalImmIndexMap;
2288 
2289 void addOptionalImmOperand(MCInst& Inst, const OperandVector& Operands,
2290                            OptionalImmIndexMap& OptionalIdx,
2291                            enum AMDGPUOperand::ImmTy ImmT, int64_t Default = 0) {
2292   auto i = OptionalIdx.find(ImmT);
2293   if (i != OptionalIdx.end()) {
2294     unsigned Idx = i->second;
2295     ((AMDGPUOperand &)*Operands[Idx]).addImmOperands(Inst, 1);
2296   } else {
2297     Inst.addOperand(MCOperand::createImm(Default));
2298   }
2299 }
2300 
2301 OperandMatchResultTy
2302 AMDGPUAsmParser::parseStringWithPrefix(StringRef Prefix, StringRef &Value) {
2303   if (getLexer().isNot(AsmToken::Identifier)) {
2304     return MatchOperand_NoMatch;
2305   }
2306   StringRef Tok = Parser.getTok().getString();
2307   if (Tok != Prefix) {
2308     return MatchOperand_NoMatch;
2309   }
2310 
2311   Parser.Lex();
2312   if (getLexer().isNot(AsmToken::Colon)) {
2313     return MatchOperand_ParseFail;
2314   }
2315 
2316   Parser.Lex();
2317   if (getLexer().isNot(AsmToken::Identifier)) {
2318     return MatchOperand_ParseFail;
2319   }
2320 
2321   Value = Parser.getTok().getString();
2322   return MatchOperand_Success;
2323 }
2324 
2325 //===----------------------------------------------------------------------===//
2326 // ds
2327 //===----------------------------------------------------------------------===//
2328 
2329 void AMDGPUAsmParser::cvtDSOffset01(MCInst &Inst,
2330                                     const OperandVector &Operands) {
2331   OptionalImmIndexMap OptionalIdx;
2332 
2333   for (unsigned i = 1, e = Operands.size(); i != e; ++i) {
2334     AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[i]);
2335 
2336     // Add the register arguments
2337     if (Op.isReg()) {
2338       Op.addRegOperands(Inst, 1);
2339       continue;
2340     }
2341 
2342     // Handle optional arguments
2343     OptionalIdx[Op.getImmTy()] = i;
2344   }
2345 
2346   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyOffset0);
2347   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyOffset1);
2348   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyGDS);
2349 
2350   Inst.addOperand(MCOperand::createReg(AMDGPU::M0)); // m0
2351 }
2352 
2353 void AMDGPUAsmParser::cvtDS(MCInst &Inst, const OperandVector &Operands) {
2354   std::map<enum AMDGPUOperand::ImmTy, unsigned> OptionalIdx;
2355   bool GDSOnly = false;
2356 
2357   for (unsigned i = 1, e = Operands.size(); i != e; ++i) {
2358     AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[i]);
2359 
2360     // Add the register arguments
2361     if (Op.isReg()) {
2362       Op.addRegOperands(Inst, 1);
2363       continue;
2364     }
2365 
2366     if (Op.isToken() && Op.getToken() == "gds") {
2367       GDSOnly = true;
2368       continue;
2369     }
2370 
2371     // Handle optional arguments
2372     OptionalIdx[Op.getImmTy()] = i;
2373   }
2374 
2375   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyOffset);
2376   if (!GDSOnly) {
2377     addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyGDS);
2378   }
2379   Inst.addOperand(MCOperand::createReg(AMDGPU::M0)); // m0
2380 }
2381 
2382 void AMDGPUAsmParser::cvtExp(MCInst &Inst, const OperandVector &Operands) {
2383   OptionalImmIndexMap OptionalIdx;
2384 
2385   unsigned EnMask = 0;
2386   int SrcIdx = 0;
2387 
2388   for (unsigned i = 1, e = Operands.size(); i != e; ++i) {
2389     AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[i]);
2390 
2391     // Add the register arguments
2392     if (Op.isReg()) {
2393       EnMask |= (1 << SrcIdx);
2394       Op.addRegOperands(Inst, 1);
2395       ++SrcIdx;
2396       continue;
2397     }
2398 
2399     if (Op.isOff()) {
2400       ++SrcIdx;
2401       Inst.addOperand(MCOperand::createReg(AMDGPU::NoRegister));
2402       continue;
2403     }
2404 
2405     if (Op.isImm() && Op.getImmTy() == AMDGPUOperand::ImmTyExpTgt) {
2406       Op.addImmOperands(Inst, 1);
2407       continue;
2408     }
2409 
2410     if (Op.isToken() && Op.getToken() == "done")
2411       continue;
2412 
2413     // Handle optional arguments
2414     OptionalIdx[Op.getImmTy()] = i;
2415   }
2416 
2417   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyExpVM);
2418   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyExpCompr);
2419 
2420   Inst.addOperand(MCOperand::createImm(EnMask));
2421 }
2422 
2423 //===----------------------------------------------------------------------===//
2424 // s_waitcnt
2425 //===----------------------------------------------------------------------===//
2426 
2427 bool AMDGPUAsmParser::parseCnt(int64_t &IntVal) {
2428   StringRef CntName = Parser.getTok().getString();
2429   int64_t CntVal;
2430 
2431   Parser.Lex();
2432   if (getLexer().isNot(AsmToken::LParen))
2433     return true;
2434 
2435   Parser.Lex();
2436   if (getLexer().isNot(AsmToken::Integer))
2437     return true;
2438 
2439   if (getParser().parseAbsoluteExpression(CntVal))
2440     return true;
2441 
2442   if (getLexer().isNot(AsmToken::RParen))
2443     return true;
2444 
2445   Parser.Lex();
2446   if (getLexer().is(AsmToken::Amp) || getLexer().is(AsmToken::Comma))
2447     Parser.Lex();
2448 
2449   IsaVersion IV = getIsaVersion(getSTI().getFeatureBits());
2450   if (CntName == "vmcnt")
2451     IntVal = encodeVmcnt(IV, IntVal, CntVal);
2452   else if (CntName == "expcnt")
2453     IntVal = encodeExpcnt(IV, IntVal, CntVal);
2454   else if (CntName == "lgkmcnt")
2455     IntVal = encodeLgkmcnt(IV, IntVal, CntVal);
2456   else
2457     return true;
2458 
2459   return false;
2460 }
2461 
2462 OperandMatchResultTy
2463 AMDGPUAsmParser::parseSWaitCntOps(OperandVector &Operands) {
2464   IsaVersion IV = getIsaVersion(getSTI().getFeatureBits());
2465   int64_t Waitcnt = getWaitcntBitMask(IV);
2466   SMLoc S = Parser.getTok().getLoc();
2467 
2468   switch(getLexer().getKind()) {
2469     default: return MatchOperand_ParseFail;
2470     case AsmToken::Integer:
2471       // The operand can be an integer value.
2472       if (getParser().parseAbsoluteExpression(Waitcnt))
2473         return MatchOperand_ParseFail;
2474       break;
2475 
2476     case AsmToken::Identifier:
2477       do {
2478         if (parseCnt(Waitcnt))
2479           return MatchOperand_ParseFail;
2480       } while(getLexer().isNot(AsmToken::EndOfStatement));
2481       break;
2482   }
2483   Operands.push_back(AMDGPUOperand::CreateImm(this, Waitcnt, S));
2484   return MatchOperand_Success;
2485 }
2486 
2487 bool AMDGPUAsmParser::parseHwregConstruct(OperandInfoTy &HwReg, int64_t &Offset, int64_t &Width) {
2488   using namespace llvm::AMDGPU::Hwreg;
2489 
2490   if (Parser.getTok().getString() != "hwreg")
2491     return true;
2492   Parser.Lex();
2493 
2494   if (getLexer().isNot(AsmToken::LParen))
2495     return true;
2496   Parser.Lex();
2497 
2498   if (getLexer().is(AsmToken::Identifier)) {
2499     HwReg.IsSymbolic = true;
2500     HwReg.Id = ID_UNKNOWN_;
2501     const StringRef tok = Parser.getTok().getString();
2502     for (int i = ID_SYMBOLIC_FIRST_; i < ID_SYMBOLIC_LAST_; ++i) {
2503       if (tok == IdSymbolic[i]) {
2504         HwReg.Id = i;
2505         break;
2506       }
2507     }
2508     Parser.Lex();
2509   } else {
2510     HwReg.IsSymbolic = false;
2511     if (getLexer().isNot(AsmToken::Integer))
2512       return true;
2513     if (getParser().parseAbsoluteExpression(HwReg.Id))
2514       return true;
2515   }
2516 
2517   if (getLexer().is(AsmToken::RParen)) {
2518     Parser.Lex();
2519     return false;
2520   }
2521 
2522   // optional params
2523   if (getLexer().isNot(AsmToken::Comma))
2524     return true;
2525   Parser.Lex();
2526 
2527   if (getLexer().isNot(AsmToken::Integer))
2528     return true;
2529   if (getParser().parseAbsoluteExpression(Offset))
2530     return true;
2531 
2532   if (getLexer().isNot(AsmToken::Comma))
2533     return true;
2534   Parser.Lex();
2535 
2536   if (getLexer().isNot(AsmToken::Integer))
2537     return true;
2538   if (getParser().parseAbsoluteExpression(Width))
2539     return true;
2540 
2541   if (getLexer().isNot(AsmToken::RParen))
2542     return true;
2543   Parser.Lex();
2544 
2545   return false;
2546 }
2547 
2548 OperandMatchResultTy
2549 AMDGPUAsmParser::parseHwreg(OperandVector &Operands) {
2550   using namespace llvm::AMDGPU::Hwreg;
2551 
2552   int64_t Imm16Val = 0;
2553   SMLoc S = Parser.getTok().getLoc();
2554 
2555   switch(getLexer().getKind()) {
2556     default: return MatchOperand_NoMatch;
2557     case AsmToken::Integer:
2558       // The operand can be an integer value.
2559       if (getParser().parseAbsoluteExpression(Imm16Val))
2560         return MatchOperand_NoMatch;
2561       if (Imm16Val < 0 || !isUInt<16>(Imm16Val)) {
2562         Error(S, "invalid immediate: only 16-bit values are legal");
2563         // Do not return error code, but create an imm operand anyway and proceed
2564         // to the next operand, if any. That avoids unneccessary error messages.
2565       }
2566       break;
2567 
2568     case AsmToken::Identifier: {
2569         OperandInfoTy HwReg(ID_UNKNOWN_);
2570         int64_t Offset = OFFSET_DEFAULT_;
2571         int64_t Width = WIDTH_M1_DEFAULT_ + 1;
2572         if (parseHwregConstruct(HwReg, Offset, Width))
2573           return MatchOperand_ParseFail;
2574         if (HwReg.Id < 0 || !isUInt<ID_WIDTH_>(HwReg.Id)) {
2575           if (HwReg.IsSymbolic)
2576             Error(S, "invalid symbolic name of hardware register");
2577           else
2578             Error(S, "invalid code of hardware register: only 6-bit values are legal");
2579         }
2580         if (Offset < 0 || !isUInt<OFFSET_WIDTH_>(Offset))
2581           Error(S, "invalid bit offset: only 5-bit values are legal");
2582         if ((Width-1) < 0 || !isUInt<WIDTH_M1_WIDTH_>(Width-1))
2583           Error(S, "invalid bitfield width: only values from 1 to 32 are legal");
2584         Imm16Val = (HwReg.Id << ID_SHIFT_) | (Offset << OFFSET_SHIFT_) | ((Width-1) << WIDTH_M1_SHIFT_);
2585       }
2586       break;
2587   }
2588   Operands.push_back(AMDGPUOperand::CreateImm(this, Imm16Val, S, AMDGPUOperand::ImmTyHwreg));
2589   return MatchOperand_Success;
2590 }
2591 
2592 bool AMDGPUOperand::isSWaitCnt() const {
2593   return isImm();
2594 }
2595 
2596 bool AMDGPUOperand::isHwreg() const {
2597   return isImmTy(ImmTyHwreg);
2598 }
2599 
2600 bool AMDGPUAsmParser::parseSendMsgConstruct(OperandInfoTy &Msg, OperandInfoTy &Operation, int64_t &StreamId) {
2601   using namespace llvm::AMDGPU::SendMsg;
2602 
2603   if (Parser.getTok().getString() != "sendmsg")
2604     return true;
2605   Parser.Lex();
2606 
2607   if (getLexer().isNot(AsmToken::LParen))
2608     return true;
2609   Parser.Lex();
2610 
2611   if (getLexer().is(AsmToken::Identifier)) {
2612     Msg.IsSymbolic = true;
2613     Msg.Id = ID_UNKNOWN_;
2614     const std::string tok = Parser.getTok().getString();
2615     for (int i = ID_GAPS_FIRST_; i < ID_GAPS_LAST_; ++i) {
2616       switch(i) {
2617         default: continue; // Omit gaps.
2618         case ID_INTERRUPT: case ID_GS: case ID_GS_DONE:  case ID_SYSMSG: break;
2619       }
2620       if (tok == IdSymbolic[i]) {
2621         Msg.Id = i;
2622         break;
2623       }
2624     }
2625     Parser.Lex();
2626   } else {
2627     Msg.IsSymbolic = false;
2628     if (getLexer().isNot(AsmToken::Integer))
2629       return true;
2630     if (getParser().parseAbsoluteExpression(Msg.Id))
2631       return true;
2632     if (getLexer().is(AsmToken::Integer))
2633       if (getParser().parseAbsoluteExpression(Msg.Id))
2634         Msg.Id = ID_UNKNOWN_;
2635   }
2636   if (Msg.Id == ID_UNKNOWN_) // Don't know how to parse the rest.
2637     return false;
2638 
2639   if (!(Msg.Id == ID_GS || Msg.Id == ID_GS_DONE || Msg.Id == ID_SYSMSG)) {
2640     if (getLexer().isNot(AsmToken::RParen))
2641       return true;
2642     Parser.Lex();
2643     return false;
2644   }
2645 
2646   if (getLexer().isNot(AsmToken::Comma))
2647     return true;
2648   Parser.Lex();
2649 
2650   assert(Msg.Id == ID_GS || Msg.Id == ID_GS_DONE || Msg.Id == ID_SYSMSG);
2651   Operation.Id = ID_UNKNOWN_;
2652   if (getLexer().is(AsmToken::Identifier)) {
2653     Operation.IsSymbolic = true;
2654     const char* const *S = (Msg.Id == ID_SYSMSG) ? OpSysSymbolic : OpGsSymbolic;
2655     const int F = (Msg.Id == ID_SYSMSG) ? OP_SYS_FIRST_ : OP_GS_FIRST_;
2656     const int L = (Msg.Id == ID_SYSMSG) ? OP_SYS_LAST_ : OP_GS_LAST_;
2657     const StringRef Tok = Parser.getTok().getString();
2658     for (int i = F; i < L; ++i) {
2659       if (Tok == S[i]) {
2660         Operation.Id = i;
2661         break;
2662       }
2663     }
2664     Parser.Lex();
2665   } else {
2666     Operation.IsSymbolic = false;
2667     if (getLexer().isNot(AsmToken::Integer))
2668       return true;
2669     if (getParser().parseAbsoluteExpression(Operation.Id))
2670       return true;
2671   }
2672 
2673   if ((Msg.Id == ID_GS || Msg.Id == ID_GS_DONE) && Operation.Id != OP_GS_NOP) {
2674     // Stream id is optional.
2675     if (getLexer().is(AsmToken::RParen)) {
2676       Parser.Lex();
2677       return false;
2678     }
2679 
2680     if (getLexer().isNot(AsmToken::Comma))
2681       return true;
2682     Parser.Lex();
2683 
2684     if (getLexer().isNot(AsmToken::Integer))
2685       return true;
2686     if (getParser().parseAbsoluteExpression(StreamId))
2687       return true;
2688   }
2689 
2690   if (getLexer().isNot(AsmToken::RParen))
2691     return true;
2692   Parser.Lex();
2693   return false;
2694 }
2695 
2696 OperandMatchResultTy AMDGPUAsmParser::parseInterpSlot(OperandVector &Operands) {
2697   if (getLexer().getKind() != AsmToken::Identifier)
2698     return MatchOperand_NoMatch;
2699 
2700   StringRef Str = Parser.getTok().getString();
2701   int Slot = StringSwitch<int>(Str)
2702     .Case("p10", 0)
2703     .Case("p20", 1)
2704     .Case("p0", 2)
2705     .Default(-1);
2706 
2707   SMLoc S = Parser.getTok().getLoc();
2708   if (Slot == -1)
2709     return MatchOperand_ParseFail;
2710 
2711   Parser.Lex();
2712   Operands.push_back(AMDGPUOperand::CreateImm(this, Slot, S,
2713                                               AMDGPUOperand::ImmTyInterpSlot));
2714   return MatchOperand_Success;
2715 }
2716 
2717 OperandMatchResultTy AMDGPUAsmParser::parseInterpAttr(OperandVector &Operands) {
2718   if (getLexer().getKind() != AsmToken::Identifier)
2719     return MatchOperand_NoMatch;
2720 
2721   StringRef Str = Parser.getTok().getString();
2722   if (!Str.startswith("attr"))
2723     return MatchOperand_NoMatch;
2724 
2725   StringRef Chan = Str.take_back(2);
2726   int AttrChan = StringSwitch<int>(Chan)
2727     .Case(".x", 0)
2728     .Case(".y", 1)
2729     .Case(".z", 2)
2730     .Case(".w", 3)
2731     .Default(-1);
2732   if (AttrChan == -1)
2733     return MatchOperand_ParseFail;
2734 
2735   Str = Str.drop_back(2).drop_front(4);
2736 
2737   uint8_t Attr;
2738   if (Str.getAsInteger(10, Attr))
2739     return MatchOperand_ParseFail;
2740 
2741   SMLoc S = Parser.getTok().getLoc();
2742   Parser.Lex();
2743   if (Attr > 63) {
2744     Error(S, "out of bounds attr");
2745     return MatchOperand_Success;
2746   }
2747 
2748   SMLoc SChan = SMLoc::getFromPointer(Chan.data());
2749 
2750   Operands.push_back(AMDGPUOperand::CreateImm(this, Attr, S,
2751                                               AMDGPUOperand::ImmTyInterpAttr));
2752   Operands.push_back(AMDGPUOperand::CreateImm(this, AttrChan, SChan,
2753                                               AMDGPUOperand::ImmTyAttrChan));
2754   return MatchOperand_Success;
2755 }
2756 
2757 void AMDGPUAsmParser::errorExpTgt() {
2758   Error(Parser.getTok().getLoc(), "invalid exp target");
2759 }
2760 
2761 OperandMatchResultTy AMDGPUAsmParser::parseExpTgtImpl(StringRef Str,
2762                                                       uint8_t &Val) {
2763   if (Str == "null") {
2764     Val = 9;
2765     return MatchOperand_Success;
2766   }
2767 
2768   if (Str.startswith("mrt")) {
2769     Str = Str.drop_front(3);
2770     if (Str == "z") { // == mrtz
2771       Val = 8;
2772       return MatchOperand_Success;
2773     }
2774 
2775     if (Str.getAsInteger(10, Val))
2776       return MatchOperand_ParseFail;
2777 
2778     if (Val > 7)
2779       errorExpTgt();
2780 
2781     return MatchOperand_Success;
2782   }
2783 
2784   if (Str.startswith("pos")) {
2785     Str = Str.drop_front(3);
2786     if (Str.getAsInteger(10, Val))
2787       return MatchOperand_ParseFail;
2788 
2789     if (Val > 3)
2790       errorExpTgt();
2791 
2792     Val += 12;
2793     return MatchOperand_Success;
2794   }
2795 
2796   if (Str.startswith("param")) {
2797     Str = Str.drop_front(5);
2798     if (Str.getAsInteger(10, Val))
2799       return MatchOperand_ParseFail;
2800 
2801     if (Val >= 32)
2802       errorExpTgt();
2803 
2804     Val += 32;
2805     return MatchOperand_Success;
2806   }
2807 
2808   if (Str.startswith("invalid_target_")) {
2809     Str = Str.drop_front(15);
2810     if (Str.getAsInteger(10, Val))
2811       return MatchOperand_ParseFail;
2812 
2813     errorExpTgt();
2814     return MatchOperand_Success;
2815   }
2816 
2817   return MatchOperand_NoMatch;
2818 }
2819 
2820 OperandMatchResultTy AMDGPUAsmParser::parseExpTgt(OperandVector &Operands) {
2821   uint8_t Val;
2822   StringRef Str = Parser.getTok().getString();
2823 
2824   auto Res = parseExpTgtImpl(Str, Val);
2825   if (Res != MatchOperand_Success)
2826     return Res;
2827 
2828   SMLoc S = Parser.getTok().getLoc();
2829   Parser.Lex();
2830 
2831   Operands.push_back(AMDGPUOperand::CreateImm(this, Val, S,
2832                                               AMDGPUOperand::ImmTyExpTgt));
2833   return MatchOperand_Success;
2834 }
2835 
2836 OperandMatchResultTy
2837 AMDGPUAsmParser::parseSendMsgOp(OperandVector &Operands) {
2838   using namespace llvm::AMDGPU::SendMsg;
2839 
2840   int64_t Imm16Val = 0;
2841   SMLoc S = Parser.getTok().getLoc();
2842 
2843   switch(getLexer().getKind()) {
2844   default:
2845     return MatchOperand_NoMatch;
2846   case AsmToken::Integer:
2847     // The operand can be an integer value.
2848     if (getParser().parseAbsoluteExpression(Imm16Val))
2849       return MatchOperand_NoMatch;
2850     if (Imm16Val < 0 || !isUInt<16>(Imm16Val)) {
2851       Error(S, "invalid immediate: only 16-bit values are legal");
2852       // Do not return error code, but create an imm operand anyway and proceed
2853       // to the next operand, if any. That avoids unneccessary error messages.
2854     }
2855     break;
2856   case AsmToken::Identifier: {
2857       OperandInfoTy Msg(ID_UNKNOWN_);
2858       OperandInfoTy Operation(OP_UNKNOWN_);
2859       int64_t StreamId = STREAM_ID_DEFAULT_;
2860       if (parseSendMsgConstruct(Msg, Operation, StreamId))
2861         return MatchOperand_ParseFail;
2862       do {
2863         // Validate and encode message ID.
2864         if (! ((ID_INTERRUPT <= Msg.Id && Msg.Id <= ID_GS_DONE)
2865                 || Msg.Id == ID_SYSMSG)) {
2866           if (Msg.IsSymbolic)
2867             Error(S, "invalid/unsupported symbolic name of message");
2868           else
2869             Error(S, "invalid/unsupported code of message");
2870           break;
2871         }
2872         Imm16Val = (Msg.Id << ID_SHIFT_);
2873         // Validate and encode operation ID.
2874         if (Msg.Id == ID_GS || Msg.Id == ID_GS_DONE) {
2875           if (! (OP_GS_FIRST_ <= Operation.Id && Operation.Id < OP_GS_LAST_)) {
2876             if (Operation.IsSymbolic)
2877               Error(S, "invalid symbolic name of GS_OP");
2878             else
2879               Error(S, "invalid code of GS_OP: only 2-bit values are legal");
2880             break;
2881           }
2882           if (Operation.Id == OP_GS_NOP
2883               && Msg.Id != ID_GS_DONE) {
2884             Error(S, "invalid GS_OP: NOP is for GS_DONE only");
2885             break;
2886           }
2887           Imm16Val |= (Operation.Id << OP_SHIFT_);
2888         }
2889         if (Msg.Id == ID_SYSMSG) {
2890           if (! (OP_SYS_FIRST_ <= Operation.Id && Operation.Id < OP_SYS_LAST_)) {
2891             if (Operation.IsSymbolic)
2892               Error(S, "invalid/unsupported symbolic name of SYSMSG_OP");
2893             else
2894               Error(S, "invalid/unsupported code of SYSMSG_OP");
2895             break;
2896           }
2897           Imm16Val |= (Operation.Id << OP_SHIFT_);
2898         }
2899         // Validate and encode stream ID.
2900         if ((Msg.Id == ID_GS || Msg.Id == ID_GS_DONE) && Operation.Id != OP_GS_NOP) {
2901           if (! (STREAM_ID_FIRST_ <= StreamId && StreamId < STREAM_ID_LAST_)) {
2902             Error(S, "invalid stream id: only 2-bit values are legal");
2903             break;
2904           }
2905           Imm16Val |= (StreamId << STREAM_ID_SHIFT_);
2906         }
2907       } while (false);
2908     }
2909     break;
2910   }
2911   Operands.push_back(AMDGPUOperand::CreateImm(this, Imm16Val, S, AMDGPUOperand::ImmTySendMsg));
2912   return MatchOperand_Success;
2913 }
2914 
2915 bool AMDGPUOperand::isSendMsg() const {
2916   return isImmTy(ImmTySendMsg);
2917 }
2918 
2919 //===----------------------------------------------------------------------===//
2920 // sopp branch targets
2921 //===----------------------------------------------------------------------===//
2922 
2923 OperandMatchResultTy
2924 AMDGPUAsmParser::parseSOppBrTarget(OperandVector &Operands) {
2925   SMLoc S = Parser.getTok().getLoc();
2926 
2927   switch (getLexer().getKind()) {
2928     default: return MatchOperand_ParseFail;
2929     case AsmToken::Integer: {
2930       int64_t Imm;
2931       if (getParser().parseAbsoluteExpression(Imm))
2932         return MatchOperand_ParseFail;
2933       Operands.push_back(AMDGPUOperand::CreateImm(this, Imm, S));
2934       return MatchOperand_Success;
2935     }
2936 
2937     case AsmToken::Identifier:
2938       Operands.push_back(AMDGPUOperand::CreateExpr(this,
2939           MCSymbolRefExpr::create(getContext().getOrCreateSymbol(
2940                                   Parser.getTok().getString()), getContext()), S));
2941       Parser.Lex();
2942       return MatchOperand_Success;
2943   }
2944 }
2945 
2946 //===----------------------------------------------------------------------===//
2947 // mubuf
2948 //===----------------------------------------------------------------------===//
2949 
2950 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultGLC() const {
2951   return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTyGLC);
2952 }
2953 
2954 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultSLC() const {
2955   return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTySLC);
2956 }
2957 
2958 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultTFE() const {
2959   return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTyTFE);
2960 }
2961 
2962 void AMDGPUAsmParser::cvtMubufImpl(MCInst &Inst,
2963                                const OperandVector &Operands,
2964                                bool IsAtomic, bool IsAtomicReturn) {
2965   OptionalImmIndexMap OptionalIdx;
2966   assert(IsAtomicReturn ? IsAtomic : true);
2967 
2968   for (unsigned i = 1, e = Operands.size(); i != e; ++i) {
2969     AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[i]);
2970 
2971     // Add the register arguments
2972     if (Op.isReg()) {
2973       Op.addRegOperands(Inst, 1);
2974       continue;
2975     }
2976 
2977     // Handle the case where soffset is an immediate
2978     if (Op.isImm() && Op.getImmTy() == AMDGPUOperand::ImmTyNone) {
2979       Op.addImmOperands(Inst, 1);
2980       continue;
2981     }
2982 
2983     // Handle tokens like 'offen' which are sometimes hard-coded into the
2984     // asm string.  There are no MCInst operands for these.
2985     if (Op.isToken()) {
2986       continue;
2987     }
2988     assert(Op.isImm());
2989 
2990     // Handle optional arguments
2991     OptionalIdx[Op.getImmTy()] = i;
2992   }
2993 
2994   // Copy $vdata_in operand and insert as $vdata for MUBUF_Atomic RTN insns.
2995   if (IsAtomicReturn) {
2996     MCInst::iterator I = Inst.begin(); // $vdata_in is always at the beginning.
2997     Inst.insert(I, *I);
2998   }
2999 
3000   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyOffset);
3001   if (!IsAtomic) { // glc is hard-coded.
3002     addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyGLC);
3003   }
3004   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySLC);
3005   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyTFE);
3006 }
3007 
3008 //===----------------------------------------------------------------------===//
3009 // mimg
3010 //===----------------------------------------------------------------------===//
3011 
3012 void AMDGPUAsmParser::cvtMIMG(MCInst &Inst, const OperandVector &Operands) {
3013   unsigned I = 1;
3014   const MCInstrDesc &Desc = MII.get(Inst.getOpcode());
3015   for (unsigned J = 0; J < Desc.getNumDefs(); ++J) {
3016     ((AMDGPUOperand &)*Operands[I++]).addRegOperands(Inst, 1);
3017   }
3018 
3019   OptionalImmIndexMap OptionalIdx;
3020 
3021   for (unsigned E = Operands.size(); I != E; ++I) {
3022     AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[I]);
3023 
3024     // Add the register arguments
3025     if (Op.isRegOrImm()) {
3026       Op.addRegOrImmOperands(Inst, 1);
3027       continue;
3028     } else if (Op.isImmModifier()) {
3029       OptionalIdx[Op.getImmTy()] = I;
3030     } else {
3031       llvm_unreachable("unexpected operand type");
3032     }
3033   }
3034 
3035   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyDMask);
3036   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyUNorm);
3037   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyGLC);
3038   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyDA);
3039   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyR128);
3040   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyTFE);
3041   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyLWE);
3042   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySLC);
3043 }
3044 
3045 void AMDGPUAsmParser::cvtMIMGAtomic(MCInst &Inst, const OperandVector &Operands) {
3046   unsigned I = 1;
3047   const MCInstrDesc &Desc = MII.get(Inst.getOpcode());
3048   for (unsigned J = 0; J < Desc.getNumDefs(); ++J) {
3049     ((AMDGPUOperand &)*Operands[I++]).addRegOperands(Inst, 1);
3050   }
3051 
3052   // Add src, same as dst
3053   ((AMDGPUOperand &)*Operands[I]).addRegOperands(Inst, 1);
3054 
3055   OptionalImmIndexMap OptionalIdx;
3056 
3057   for (unsigned E = Operands.size(); I != E; ++I) {
3058     AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[I]);
3059 
3060     // Add the register arguments
3061     if (Op.isRegOrImm()) {
3062       Op.addRegOrImmOperands(Inst, 1);
3063       continue;
3064     } else if (Op.isImmModifier()) {
3065       OptionalIdx[Op.getImmTy()] = I;
3066     } else {
3067       llvm_unreachable("unexpected operand type");
3068     }
3069   }
3070 
3071   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyDMask);
3072   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyUNorm);
3073   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyGLC);
3074   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyDA);
3075   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyR128);
3076   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyTFE);
3077   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyLWE);
3078   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySLC);
3079 }
3080 
3081 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultDMask() const {
3082   return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTyDMask);
3083 }
3084 
3085 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultUNorm() const {
3086   return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTyUNorm);
3087 }
3088 
3089 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultDA() const {
3090   return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTyDA);
3091 }
3092 
3093 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultR128() const {
3094   return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTyR128);
3095 }
3096 
3097 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultLWE() const {
3098   return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTyLWE);
3099 }
3100 
3101 //===----------------------------------------------------------------------===//
3102 // smrd
3103 //===----------------------------------------------------------------------===//
3104 
3105 bool AMDGPUOperand::isSMRDOffset8() const {
3106   return isImm() && isUInt<8>(getImm());
3107 }
3108 
3109 bool AMDGPUOperand::isSMRDOffset20() const {
3110   return isImm() && isUInt<20>(getImm());
3111 }
3112 
3113 bool AMDGPUOperand::isSMRDLiteralOffset() const {
3114   // 32-bit literals are only supported on CI and we only want to use them
3115   // when the offset is > 8-bits.
3116   return isImm() && !isUInt<8>(getImm()) && isUInt<32>(getImm());
3117 }
3118 
3119 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultSMRDOffset8() const {
3120   return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTyOffset);
3121 }
3122 
3123 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultSMRDOffset20() const {
3124   return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTyOffset);
3125 }
3126 
3127 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultSMRDLiteralOffset() const {
3128   return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTyOffset);
3129 }
3130 
3131 //===----------------------------------------------------------------------===//
3132 // vop3
3133 //===----------------------------------------------------------------------===//
3134 
3135 static bool ConvertOmodMul(int64_t &Mul) {
3136   if (Mul != 1 && Mul != 2 && Mul != 4)
3137     return false;
3138 
3139   Mul >>= 1;
3140   return true;
3141 }
3142 
3143 static bool ConvertOmodDiv(int64_t &Div) {
3144   if (Div == 1) {
3145     Div = 0;
3146     return true;
3147   }
3148 
3149   if (Div == 2) {
3150     Div = 3;
3151     return true;
3152   }
3153 
3154   return false;
3155 }
3156 
3157 static bool ConvertBoundCtrl(int64_t &BoundCtrl) {
3158   if (BoundCtrl == 0) {
3159     BoundCtrl = 1;
3160     return true;
3161   }
3162 
3163   if (BoundCtrl == -1) {
3164     BoundCtrl = 0;
3165     return true;
3166   }
3167 
3168   return false;
3169 }
3170 
3171 // Note: the order in this table matches the order of operands in AsmString.
3172 static const OptionalOperand AMDGPUOptionalOperandTable[] = {
3173   {"offen",   AMDGPUOperand::ImmTyOffen, true, nullptr},
3174   {"idxen",   AMDGPUOperand::ImmTyIdxen, true, nullptr},
3175   {"addr64",  AMDGPUOperand::ImmTyAddr64, true, nullptr},
3176   {"offset0", AMDGPUOperand::ImmTyOffset0, false, nullptr},
3177   {"offset1", AMDGPUOperand::ImmTyOffset1, false, nullptr},
3178   {"gds",     AMDGPUOperand::ImmTyGDS, true, nullptr},
3179   {"offset",  AMDGPUOperand::ImmTyOffset, false, nullptr},
3180   {"glc",     AMDGPUOperand::ImmTyGLC, true, nullptr},
3181   {"slc",     AMDGPUOperand::ImmTySLC, true, nullptr},
3182   {"tfe",     AMDGPUOperand::ImmTyTFE, true, nullptr},
3183   {"clamp",   AMDGPUOperand::ImmTyClampSI, true, nullptr},
3184   {"omod",    AMDGPUOperand::ImmTyOModSI, false, ConvertOmodMul},
3185   {"unorm",   AMDGPUOperand::ImmTyUNorm, true, nullptr},
3186   {"da",      AMDGPUOperand::ImmTyDA,    true, nullptr},
3187   {"r128",    AMDGPUOperand::ImmTyR128,  true, nullptr},
3188   {"lwe",     AMDGPUOperand::ImmTyLWE,   true, nullptr},
3189   {"dmask",   AMDGPUOperand::ImmTyDMask, false, nullptr},
3190   {"row_mask",   AMDGPUOperand::ImmTyDppRowMask, false, nullptr},
3191   {"bank_mask",  AMDGPUOperand::ImmTyDppBankMask, false, nullptr},
3192   {"bound_ctrl", AMDGPUOperand::ImmTyDppBoundCtrl, false, ConvertBoundCtrl},
3193   {"dst_sel",    AMDGPUOperand::ImmTySdwaDstSel, false, nullptr},
3194   {"src0_sel",   AMDGPUOperand::ImmTySdwaSrc0Sel, false, nullptr},
3195   {"src1_sel",   AMDGPUOperand::ImmTySdwaSrc1Sel, false, nullptr},
3196   {"dst_unused", AMDGPUOperand::ImmTySdwaDstUnused, false, nullptr},
3197   {"vm", AMDGPUOperand::ImmTyExpVM, true, nullptr},
3198 };
3199 
3200 OperandMatchResultTy AMDGPUAsmParser::parseOptionalOperand(OperandVector &Operands) {
3201   OperandMatchResultTy res;
3202   for (const OptionalOperand &Op : AMDGPUOptionalOperandTable) {
3203     // try to parse any optional operand here
3204     if (Op.IsBit) {
3205       res = parseNamedBit(Op.Name, Operands, Op.Type);
3206     } else if (Op.Type == AMDGPUOperand::ImmTyOModSI) {
3207       res = parseOModOperand(Operands);
3208     } else if (Op.Type == AMDGPUOperand::ImmTySdwaDstSel ||
3209                Op.Type == AMDGPUOperand::ImmTySdwaSrc0Sel ||
3210                Op.Type == AMDGPUOperand::ImmTySdwaSrc1Sel) {
3211       res = parseSDWASel(Operands, Op.Name, Op.Type);
3212     } else if (Op.Type == AMDGPUOperand::ImmTySdwaDstUnused) {
3213       res = parseSDWADstUnused(Operands);
3214     } else {
3215       res = parseIntWithPrefix(Op.Name, Operands, Op.Type, Op.ConvertResult);
3216     }
3217     if (res != MatchOperand_NoMatch) {
3218       return res;
3219     }
3220   }
3221   return MatchOperand_NoMatch;
3222 }
3223 
3224 OperandMatchResultTy AMDGPUAsmParser::parseOModOperand(OperandVector &Operands) {
3225   StringRef Name = Parser.getTok().getString();
3226   if (Name == "mul") {
3227     return parseIntWithPrefix("mul", Operands,
3228                               AMDGPUOperand::ImmTyOModSI, ConvertOmodMul);
3229   }
3230 
3231   if (Name == "div") {
3232     return parseIntWithPrefix("div", Operands,
3233                               AMDGPUOperand::ImmTyOModSI, ConvertOmodDiv);
3234   }
3235 
3236   return MatchOperand_NoMatch;
3237 }
3238 
3239 void AMDGPUAsmParser::cvtId(MCInst &Inst, const OperandVector &Operands) {
3240   unsigned I = 1;
3241   const MCInstrDesc &Desc = MII.get(Inst.getOpcode());
3242   for (unsigned J = 0; J < Desc.getNumDefs(); ++J) {
3243     ((AMDGPUOperand &)*Operands[I++]).addRegOperands(Inst, 1);
3244   }
3245   for (unsigned E = Operands.size(); I != E; ++I)
3246     ((AMDGPUOperand &)*Operands[I]).addRegOrImmOperands(Inst, 1);
3247 }
3248 
3249 void AMDGPUAsmParser::cvtVOP3_2_mod(MCInst &Inst, const OperandVector &Operands) {
3250   uint64_t TSFlags = MII.get(Inst.getOpcode()).TSFlags;
3251   if (TSFlags & SIInstrFlags::VOP3) {
3252     cvtVOP3(Inst, Operands);
3253   } else {
3254     cvtId(Inst, Operands);
3255   }
3256 }
3257 
3258 static bool isRegOrImmWithInputMods(const MCInstrDesc &Desc, unsigned OpNum) {
3259       // 1. This operand is input modifiers
3260   return Desc.OpInfo[OpNum].OperandType == AMDGPU::OPERAND_INPUT_MODS
3261       // 2. This is not last operand
3262       && Desc.NumOperands > (OpNum + 1)
3263       // 3. Next operand is register class
3264       && Desc.OpInfo[OpNum + 1].RegClass != -1
3265       // 4. Next register is not tied to any other operand
3266       && Desc.getOperandConstraint(OpNum + 1, MCOI::OperandConstraint::TIED_TO) == -1;
3267 }
3268 
3269 void AMDGPUAsmParser::cvtVOP3(MCInst &Inst, const OperandVector &Operands) {
3270   OptionalImmIndexMap OptionalIdx;
3271   unsigned I = 1;
3272   const MCInstrDesc &Desc = MII.get(Inst.getOpcode());
3273   for (unsigned J = 0; J < Desc.getNumDefs(); ++J) {
3274     ((AMDGPUOperand &)*Operands[I++]).addRegOperands(Inst, 1);
3275   }
3276 
3277   for (unsigned E = Operands.size(); I != E; ++I) {
3278     AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[I]);
3279     if (isRegOrImmWithInputMods(Desc, Inst.getNumOperands())) {
3280       Op.addRegOrImmWithFPInputModsOperands(Inst, 2);
3281     } else if (Op.isImm()) {
3282       OptionalIdx[Op.getImmTy()] = I;
3283     } else {
3284       llvm_unreachable("unhandled operand type");
3285     }
3286   }
3287 
3288   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyClampSI);
3289   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyOModSI);
3290 
3291   // special case v_mac_{f16, f32}:
3292   // it has src2 register operand that is tied to dst operand
3293   // we don't allow modifiers for this operand in assembler so src2_modifiers
3294   // should be 0
3295   if (Inst.getOpcode() == AMDGPU::V_MAC_F32_e64_si ||
3296       Inst.getOpcode() == AMDGPU::V_MAC_F32_e64_vi ||
3297       Inst.getOpcode() == AMDGPU::V_MAC_F16_e64_vi) {
3298     auto it = Inst.begin();
3299     std::advance(
3300       it,
3301       AMDGPU::getNamedOperandIdx(Inst.getOpcode() == AMDGPU::V_MAC_F16_e64_vi ?
3302                                      AMDGPU::V_MAC_F16_e64 :
3303                                      AMDGPU::V_MAC_F32_e64,
3304                                  AMDGPU::OpName::src2_modifiers));
3305     it = Inst.insert(it, MCOperand::createImm(0)); // no modifiers for src2
3306     ++it;
3307     Inst.insert(it, Inst.getOperand(0)); // src2 = dst
3308   }
3309 }
3310 
3311 //===----------------------------------------------------------------------===//
3312 // dpp
3313 //===----------------------------------------------------------------------===//
3314 
3315 bool AMDGPUOperand::isDPPCtrl() const {
3316   bool result = isImm() && getImmTy() == ImmTyDppCtrl && isUInt<9>(getImm());
3317   if (result) {
3318     int64_t Imm = getImm();
3319     return ((Imm >= 0x000) && (Imm <= 0x0ff)) ||
3320            ((Imm >= 0x101) && (Imm <= 0x10f)) ||
3321            ((Imm >= 0x111) && (Imm <= 0x11f)) ||
3322            ((Imm >= 0x121) && (Imm <= 0x12f)) ||
3323            (Imm == 0x130) ||
3324            (Imm == 0x134) ||
3325            (Imm == 0x138) ||
3326            (Imm == 0x13c) ||
3327            (Imm == 0x140) ||
3328            (Imm == 0x141) ||
3329            (Imm == 0x142) ||
3330            (Imm == 0x143);
3331   }
3332   return false;
3333 }
3334 
3335 bool AMDGPUOperand::isGPRIdxMode() const {
3336   return isImm() && isUInt<4>(getImm());
3337 }
3338 
3339 OperandMatchResultTy
3340 AMDGPUAsmParser::parseDPPCtrl(OperandVector &Operands) {
3341   SMLoc S = Parser.getTok().getLoc();
3342   StringRef Prefix;
3343   int64_t Int;
3344 
3345   if (getLexer().getKind() == AsmToken::Identifier) {
3346     Prefix = Parser.getTok().getString();
3347   } else {
3348     return MatchOperand_NoMatch;
3349   }
3350 
3351   if (Prefix == "row_mirror") {
3352     Int = 0x140;
3353     Parser.Lex();
3354   } else if (Prefix == "row_half_mirror") {
3355     Int = 0x141;
3356     Parser.Lex();
3357   } else {
3358     // Check to prevent parseDPPCtrlOps from eating invalid tokens
3359     if (Prefix != "quad_perm"
3360         && Prefix != "row_shl"
3361         && Prefix != "row_shr"
3362         && Prefix != "row_ror"
3363         && Prefix != "wave_shl"
3364         && Prefix != "wave_rol"
3365         && Prefix != "wave_shr"
3366         && Prefix != "wave_ror"
3367         && Prefix != "row_bcast") {
3368       return MatchOperand_NoMatch;
3369     }
3370 
3371     Parser.Lex();
3372     if (getLexer().isNot(AsmToken::Colon))
3373       return MatchOperand_ParseFail;
3374 
3375     if (Prefix == "quad_perm") {
3376       // quad_perm:[%d,%d,%d,%d]
3377       Parser.Lex();
3378       if (getLexer().isNot(AsmToken::LBrac))
3379         return MatchOperand_ParseFail;
3380       Parser.Lex();
3381 
3382       if (getParser().parseAbsoluteExpression(Int) || !(0 <= Int && Int <=3))
3383         return MatchOperand_ParseFail;
3384 
3385       for (int i = 0; i < 3; ++i) {
3386         if (getLexer().isNot(AsmToken::Comma))
3387           return MatchOperand_ParseFail;
3388         Parser.Lex();
3389 
3390         int64_t Temp;
3391         if (getParser().parseAbsoluteExpression(Temp) || !(0 <= Temp && Temp <=3))
3392           return MatchOperand_ParseFail;
3393         const int shift = i*2 + 2;
3394         Int += (Temp << shift);
3395       }
3396 
3397       if (getLexer().isNot(AsmToken::RBrac))
3398         return MatchOperand_ParseFail;
3399       Parser.Lex();
3400 
3401     } else {
3402       // sel:%d
3403       Parser.Lex();
3404       if (getParser().parseAbsoluteExpression(Int))
3405         return MatchOperand_ParseFail;
3406 
3407       if (Prefix == "row_shl" && 1 <= Int && Int <= 15) {
3408         Int |= 0x100;
3409       } else if (Prefix == "row_shr" && 1 <= Int && Int <= 15) {
3410         Int |= 0x110;
3411       } else if (Prefix == "row_ror" && 1 <= Int && Int <= 15) {
3412         Int |= 0x120;
3413       } else if (Prefix == "wave_shl" && 1 == Int) {
3414         Int = 0x130;
3415       } else if (Prefix == "wave_rol" && 1 == Int) {
3416         Int = 0x134;
3417       } else if (Prefix == "wave_shr" && 1 == Int) {
3418         Int = 0x138;
3419       } else if (Prefix == "wave_ror" && 1 == Int) {
3420         Int = 0x13C;
3421       } else if (Prefix == "row_bcast") {
3422         if (Int == 15) {
3423           Int = 0x142;
3424         } else if (Int == 31) {
3425           Int = 0x143;
3426         } else {
3427           return MatchOperand_ParseFail;
3428         }
3429       } else {
3430         return MatchOperand_ParseFail;
3431       }
3432     }
3433   }
3434 
3435   Operands.push_back(AMDGPUOperand::CreateImm(this, Int, S, AMDGPUOperand::ImmTyDppCtrl));
3436   return MatchOperand_Success;
3437 }
3438 
3439 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultRowMask() const {
3440   return AMDGPUOperand::CreateImm(this, 0xf, SMLoc(), AMDGPUOperand::ImmTyDppRowMask);
3441 }
3442 
3443 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultBankMask() const {
3444   return AMDGPUOperand::CreateImm(this, 0xf, SMLoc(), AMDGPUOperand::ImmTyDppBankMask);
3445 }
3446 
3447 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultBoundCtrl() const {
3448   return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTyDppBoundCtrl);
3449 }
3450 
3451 void AMDGPUAsmParser::cvtDPP(MCInst &Inst, const OperandVector &Operands) {
3452   OptionalImmIndexMap OptionalIdx;
3453 
3454   unsigned I = 1;
3455   const MCInstrDesc &Desc = MII.get(Inst.getOpcode());
3456   for (unsigned J = 0; J < Desc.getNumDefs(); ++J) {
3457     ((AMDGPUOperand &)*Operands[I++]).addRegOperands(Inst, 1);
3458   }
3459 
3460   for (unsigned E = Operands.size(); I != E; ++I) {
3461     AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[I]);
3462     // Add the register arguments
3463     if (Op.isReg() && Op.Reg.RegNo == AMDGPU::VCC) {
3464       // VOP2b (v_add_u32, v_sub_u32 ...) dpp use "vcc" token.
3465       // Skip it.
3466       continue;
3467     } if (isRegOrImmWithInputMods(Desc, Inst.getNumOperands())) {
3468       Op.addRegWithFPInputModsOperands(Inst, 2);
3469     } else if (Op.isDPPCtrl()) {
3470       Op.addImmOperands(Inst, 1);
3471     } else if (Op.isImm()) {
3472       // Handle optional arguments
3473       OptionalIdx[Op.getImmTy()] = I;
3474     } else {
3475       llvm_unreachable("Invalid operand type");
3476     }
3477   }
3478 
3479   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyDppRowMask, 0xf);
3480   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyDppBankMask, 0xf);
3481   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyDppBoundCtrl);
3482 
3483   // special case v_mac_{f16, f32}:
3484   // it has src2 register operand that is tied to dst operand
3485   if (Inst.getOpcode() == AMDGPU::V_MAC_F32_dpp ||
3486       Inst.getOpcode() == AMDGPU::V_MAC_F16_dpp) {
3487     auto it = Inst.begin();
3488     std::advance(
3489         it, AMDGPU::getNamedOperandIdx(Inst.getOpcode(), AMDGPU::OpName::src2));
3490     Inst.insert(it, Inst.getOperand(0)); // src2 = dst
3491   }
3492 }
3493 
3494 //===----------------------------------------------------------------------===//
3495 // sdwa
3496 //===----------------------------------------------------------------------===//
3497 
3498 OperandMatchResultTy
3499 AMDGPUAsmParser::parseSDWASel(OperandVector &Operands, StringRef Prefix,
3500                               AMDGPUOperand::ImmTy Type) {
3501   using namespace llvm::AMDGPU::SDWA;
3502 
3503   SMLoc S = Parser.getTok().getLoc();
3504   StringRef Value;
3505   OperandMatchResultTy res;
3506 
3507   res = parseStringWithPrefix(Prefix, Value);
3508   if (res != MatchOperand_Success) {
3509     return res;
3510   }
3511 
3512   int64_t Int;
3513   Int = StringSwitch<int64_t>(Value)
3514         .Case("BYTE_0", SdwaSel::BYTE_0)
3515         .Case("BYTE_1", SdwaSel::BYTE_1)
3516         .Case("BYTE_2", SdwaSel::BYTE_2)
3517         .Case("BYTE_3", SdwaSel::BYTE_3)
3518         .Case("WORD_0", SdwaSel::WORD_0)
3519         .Case("WORD_1", SdwaSel::WORD_1)
3520         .Case("DWORD", SdwaSel::DWORD)
3521         .Default(0xffffffff);
3522   Parser.Lex(); // eat last token
3523 
3524   if (Int == 0xffffffff) {
3525     return MatchOperand_ParseFail;
3526   }
3527 
3528   Operands.push_back(AMDGPUOperand::CreateImm(this, Int, S, Type));
3529   return MatchOperand_Success;
3530 }
3531 
3532 OperandMatchResultTy
3533 AMDGPUAsmParser::parseSDWADstUnused(OperandVector &Operands) {
3534   using namespace llvm::AMDGPU::SDWA;
3535 
3536   SMLoc S = Parser.getTok().getLoc();
3537   StringRef Value;
3538   OperandMatchResultTy res;
3539 
3540   res = parseStringWithPrefix("dst_unused", Value);
3541   if (res != MatchOperand_Success) {
3542     return res;
3543   }
3544 
3545   int64_t Int;
3546   Int = StringSwitch<int64_t>(Value)
3547         .Case("UNUSED_PAD", DstUnused::UNUSED_PAD)
3548         .Case("UNUSED_SEXT", DstUnused::UNUSED_SEXT)
3549         .Case("UNUSED_PRESERVE", DstUnused::UNUSED_PRESERVE)
3550         .Default(0xffffffff);
3551   Parser.Lex(); // eat last token
3552 
3553   if (Int == 0xffffffff) {
3554     return MatchOperand_ParseFail;
3555   }
3556 
3557   Operands.push_back(AMDGPUOperand::CreateImm(this, Int, S, AMDGPUOperand::ImmTySdwaDstUnused));
3558   return MatchOperand_Success;
3559 }
3560 
3561 void AMDGPUAsmParser::cvtSdwaVOP1(MCInst &Inst, const OperandVector &Operands) {
3562   cvtSDWA(Inst, Operands, SIInstrFlags::VOP1);
3563 }
3564 
3565 void AMDGPUAsmParser::cvtSdwaVOP2(MCInst &Inst, const OperandVector &Operands) {
3566   cvtSDWA(Inst, Operands, SIInstrFlags::VOP2);
3567 }
3568 
3569 void AMDGPUAsmParser::cvtSdwaVOPC(MCInst &Inst, const OperandVector &Operands) {
3570   cvtSDWA(Inst, Operands, SIInstrFlags::VOPC);
3571 }
3572 
3573 void AMDGPUAsmParser::cvtSDWA(MCInst &Inst, const OperandVector &Operands,
3574                               uint64_t BasicInstType) {
3575   using namespace llvm::AMDGPU::SDWA;
3576   OptionalImmIndexMap OptionalIdx;
3577 
3578   unsigned I = 1;
3579   const MCInstrDesc &Desc = MII.get(Inst.getOpcode());
3580   for (unsigned J = 0; J < Desc.getNumDefs(); ++J) {
3581     ((AMDGPUOperand &)*Operands[I++]).addRegOperands(Inst, 1);
3582   }
3583 
3584   for (unsigned E = Operands.size(); I != E; ++I) {
3585     AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[I]);
3586     // Add the register arguments
3587     if ((BasicInstType == SIInstrFlags::VOPC ||
3588          BasicInstType == SIInstrFlags::VOP2)&&
3589         Op.isReg() &&
3590         Op.Reg.RegNo == AMDGPU::VCC) {
3591       // VOPC and VOP2b (v_add_u32, v_sub_u32 ...) sdwa use "vcc" token as dst.
3592       // Skip it.
3593       continue;
3594     } else if (isRegOrImmWithInputMods(Desc, Inst.getNumOperands())) {
3595       Op.addRegWithInputModsOperands(Inst, 2);
3596     } else if (Op.isImm()) {
3597       // Handle optional arguments
3598       OptionalIdx[Op.getImmTy()] = I;
3599     } else {
3600       llvm_unreachable("Invalid operand type");
3601     }
3602   }
3603 
3604   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyClampSI, 0);
3605 
3606   if (Inst.getOpcode() != AMDGPU::V_NOP_sdwa_vi) {
3607     // V_NOP_sdwa_vi has no optional sdwa arguments
3608     switch (BasicInstType) {
3609     case SIInstrFlags::VOP1:
3610       addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySdwaDstSel, SdwaSel::DWORD);
3611       addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySdwaDstUnused, DstUnused::UNUSED_PRESERVE);
3612       addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySdwaSrc0Sel, SdwaSel::DWORD);
3613       break;
3614 
3615     case SIInstrFlags::VOP2:
3616       addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySdwaDstSel, SdwaSel::DWORD);
3617       addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySdwaDstUnused, DstUnused::UNUSED_PRESERVE);
3618       addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySdwaSrc0Sel, SdwaSel::DWORD);
3619       addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySdwaSrc1Sel, SdwaSel::DWORD);
3620       break;
3621 
3622     case SIInstrFlags::VOPC:
3623       addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySdwaSrc0Sel, SdwaSel::DWORD);
3624       addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySdwaSrc1Sel, SdwaSel::DWORD);
3625       break;
3626 
3627     default:
3628       llvm_unreachable("Invalid instruction type. Only VOP1, VOP2 and VOPC allowed");
3629     }
3630   }
3631 
3632   // special case v_mac_{f16, f32}:
3633   // it has src2 register operand that is tied to dst operand
3634   if (Inst.getOpcode() == AMDGPU::V_MAC_F32_sdwa_vi ||
3635       Inst.getOpcode() == AMDGPU::V_MAC_F16_sdwa_vi)  {
3636     auto it = Inst.begin();
3637     std::advance(
3638         it, AMDGPU::getNamedOperandIdx(Inst.getOpcode(), AMDGPU::OpName::src2));
3639     Inst.insert(it, Inst.getOperand(0)); // src2 = dst
3640   }
3641 
3642 }
3643 
3644 /// Force static initialization.
3645 extern "C" void LLVMInitializeAMDGPUAsmParser() {
3646   RegisterMCAsmParser<AMDGPUAsmParser> A(getTheAMDGPUTarget());
3647   RegisterMCAsmParser<AMDGPUAsmParser> B(getTheGCNTarget());
3648 }
3649 
3650 #define GET_REGISTER_MATCHER
3651 #define GET_MATCHER_IMPLEMENTATION
3652 #include "AMDGPUGenAsmMatcher.inc"
3653 
3654 // This fuction should be defined after auto-generated include so that we have
3655 // MatchClassKind enum defined
3656 unsigned AMDGPUAsmParser::validateTargetOperandClass(MCParsedAsmOperand &Op,
3657                                                      unsigned Kind) {
3658   // Tokens like "glc" would be parsed as immediate operands in ParseOperand().
3659   // But MatchInstructionImpl() expects to meet token and fails to validate
3660   // operand. This method checks if we are given immediate operand but expect to
3661   // get corresponding token.
3662   AMDGPUOperand &Operand = (AMDGPUOperand&)Op;
3663   switch (Kind) {
3664   case MCK_addr64:
3665     return Operand.isAddr64() ? Match_Success : Match_InvalidOperand;
3666   case MCK_gds:
3667     return Operand.isGDS() ? Match_Success : Match_InvalidOperand;
3668   case MCK_glc:
3669     return Operand.isGLC() ? Match_Success : Match_InvalidOperand;
3670   case MCK_idxen:
3671     return Operand.isIdxen() ? Match_Success : Match_InvalidOperand;
3672   case MCK_offen:
3673     return Operand.isOffen() ? Match_Success : Match_InvalidOperand;
3674   case MCK_SSrcB32:
3675     // When operands have expression values, they will return true for isToken,
3676     // because it is not possible to distinguish between a token and an
3677     // expression at parse time. MatchInstructionImpl() will always try to
3678     // match an operand as a token, when isToken returns true, and when the
3679     // name of the expression is not a valid token, the match will fail,
3680     // so we need to handle it here.
3681     return Operand.isSSrcB32() ? Match_Success : Match_InvalidOperand;
3682   case MCK_SSrcF32:
3683     return Operand.isSSrcF32() ? Match_Success : Match_InvalidOperand;
3684   case MCK_SoppBrTarget:
3685     return Operand.isSoppBrTarget() ? Match_Success : Match_InvalidOperand;
3686   case MCK_VReg32OrOff:
3687     return Operand.isVReg32OrOff() ? Match_Success : Match_InvalidOperand;
3688   case MCK_InterpSlot:
3689     return Operand.isInterpSlot() ? Match_Success : Match_InvalidOperand;
3690   case MCK_Attr:
3691     return Operand.isInterpAttr() ? Match_Success : Match_InvalidOperand;
3692   case MCK_AttrChan:
3693     return Operand.isAttrChan() ? Match_Success : Match_InvalidOperand;
3694   default:
3695     return Match_InvalidOperand;
3696   }
3697 }
3698