1 //===-- AMDGPUAsmParser.cpp - Parse SI asm to MCInst instructions ---------===//
2 //
3 //                     The LLVM Compiler Infrastructure
4 //
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
7 //
8 //===----------------------------------------------------------------------===//
9 
10 #include "AMDKernelCodeT.h"
11 #include "MCTargetDesc/AMDGPUMCTargetDesc.h"
12 #include "MCTargetDesc/AMDGPUTargetStreamer.h"
13 #include "SIDefines.h"
14 #include "Utils/AMDGPUBaseInfo.h"
15 #include "Utils/AMDKernelCodeTUtils.h"
16 #include "Utils/AMDGPUAsmUtils.h"
17 #include "llvm/ADT/APFloat.h"
18 #include "llvm/ADT/APInt.h"
19 #include "llvm/ADT/ArrayRef.h"
20 #include "llvm/ADT/SmallBitVector.h"
21 #include "llvm/ADT/SmallString.h"
22 #include "llvm/ADT/STLExtras.h"
23 #include "llvm/ADT/StringRef.h"
24 #include "llvm/ADT/StringSwitch.h"
25 #include "llvm/ADT/Twine.h"
26 #include "llvm/CodeGen/MachineValueType.h"
27 #include "llvm/MC/MCAsmInfo.h"
28 #include "llvm/MC/MCContext.h"
29 #include "llvm/MC/MCExpr.h"
30 #include "llvm/MC/MCInst.h"
31 #include "llvm/MC/MCInstrDesc.h"
32 #include "llvm/MC/MCInstrInfo.h"
33 #include "llvm/MC/MCParser/MCAsmLexer.h"
34 #include "llvm/MC/MCParser/MCAsmParser.h"
35 #include "llvm/MC/MCParser/MCAsmParserExtension.h"
36 #include "llvm/MC/MCParser/MCParsedAsmOperand.h"
37 #include "llvm/MC/MCParser/MCTargetAsmParser.h"
38 #include "llvm/MC/MCRegisterInfo.h"
39 #include "llvm/MC/MCStreamer.h"
40 #include "llvm/MC/MCSubtargetInfo.h"
41 #include "llvm/MC/MCSymbol.h"
42 #include "llvm/Support/Casting.h"
43 #include "llvm/Support/ELF.h"
44 #include "llvm/Support/ErrorHandling.h"
45 #include "llvm/Support/MathExtras.h"
46 #include "llvm/Support/raw_ostream.h"
47 #include "llvm/Support/SMLoc.h"
48 #include "llvm/Support/TargetRegistry.h"
49 #include <algorithm>
50 #include <cassert>
51 #include <cstdint>
52 #include <cstring>
53 #include <iterator>
54 #include <map>
55 #include <memory>
56 #include <string>
57 
58 using namespace llvm;
59 using namespace llvm::AMDGPU;
60 
61 namespace {
62 
63 class AMDGPUAsmParser;
64 
65 enum RegisterKind { IS_UNKNOWN, IS_VGPR, IS_SGPR, IS_TTMP, IS_SPECIAL };
66 
67 //===----------------------------------------------------------------------===//
68 // Operand
69 //===----------------------------------------------------------------------===//
70 
71 class AMDGPUOperand : public MCParsedAsmOperand {
72   enum KindTy {
73     Token,
74     Immediate,
75     Register,
76     Expression
77   } Kind;
78 
79   SMLoc StartLoc, EndLoc;
80   const AMDGPUAsmParser *AsmParser;
81 
82 public:
83   AMDGPUOperand(enum KindTy Kind_, const AMDGPUAsmParser *AsmParser_)
84     : MCParsedAsmOperand(), Kind(Kind_), AsmParser(AsmParser_) {}
85 
86   typedef std::unique_ptr<AMDGPUOperand> Ptr;
87 
88   struct Modifiers {
89     bool Abs = false;
90     bool Neg = false;
91     bool Sext = false;
92 
93     bool hasFPModifiers() const { return Abs || Neg; }
94     bool hasIntModifiers() const { return Sext; }
95     bool hasModifiers() const { return hasFPModifiers() || hasIntModifiers(); }
96 
97     int64_t getFPModifiersOperand() const {
98       int64_t Operand = 0;
99       Operand |= Abs ? SISrcMods::ABS : 0;
100       Operand |= Neg ? SISrcMods::NEG : 0;
101       return Operand;
102     }
103 
104     int64_t getIntModifiersOperand() const {
105       int64_t Operand = 0;
106       Operand |= Sext ? SISrcMods::SEXT : 0;
107       return Operand;
108     }
109 
110     int64_t getModifiersOperand() const {
111       assert(!(hasFPModifiers() && hasIntModifiers())
112            && "fp and int modifiers should not be used simultaneously");
113       if (hasFPModifiers()) {
114         return getFPModifiersOperand();
115       } else if (hasIntModifiers()) {
116         return getIntModifiersOperand();
117       } else {
118         return 0;
119       }
120     }
121 
122     friend raw_ostream &operator <<(raw_ostream &OS, AMDGPUOperand::Modifiers Mods);
123   };
124 
125   enum ImmTy {
126     ImmTyNone,
127     ImmTyGDS,
128     ImmTyOffen,
129     ImmTyIdxen,
130     ImmTyAddr64,
131     ImmTyOffset,
132     ImmTyOffset0,
133     ImmTyOffset1,
134     ImmTyGLC,
135     ImmTySLC,
136     ImmTyTFE,
137     ImmTyClampSI,
138     ImmTyOModSI,
139     ImmTyDppCtrl,
140     ImmTyDppRowMask,
141     ImmTyDppBankMask,
142     ImmTyDppBoundCtrl,
143     ImmTySdwaDstSel,
144     ImmTySdwaSrc0Sel,
145     ImmTySdwaSrc1Sel,
146     ImmTySdwaDstUnused,
147     ImmTyDMask,
148     ImmTyUNorm,
149     ImmTyDA,
150     ImmTyR128,
151     ImmTyLWE,
152     ImmTyExpTgt,
153     ImmTyExpCompr,
154     ImmTyExpVM,
155     ImmTyHwreg,
156     ImmTyOff,
157     ImmTySendMsg,
158     ImmTyInterpSlot,
159     ImmTyInterpAttr,
160     ImmTyAttrChan
161   };
162 
163   struct TokOp {
164     const char *Data;
165     unsigned Length;
166   };
167 
168   struct ImmOp {
169     int64_t Val;
170     ImmTy Type;
171     bool IsFPImm;
172     Modifiers Mods;
173   };
174 
175   struct RegOp {
176     unsigned RegNo;
177     bool IsForcedVOP3;
178     Modifiers Mods;
179   };
180 
181   union {
182     TokOp Tok;
183     ImmOp Imm;
184     RegOp Reg;
185     const MCExpr *Expr;
186   };
187 
188   bool isToken() const override {
189     if (Kind == Token)
190       return true;
191 
192     if (Kind != Expression || !Expr)
193       return false;
194 
195     // When parsing operands, we can't always tell if something was meant to be
196     // a token, like 'gds', or an expression that references a global variable.
197     // In this case, we assume the string is an expression, and if we need to
198     // interpret is a token, then we treat the symbol name as the token.
199     return isa<MCSymbolRefExpr>(Expr);
200   }
201 
202   bool isImm() const override {
203     return Kind == Immediate;
204   }
205 
206   bool isInlinableImm(MVT type) const;
207   bool isLiteralImm(MVT type) const;
208 
209   bool isRegKind() const {
210     return Kind == Register;
211   }
212 
213   bool isReg() const override {
214     return isRegKind() && !hasModifiers();
215   }
216 
217   bool isRegOrImmWithInputMods(MVT type) const {
218     return isRegKind() || isInlinableImm(type);
219   }
220 
221   bool isRegOrImmWithInt16InputMods() const {
222     return isRegOrImmWithInputMods(MVT::i16);
223   }
224 
225   bool isRegOrImmWithInt32InputMods() const {
226     return isRegOrImmWithInputMods(MVT::i32);
227   }
228 
229   bool isRegOrImmWithInt64InputMods() const {
230     return isRegOrImmWithInputMods(MVT::i64);
231   }
232 
233   bool isRegOrImmWithFP16InputMods() const {
234     return isRegOrImmWithInputMods(MVT::f16);
235   }
236 
237   bool isRegOrImmWithFP32InputMods() const {
238     return isRegOrImmWithInputMods(MVT::f32);
239   }
240 
241   bool isRegOrImmWithFP64InputMods() const {
242     return isRegOrImmWithInputMods(MVT::f64);
243   }
244 
245   bool isVReg() const {
246     return isRegClass(AMDGPU::VGPR_32RegClassID) ||
247            isRegClass(AMDGPU::VReg_64RegClassID) ||
248            isRegClass(AMDGPU::VReg_96RegClassID) ||
249            isRegClass(AMDGPU::VReg_128RegClassID) ||
250            isRegClass(AMDGPU::VReg_256RegClassID) ||
251            isRegClass(AMDGPU::VReg_512RegClassID);
252   }
253 
254   bool isVReg32OrOff() const {
255     return isOff() || isRegClass(AMDGPU::VGPR_32RegClassID);
256   }
257 
258   bool isImmTy(ImmTy ImmT) const {
259     return isImm() && Imm.Type == ImmT;
260   }
261 
262   bool isImmModifier() const {
263     return isImm() && Imm.Type != ImmTyNone;
264   }
265 
266   bool isClampSI() const { return isImmTy(ImmTyClampSI); }
267   bool isOModSI() const { return isImmTy(ImmTyOModSI); }
268   bool isDMask() const { return isImmTy(ImmTyDMask); }
269   bool isUNorm() const { return isImmTy(ImmTyUNorm); }
270   bool isDA() const { return isImmTy(ImmTyDA); }
271   bool isR128() const { return isImmTy(ImmTyUNorm); }
272   bool isLWE() const { return isImmTy(ImmTyLWE); }
273   bool isOff() const { return isImmTy(ImmTyOff); }
274   bool isExpTgt() const { return isImmTy(ImmTyExpTgt); }
275   bool isExpVM() const { return isImmTy(ImmTyExpVM); }
276   bool isExpCompr() const { return isImmTy(ImmTyExpCompr); }
277   bool isOffen() const { return isImmTy(ImmTyOffen); }
278   bool isIdxen() const { return isImmTy(ImmTyIdxen); }
279   bool isAddr64() const { return isImmTy(ImmTyAddr64); }
280   bool isOffset() const { return isImmTy(ImmTyOffset) && isUInt<16>(getImm()); }
281   bool isOffset0() const { return isImmTy(ImmTyOffset0) && isUInt<16>(getImm()); }
282   bool isOffset1() const { return isImmTy(ImmTyOffset1) && isUInt<8>(getImm()); }
283   bool isGDS() const { return isImmTy(ImmTyGDS); }
284   bool isGLC() const { return isImmTy(ImmTyGLC); }
285   bool isSLC() const { return isImmTy(ImmTySLC); }
286   bool isTFE() const { return isImmTy(ImmTyTFE); }
287   bool isBankMask() const { return isImmTy(ImmTyDppBankMask); }
288   bool isRowMask() const { return isImmTy(ImmTyDppRowMask); }
289   bool isBoundCtrl() const { return isImmTy(ImmTyDppBoundCtrl); }
290   bool isSDWADstSel() const { return isImmTy(ImmTySdwaDstSel); }
291   bool isSDWASrc0Sel() const { return isImmTy(ImmTySdwaSrc0Sel); }
292   bool isSDWASrc1Sel() const { return isImmTy(ImmTySdwaSrc1Sel); }
293   bool isSDWADstUnused() const { return isImmTy(ImmTySdwaDstUnused); }
294   bool isInterpSlot() const { return isImmTy(ImmTyInterpSlot); }
295   bool isInterpAttr() const { return isImmTy(ImmTyInterpAttr); }
296   bool isAttrChan() const { return isImmTy(ImmTyAttrChan); }
297 
298   bool isMod() const {
299     return isClampSI() || isOModSI();
300   }
301 
302   bool isRegOrImm() const {
303     return isReg() || isImm();
304   }
305 
306   bool isRegClass(unsigned RCID) const;
307 
308   bool isRegOrInlineNoMods(unsigned RCID, MVT type) const {
309     return (isRegClass(RCID) || isInlinableImm(type)) && !hasModifiers();
310   }
311 
312   bool isSCSrcB16() const {
313     return isRegOrInlineNoMods(AMDGPU::SReg_32RegClassID, MVT::i16);
314   }
315 
316   bool isSCSrcB32() const {
317     return isRegOrInlineNoMods(AMDGPU::SReg_32RegClassID, MVT::i32);
318   }
319 
320   bool isSCSrcB64() const {
321     return isRegOrInlineNoMods(AMDGPU::SReg_64RegClassID, MVT::i64);
322   }
323 
324   bool isSCSrcF16() const {
325     return isRegOrInlineNoMods(AMDGPU::SReg_32RegClassID, MVT::f16);
326   }
327 
328   bool isSCSrcF32() const {
329     return isRegOrInlineNoMods(AMDGPU::SReg_32RegClassID, MVT::f32);
330   }
331 
332   bool isSCSrcF64() const {
333     return isRegOrInlineNoMods(AMDGPU::SReg_64RegClassID, MVT::f64);
334   }
335 
336   bool isSSrcB32() const {
337     return isSCSrcB32() || isLiteralImm(MVT::i32) || isExpr();
338   }
339 
340   bool isSSrcB16() const {
341     return isSCSrcB16() || isLiteralImm(MVT::i16);
342   }
343 
344   bool isSSrcB64() const {
345     // TODO: Find out how SALU supports extension of 32-bit literals to 64 bits.
346     // See isVSrc64().
347     return isSCSrcB64() || isLiteralImm(MVT::i64);
348   }
349 
350   bool isSSrcF32() const {
351     return isSCSrcB32() || isLiteralImm(MVT::f32) || isExpr();
352   }
353 
354   bool isSSrcF64() const {
355     return isSCSrcB64() || isLiteralImm(MVT::f64);
356   }
357 
358   bool isSSrcF16() const {
359     return isSCSrcB16() || isLiteralImm(MVT::f16);
360   }
361 
362   bool isVCSrcB32() const {
363     return isRegOrInlineNoMods(AMDGPU::VS_32RegClassID, MVT::i32);
364   }
365 
366   bool isVCSrcB64() const {
367     return isRegOrInlineNoMods(AMDGPU::VS_64RegClassID, MVT::i64);
368   }
369 
370   bool isVCSrcB16() const {
371     return isRegOrInlineNoMods(AMDGPU::VS_32RegClassID, MVT::i16);
372   }
373 
374   bool isVCSrcF32() const {
375     return isRegOrInlineNoMods(AMDGPU::VS_32RegClassID, MVT::f32);
376   }
377 
378   bool isVCSrcF64() const {
379     return isRegOrInlineNoMods(AMDGPU::VS_64RegClassID, MVT::f64);
380   }
381 
382   bool isVCSrcF16() const {
383     return isRegOrInlineNoMods(AMDGPU::VS_32RegClassID, MVT::f16);
384   }
385 
386   bool isVSrcB32() const {
387     return isVCSrcF32() || isLiteralImm(MVT::i32);
388   }
389 
390   bool isVSrcB64() const {
391     return isVCSrcF64() || isLiteralImm(MVT::i64);
392   }
393 
394   bool isVSrcB16() const {
395     return isVCSrcF16() || isLiteralImm(MVT::i16);
396   }
397 
398   bool isVSrcF32() const {
399     return isVCSrcF32() || isLiteralImm(MVT::f32);
400   }
401 
402   bool isVSrcF64() const {
403     return isVCSrcF64() || isLiteralImm(MVT::f64);
404   }
405 
406   bool isVSrcF16() const {
407     return isVCSrcF16() || isLiteralImm(MVT::f16);
408   }
409 
410   bool isKImmFP32() const {
411     return isLiteralImm(MVT::f32);
412   }
413 
414   bool isKImmFP16() const {
415     return isLiteralImm(MVT::f16);
416   }
417 
418   bool isMem() const override {
419     return false;
420   }
421 
422   bool isExpr() const {
423     return Kind == Expression;
424   }
425 
426   bool isSoppBrTarget() const {
427     return isExpr() || isImm();
428   }
429 
430   bool isSWaitCnt() const;
431   bool isHwreg() const;
432   bool isSendMsg() const;
433   bool isSMRDOffset8() const;
434   bool isSMRDOffset20() const;
435   bool isSMRDLiteralOffset() const;
436   bool isDPPCtrl() const;
437   bool isGPRIdxMode() const;
438 
439   StringRef getExpressionAsToken() const {
440     assert(isExpr());
441     const MCSymbolRefExpr *S = cast<MCSymbolRefExpr>(Expr);
442     return S->getSymbol().getName();
443   }
444 
445   StringRef getToken() const {
446     assert(isToken());
447 
448     if (Kind == Expression)
449       return getExpressionAsToken();
450 
451     return StringRef(Tok.Data, Tok.Length);
452   }
453 
454   int64_t getImm() const {
455     assert(isImm());
456     return Imm.Val;
457   }
458 
459   enum ImmTy getImmTy() const {
460     assert(isImm());
461     return Imm.Type;
462   }
463 
464   unsigned getReg() const override {
465     return Reg.RegNo;
466   }
467 
468   SMLoc getStartLoc() const override {
469     return StartLoc;
470   }
471 
472   SMLoc getEndLoc() const override {
473     return EndLoc;
474   }
475 
476   Modifiers getModifiers() const {
477     assert(isRegKind() || isImmTy(ImmTyNone));
478     return isRegKind() ? Reg.Mods : Imm.Mods;
479   }
480 
481   void setModifiers(Modifiers Mods) {
482     assert(isRegKind() || isImmTy(ImmTyNone));
483     if (isRegKind())
484       Reg.Mods = Mods;
485     else
486       Imm.Mods = Mods;
487   }
488 
489   bool hasModifiers() const {
490     return getModifiers().hasModifiers();
491   }
492 
493   bool hasFPModifiers() const {
494     return getModifiers().hasFPModifiers();
495   }
496 
497   bool hasIntModifiers() const {
498     return getModifiers().hasIntModifiers();
499   }
500 
501   void addImmOperands(MCInst &Inst, unsigned N, bool ApplyModifiers = true) const;
502 
503   void addLiteralImmOperand(MCInst &Inst, int64_t Val) const;
504 
505   template <unsigned Bitwidth>
506   void addKImmFPOperands(MCInst &Inst, unsigned N) const;
507 
508   void addKImmFP16Operands(MCInst &Inst, unsigned N) const {
509     addKImmFPOperands<16>(Inst, N);
510   }
511 
512   void addKImmFP32Operands(MCInst &Inst, unsigned N) const {
513     addKImmFPOperands<32>(Inst, N);
514   }
515 
516   void addRegOperands(MCInst &Inst, unsigned N) const;
517 
518   void addRegOrImmOperands(MCInst &Inst, unsigned N) const {
519     if (isRegKind())
520       addRegOperands(Inst, N);
521     else if (isExpr())
522       Inst.addOperand(MCOperand::createExpr(Expr));
523     else
524       addImmOperands(Inst, N);
525   }
526 
527   void addRegOrImmWithInputModsOperands(MCInst &Inst, unsigned N) const {
528     Modifiers Mods = getModifiers();
529     Inst.addOperand(MCOperand::createImm(Mods.getModifiersOperand()));
530     if (isRegKind()) {
531       addRegOperands(Inst, N);
532     } else {
533       addImmOperands(Inst, N, false);
534     }
535   }
536 
537   void addRegOrImmWithFPInputModsOperands(MCInst &Inst, unsigned N) const {
538     assert(!hasIntModifiers());
539     addRegOrImmWithInputModsOperands(Inst, N);
540   }
541 
542   void addRegOrImmWithIntInputModsOperands(MCInst &Inst, unsigned N) const {
543     assert(!hasFPModifiers());
544     addRegOrImmWithInputModsOperands(Inst, N);
545   }
546 
547   void addRegWithInputModsOperands(MCInst &Inst, unsigned N) const {
548     Modifiers Mods = getModifiers();
549     Inst.addOperand(MCOperand::createImm(Mods.getModifiersOperand()));
550     assert(isRegKind());
551     addRegOperands(Inst, N);
552   }
553 
554   void addRegWithFPInputModsOperands(MCInst &Inst, unsigned N) const {
555     assert(!hasIntModifiers());
556     addRegWithInputModsOperands(Inst, N);
557   }
558 
559   void addRegWithIntInputModsOperands(MCInst &Inst, unsigned N) const {
560     assert(!hasFPModifiers());
561     addRegWithInputModsOperands(Inst, N);
562   }
563 
564   void addSoppBrTargetOperands(MCInst &Inst, unsigned N) const {
565     if (isImm())
566       addImmOperands(Inst, N);
567     else {
568       assert(isExpr());
569       Inst.addOperand(MCOperand::createExpr(Expr));
570     }
571   }
572 
573   static void printImmTy(raw_ostream& OS, ImmTy Type) {
574     switch (Type) {
575     case ImmTyNone: OS << "None"; break;
576     case ImmTyGDS: OS << "GDS"; break;
577     case ImmTyOffen: OS << "Offen"; break;
578     case ImmTyIdxen: OS << "Idxen"; break;
579     case ImmTyAddr64: OS << "Addr64"; break;
580     case ImmTyOffset: OS << "Offset"; break;
581     case ImmTyOffset0: OS << "Offset0"; break;
582     case ImmTyOffset1: OS << "Offset1"; break;
583     case ImmTyGLC: OS << "GLC"; break;
584     case ImmTySLC: OS << "SLC"; break;
585     case ImmTyTFE: OS << "TFE"; break;
586     case ImmTyClampSI: OS << "ClampSI"; break;
587     case ImmTyOModSI: OS << "OModSI"; break;
588     case ImmTyDppCtrl: OS << "DppCtrl"; break;
589     case ImmTyDppRowMask: OS << "DppRowMask"; break;
590     case ImmTyDppBankMask: OS << "DppBankMask"; break;
591     case ImmTyDppBoundCtrl: OS << "DppBoundCtrl"; break;
592     case ImmTySdwaDstSel: OS << "SdwaDstSel"; break;
593     case ImmTySdwaSrc0Sel: OS << "SdwaSrc0Sel"; break;
594     case ImmTySdwaSrc1Sel: OS << "SdwaSrc1Sel"; break;
595     case ImmTySdwaDstUnused: OS << "SdwaDstUnused"; break;
596     case ImmTyDMask: OS << "DMask"; break;
597     case ImmTyUNorm: OS << "UNorm"; break;
598     case ImmTyDA: OS << "DA"; break;
599     case ImmTyR128: OS << "R128"; break;
600     case ImmTyLWE: OS << "LWE"; break;
601     case ImmTyOff: OS << "Off"; break;
602     case ImmTyExpTgt: OS << "ExpTgt"; break;
603     case ImmTyExpCompr: OS << "ExpCompr"; break;
604     case ImmTyExpVM: OS << "ExpVM"; break;
605     case ImmTyHwreg: OS << "Hwreg"; break;
606     case ImmTySendMsg: OS << "SendMsg"; break;
607     case ImmTyInterpSlot: OS << "InterpSlot"; break;
608     case ImmTyInterpAttr: OS << "InterpAttr"; break;
609     case ImmTyAttrChan: OS << "AttrChan"; break;
610     }
611   }
612 
613   void print(raw_ostream &OS) const override {
614     switch (Kind) {
615     case Register:
616       OS << "<register " << getReg() << " mods: " << Reg.Mods << '>';
617       break;
618     case Immediate:
619       OS << '<' << getImm();
620       if (getImmTy() != ImmTyNone) {
621         OS << " type: "; printImmTy(OS, getImmTy());
622       }
623       OS << " mods: " << Imm.Mods << '>';
624       break;
625     case Token:
626       OS << '\'' << getToken() << '\'';
627       break;
628     case Expression:
629       OS << "<expr " << *Expr << '>';
630       break;
631     }
632   }
633 
634   static AMDGPUOperand::Ptr CreateImm(const AMDGPUAsmParser *AsmParser,
635                                       int64_t Val, SMLoc Loc,
636                                       enum ImmTy Type = ImmTyNone,
637                                       bool IsFPImm = false) {
638     auto Op = llvm::make_unique<AMDGPUOperand>(Immediate, AsmParser);
639     Op->Imm.Val = Val;
640     Op->Imm.IsFPImm = IsFPImm;
641     Op->Imm.Type = Type;
642     Op->Imm.Mods = Modifiers();
643     Op->StartLoc = Loc;
644     Op->EndLoc = Loc;
645     return Op;
646   }
647 
648   static AMDGPUOperand::Ptr CreateToken(const AMDGPUAsmParser *AsmParser,
649                                         StringRef Str, SMLoc Loc,
650                                         bool HasExplicitEncodingSize = true) {
651     auto Res = llvm::make_unique<AMDGPUOperand>(Token, AsmParser);
652     Res->Tok.Data = Str.data();
653     Res->Tok.Length = Str.size();
654     Res->StartLoc = Loc;
655     Res->EndLoc = Loc;
656     return Res;
657   }
658 
659   static AMDGPUOperand::Ptr CreateReg(const AMDGPUAsmParser *AsmParser,
660                                       unsigned RegNo, SMLoc S,
661                                       SMLoc E,
662                                       bool ForceVOP3) {
663     auto Op = llvm::make_unique<AMDGPUOperand>(Register, AsmParser);
664     Op->Reg.RegNo = RegNo;
665     Op->Reg.Mods = Modifiers();
666     Op->Reg.IsForcedVOP3 = ForceVOP3;
667     Op->StartLoc = S;
668     Op->EndLoc = E;
669     return Op;
670   }
671 
672   static AMDGPUOperand::Ptr CreateExpr(const AMDGPUAsmParser *AsmParser,
673                                        const class MCExpr *Expr, SMLoc S) {
674     auto Op = llvm::make_unique<AMDGPUOperand>(Expression, AsmParser);
675     Op->Expr = Expr;
676     Op->StartLoc = S;
677     Op->EndLoc = S;
678     return Op;
679   }
680 };
681 
682 raw_ostream &operator <<(raw_ostream &OS, AMDGPUOperand::Modifiers Mods) {
683   OS << "abs:" << Mods.Abs << " neg: " << Mods.Neg << " sext:" << Mods.Sext;
684   return OS;
685 }
686 
687 //===----------------------------------------------------------------------===//
688 // AsmParser
689 //===----------------------------------------------------------------------===//
690 
691 // Holds info related to the current kernel, e.g. count of SGPRs used.
692 // Kernel scope begins at .amdgpu_hsa_kernel directive, ends at next
693 // .amdgpu_hsa_kernel or at EOF.
694 class KernelScopeInfo {
695   int SgprIndexUnusedMin = -1;
696   int VgprIndexUnusedMin = -1;
697   MCContext *Ctx = nullptr;
698 
699   void usesSgprAt(int i) {
700     if (i >= SgprIndexUnusedMin) {
701       SgprIndexUnusedMin = ++i;
702       if (Ctx) {
703         MCSymbol * const Sym = Ctx->getOrCreateSymbol(Twine(".kernel.sgpr_count"));
704         Sym->setVariableValue(MCConstantExpr::create(SgprIndexUnusedMin, *Ctx));
705       }
706     }
707   }
708 
709   void usesVgprAt(int i) {
710     if (i >= VgprIndexUnusedMin) {
711       VgprIndexUnusedMin = ++i;
712       if (Ctx) {
713         MCSymbol * const Sym = Ctx->getOrCreateSymbol(Twine(".kernel.vgpr_count"));
714         Sym->setVariableValue(MCConstantExpr::create(VgprIndexUnusedMin, *Ctx));
715       }
716     }
717   }
718 
719 public:
720   KernelScopeInfo() = default;
721 
722   void initialize(MCContext &Context) {
723     Ctx = &Context;
724     usesSgprAt(SgprIndexUnusedMin = -1);
725     usesVgprAt(VgprIndexUnusedMin = -1);
726   }
727 
728   void usesRegister(RegisterKind RegKind, unsigned DwordRegIndex, unsigned RegWidth) {
729     switch (RegKind) {
730       case IS_SGPR: usesSgprAt(DwordRegIndex + RegWidth - 1); break;
731       case IS_VGPR: usesVgprAt(DwordRegIndex + RegWidth - 1); break;
732       default: break;
733     }
734   }
735 };
736 
737 class AMDGPUAsmParser : public MCTargetAsmParser {
738   const MCInstrInfo &MII;
739   MCAsmParser &Parser;
740 
741   unsigned ForcedEncodingSize = 0;
742   bool ForcedDPP = false;
743   bool ForcedSDWA = false;
744   KernelScopeInfo KernelScope;
745 
746   /// @name Auto-generated Match Functions
747   /// {
748 
749 #define GET_ASSEMBLER_HEADER
750 #include "AMDGPUGenAsmMatcher.inc"
751 
752   /// }
753 
754 private:
755   bool ParseAsAbsoluteExpression(uint32_t &Ret);
756   bool ParseDirectiveMajorMinor(uint32_t &Major, uint32_t &Minor);
757   bool ParseDirectiveHSACodeObjectVersion();
758   bool ParseDirectiveHSACodeObjectISA();
759   bool ParseDirectiveRuntimeMetadata();
760   bool ParseAMDKernelCodeTValue(StringRef ID, amd_kernel_code_t &Header);
761   bool ParseDirectiveAMDKernelCodeT();
762   bool ParseSectionDirectiveHSAText();
763   bool subtargetHasRegister(const MCRegisterInfo &MRI, unsigned RegNo) const;
764   bool ParseDirectiveAMDGPUHsaKernel();
765   bool ParseDirectiveAMDGPUHsaModuleGlobal();
766   bool ParseDirectiveAMDGPUHsaProgramGlobal();
767   bool ParseSectionDirectiveHSADataGlobalAgent();
768   bool ParseSectionDirectiveHSADataGlobalProgram();
769   bool ParseSectionDirectiveHSARodataReadonlyAgent();
770   bool AddNextRegisterToList(unsigned& Reg, unsigned& RegWidth, RegisterKind RegKind, unsigned Reg1, unsigned RegNum);
771   bool ParseAMDGPURegister(RegisterKind& RegKind, unsigned& Reg, unsigned& RegNum, unsigned& RegWidth, unsigned *DwordRegIndex);
772   void cvtMubufImpl(MCInst &Inst, const OperandVector &Operands, bool IsAtomic, bool IsAtomicReturn);
773 
774 public:
775   enum AMDGPUMatchResultTy {
776     Match_PreferE32 = FIRST_TARGET_MATCH_RESULT_TY
777   };
778 
779   AMDGPUAsmParser(const MCSubtargetInfo &STI, MCAsmParser &_Parser,
780                const MCInstrInfo &MII,
781                const MCTargetOptions &Options)
782       : MCTargetAsmParser(Options, STI), MII(MII), Parser(_Parser) {
783     MCAsmParserExtension::Initialize(Parser);
784 
785     if (getSTI().getFeatureBits().none()) {
786       // Set default features.
787       copySTI().ToggleFeature("SOUTHERN_ISLANDS");
788     }
789 
790     setAvailableFeatures(ComputeAvailableFeatures(getSTI().getFeatureBits()));
791 
792     {
793       // TODO: make those pre-defined variables read-only.
794       // Currently there is none suitable machinery in the core llvm-mc for this.
795       // MCSymbol::isRedefinable is intended for another purpose, and
796       // AsmParser::parseDirectiveSet() cannot be specialized for specific target.
797       AMDGPU::IsaVersion Isa = AMDGPU::getIsaVersion(getSTI().getFeatureBits());
798       MCContext &Ctx = getContext();
799       MCSymbol *Sym = Ctx.getOrCreateSymbol(Twine(".option.machine_version_major"));
800       Sym->setVariableValue(MCConstantExpr::create(Isa.Major, Ctx));
801       Sym = Ctx.getOrCreateSymbol(Twine(".option.machine_version_minor"));
802       Sym->setVariableValue(MCConstantExpr::create(Isa.Minor, Ctx));
803       Sym = Ctx.getOrCreateSymbol(Twine(".option.machine_version_stepping"));
804       Sym->setVariableValue(MCConstantExpr::create(Isa.Stepping, Ctx));
805     }
806     KernelScope.initialize(getContext());
807   }
808 
809   bool isSI() const {
810     return AMDGPU::isSI(getSTI());
811   }
812 
813   bool isCI() const {
814     return AMDGPU::isCI(getSTI());
815   }
816 
817   bool isVI() const {
818     return AMDGPU::isVI(getSTI());
819   }
820 
821   bool hasInv2PiInlineImm() const {
822     return getSTI().getFeatureBits()[AMDGPU::FeatureInv2PiInlineImm];
823   }
824 
825   bool hasSGPR102_SGPR103() const {
826     return !isVI();
827   }
828 
829   AMDGPUTargetStreamer &getTargetStreamer() {
830     MCTargetStreamer &TS = *getParser().getStreamer().getTargetStreamer();
831     return static_cast<AMDGPUTargetStreamer &>(TS);
832   }
833 
834   const MCRegisterInfo *getMRI() const {
835     // We need this const_cast because for some reason getContext() is not const
836     // in MCAsmParser.
837     return const_cast<AMDGPUAsmParser*>(this)->getContext().getRegisterInfo();
838   }
839 
840   const MCInstrInfo *getMII() const {
841     return &MII;
842   }
843 
844   void setForcedEncodingSize(unsigned Size) { ForcedEncodingSize = Size; }
845   void setForcedDPP(bool ForceDPP_) { ForcedDPP = ForceDPP_; }
846   void setForcedSDWA(bool ForceSDWA_) { ForcedSDWA = ForceSDWA_; }
847 
848   unsigned getForcedEncodingSize() const { return ForcedEncodingSize; }
849   bool isForcedVOP3() const { return ForcedEncodingSize == 64; }
850   bool isForcedDPP() const { return ForcedDPP; }
851   bool isForcedSDWA() const { return ForcedSDWA; }
852   ArrayRef<unsigned> getMatchedVariants() const;
853 
854   std::unique_ptr<AMDGPUOperand> parseRegister();
855   bool ParseRegister(unsigned &RegNo, SMLoc &StartLoc, SMLoc &EndLoc) override;
856   unsigned checkTargetMatchPredicate(MCInst &Inst) override;
857   unsigned validateTargetOperandClass(MCParsedAsmOperand &Op,
858                                       unsigned Kind) override;
859   bool MatchAndEmitInstruction(SMLoc IDLoc, unsigned &Opcode,
860                                OperandVector &Operands, MCStreamer &Out,
861                                uint64_t &ErrorInfo,
862                                bool MatchingInlineAsm) override;
863   bool ParseDirective(AsmToken DirectiveID) override;
864   OperandMatchResultTy parseOperand(OperandVector &Operands, StringRef Mnemonic);
865   StringRef parseMnemonicSuffix(StringRef Name);
866   bool ParseInstruction(ParseInstructionInfo &Info, StringRef Name,
867                         SMLoc NameLoc, OperandVector &Operands) override;
868   //bool ProcessInstruction(MCInst &Inst);
869 
870   OperandMatchResultTy parseIntWithPrefix(const char *Prefix, int64_t &Int);
871   OperandMatchResultTy
872   parseIntWithPrefix(const char *Prefix, OperandVector &Operands,
873                      enum AMDGPUOperand::ImmTy ImmTy = AMDGPUOperand::ImmTyNone,
874                      bool (*ConvertResult)(int64_t &) = nullptr);
875   OperandMatchResultTy
876   parseNamedBit(const char *Name, OperandVector &Operands,
877                 enum AMDGPUOperand::ImmTy ImmTy = AMDGPUOperand::ImmTyNone);
878   OperandMatchResultTy parseStringWithPrefix(StringRef Prefix,
879                                              StringRef &Value);
880 
881   OperandMatchResultTy parseImm(OperandVector &Operands);
882   OperandMatchResultTy parseReg(OperandVector &Operands);
883   OperandMatchResultTy parseRegOrImm(OperandVector &Operands);
884   OperandMatchResultTy parseRegOrImmWithFPInputMods(OperandVector &Operands, bool AllowImm = true);
885   OperandMatchResultTy parseRegOrImmWithIntInputMods(OperandVector &Operands, bool AllowImm = true);
886   OperandMatchResultTy parseRegWithFPInputMods(OperandVector &Operands);
887   OperandMatchResultTy parseRegWithIntInputMods(OperandVector &Operands);
888   OperandMatchResultTy parseVReg32OrOff(OperandVector &Operands);
889 
890   void cvtDSOffset01(MCInst &Inst, const OperandVector &Operands);
891   void cvtDS(MCInst &Inst, const OperandVector &Operands);
892   void cvtExp(MCInst &Inst, const OperandVector &Operands);
893 
894   bool parseCnt(int64_t &IntVal);
895   OperandMatchResultTy parseSWaitCntOps(OperandVector &Operands);
896   OperandMatchResultTy parseHwreg(OperandVector &Operands);
897 
898 private:
899   struct OperandInfoTy {
900     int64_t Id;
901     bool IsSymbolic;
902     OperandInfoTy(int64_t Id_) : Id(Id_), IsSymbolic(false) { }
903   };
904 
905   bool parseSendMsgConstruct(OperandInfoTy &Msg, OperandInfoTy &Operation, int64_t &StreamId);
906   bool parseHwregConstruct(OperandInfoTy &HwReg, int64_t &Offset, int64_t &Width);
907 
908   void errorExpTgt();
909   OperandMatchResultTy parseExpTgtImpl(StringRef Str, uint8_t &Val);
910 
911 public:
912   OperandMatchResultTy parseOptionalOperand(OperandVector &Operands);
913 
914   OperandMatchResultTy parseExpTgt(OperandVector &Operands);
915   OperandMatchResultTy parseSendMsgOp(OperandVector &Operands);
916   OperandMatchResultTy parseInterpSlot(OperandVector &Operands);
917   OperandMatchResultTy parseInterpAttr(OperandVector &Operands);
918   OperandMatchResultTy parseSOppBrTarget(OperandVector &Operands);
919 
920   void cvtMubuf(MCInst &Inst, const OperandVector &Operands) { cvtMubufImpl(Inst, Operands, false, false); }
921   void cvtMubufAtomic(MCInst &Inst, const OperandVector &Operands) { cvtMubufImpl(Inst, Operands, true, false); }
922   void cvtMubufAtomicReturn(MCInst &Inst, const OperandVector &Operands) { cvtMubufImpl(Inst, Operands, true, true); }
923   AMDGPUOperand::Ptr defaultGLC() const;
924   AMDGPUOperand::Ptr defaultSLC() const;
925   AMDGPUOperand::Ptr defaultTFE() const;
926 
927   AMDGPUOperand::Ptr defaultDMask() const;
928   AMDGPUOperand::Ptr defaultUNorm() const;
929   AMDGPUOperand::Ptr defaultDA() const;
930   AMDGPUOperand::Ptr defaultR128() const;
931   AMDGPUOperand::Ptr defaultLWE() const;
932   AMDGPUOperand::Ptr defaultSMRDOffset8() const;
933   AMDGPUOperand::Ptr defaultSMRDOffset20() const;
934   AMDGPUOperand::Ptr defaultSMRDLiteralOffset() const;
935 
936   OperandMatchResultTy parseOModOperand(OperandVector &Operands);
937 
938   void cvtId(MCInst &Inst, const OperandVector &Operands);
939   void cvtVOP3_2_mod(MCInst &Inst, const OperandVector &Operands);
940   void cvtVOP3(MCInst &Inst, const OperandVector &Operands);
941 
942   void cvtMIMG(MCInst &Inst, const OperandVector &Operands);
943   void cvtMIMGAtomic(MCInst &Inst, const OperandVector &Operands);
944 
945   OperandMatchResultTy parseDPPCtrl(OperandVector &Operands);
946   AMDGPUOperand::Ptr defaultRowMask() const;
947   AMDGPUOperand::Ptr defaultBankMask() const;
948   AMDGPUOperand::Ptr defaultBoundCtrl() const;
949   void cvtDPP(MCInst &Inst, const OperandVector &Operands);
950 
951   OperandMatchResultTy parseSDWASel(OperandVector &Operands, StringRef Prefix,
952                                     AMDGPUOperand::ImmTy Type);
953   OperandMatchResultTy parseSDWADstUnused(OperandVector &Operands);
954   void cvtSdwaVOP1(MCInst &Inst, const OperandVector &Operands);
955   void cvtSdwaVOP2(MCInst &Inst, const OperandVector &Operands);
956   void cvtSdwaVOPC(MCInst &Inst, const OperandVector &Operands);
957   void cvtSDWA(MCInst &Inst, const OperandVector &Operands,
958                uint64_t BasicInstType);
959 };
960 
961 struct OptionalOperand {
962   const char *Name;
963   AMDGPUOperand::ImmTy Type;
964   bool IsBit;
965   bool (*ConvertResult)(int64_t&);
966 };
967 
968 } // end anonymous namespace
969 
970 // May be called with integer type with equivalent bitwidth.
971 static const fltSemantics *getFltSemantics(unsigned Size) {
972   switch (Size) {
973   case 4:
974     return &APFloat::IEEEsingle();
975   case 8:
976     return &APFloat::IEEEdouble();
977   case 2:
978     return &APFloat::IEEEhalf();
979   default:
980     llvm_unreachable("unsupported fp type");
981   }
982 }
983 
984 static const fltSemantics *getFltSemantics(MVT VT) {
985   return getFltSemantics(VT.getSizeInBits() / 8);
986 }
987 
988 //===----------------------------------------------------------------------===//
989 // Operand
990 //===----------------------------------------------------------------------===//
991 
992 static bool canLosslesslyConvertToFPType(APFloat &FPLiteral, MVT VT) {
993   bool Lost;
994 
995   // Convert literal to single precision
996   APFloat::opStatus Status = FPLiteral.convert(*getFltSemantics(VT),
997                                                APFloat::rmNearestTiesToEven,
998                                                &Lost);
999   // We allow precision lost but not overflow or underflow
1000   if (Status != APFloat::opOK &&
1001       Lost &&
1002       ((Status & APFloat::opOverflow)  != 0 ||
1003        (Status & APFloat::opUnderflow) != 0)) {
1004     return false;
1005   }
1006 
1007   return true;
1008 }
1009 
1010 bool AMDGPUOperand::isInlinableImm(MVT type) const {
1011   if (!isImmTy(ImmTyNone)) {
1012     // Only plain immediates are inlinable (e.g. "clamp" attribute is not)
1013     return false;
1014   }
1015   // TODO: We should avoid using host float here. It would be better to
1016   // check the float bit values which is what a few other places do.
1017   // We've had bot failures before due to weird NaN support on mips hosts.
1018 
1019   APInt Literal(64, Imm.Val);
1020 
1021   if (Imm.IsFPImm) { // We got fp literal token
1022     if (type == MVT::f64 || type == MVT::i64) { // Expected 64-bit operand
1023       return AMDGPU::isInlinableLiteral64(Imm.Val,
1024                                           AsmParser->hasInv2PiInlineImm());
1025     }
1026 
1027     APFloat FPLiteral(APFloat::IEEEdouble(), APInt(64, Imm.Val));
1028     if (!canLosslesslyConvertToFPType(FPLiteral, type))
1029       return false;
1030 
1031     if (type.getScalarSizeInBits() == 16) {
1032       return AMDGPU::isInlinableLiteral16(
1033         static_cast<int32_t>(FPLiteral.bitcastToAPInt().getZExtValue()),
1034         AsmParser->hasInv2PiInlineImm());
1035     }
1036 
1037     // Check if single precision literal is inlinable
1038     return AMDGPU::isInlinableLiteral32(
1039       static_cast<int32_t>(FPLiteral.bitcastToAPInt().getZExtValue()),
1040       AsmParser->hasInv2PiInlineImm());
1041   }
1042 
1043   // We got int literal token.
1044   if (type == MVT::f64 || type == MVT::i64) { // Expected 64-bit operand
1045     return AMDGPU::isInlinableLiteral64(Imm.Val,
1046                                         AsmParser->hasInv2PiInlineImm());
1047   }
1048 
1049   if (type.getScalarSizeInBits() == 16) {
1050     return AMDGPU::isInlinableLiteral16(
1051       static_cast<int16_t>(Literal.getLoBits(16).getSExtValue()),
1052       AsmParser->hasInv2PiInlineImm());
1053   }
1054 
1055   return AMDGPU::isInlinableLiteral32(
1056     static_cast<int32_t>(Literal.getLoBits(32).getZExtValue()),
1057     AsmParser->hasInv2PiInlineImm());
1058 }
1059 
1060 bool AMDGPUOperand::isLiteralImm(MVT type) const {
1061   // Check that this imediate can be added as literal
1062   if (!isImmTy(ImmTyNone)) {
1063     return false;
1064   }
1065 
1066   if (!Imm.IsFPImm) {
1067     // We got int literal token.
1068 
1069     unsigned Size = type.getSizeInBits();
1070     if (Size == 64)
1071       Size = 32;
1072 
1073     // FIXME: 64-bit operands can zero extend, sign extend, or pad zeroes for FP
1074     // types.
1075     return isUIntN(Size, Imm.Val) || isIntN(Size, Imm.Val);
1076   }
1077 
1078   // We got fp literal token
1079   if (type == MVT::f64) { // Expected 64-bit fp operand
1080     // We would set low 64-bits of literal to zeroes but we accept this literals
1081     return true;
1082   }
1083 
1084   if (type == MVT::i64) { // Expected 64-bit int operand
1085     // We don't allow fp literals in 64-bit integer instructions. It is
1086     // unclear how we should encode them.
1087     return false;
1088   }
1089 
1090   APFloat FPLiteral(APFloat::IEEEdouble(), APInt(64, Imm.Val));
1091   return canLosslesslyConvertToFPType(FPLiteral, type);
1092 }
1093 
1094 bool AMDGPUOperand::isRegClass(unsigned RCID) const {
1095   return isRegKind() && AsmParser->getMRI()->getRegClass(RCID).contains(getReg());
1096 }
1097 
1098 void AMDGPUOperand::addImmOperands(MCInst &Inst, unsigned N, bool ApplyModifiers) const {
1099   int64_t Val = Imm.Val;
1100   if (isImmTy(ImmTyNone) && ApplyModifiers && Imm.Mods.hasFPModifiers() && Imm.Mods.Neg) {
1101     // Apply modifiers to immediate value. Only negate can get here
1102     if (Imm.IsFPImm) {
1103       APFloat F(BitsToDouble(Val));
1104       F.changeSign();
1105       Val = F.bitcastToAPInt().getZExtValue();
1106     } else {
1107       Val = -Val;
1108     }
1109   }
1110 
1111   if (AMDGPU::isSISrcOperand(AsmParser->getMII()->get(Inst.getOpcode()),
1112                              Inst.getNumOperands())) {
1113     addLiteralImmOperand(Inst, Val);
1114   } else {
1115     Inst.addOperand(MCOperand::createImm(Val));
1116   }
1117 }
1118 
1119 void AMDGPUOperand::addLiteralImmOperand(MCInst &Inst, int64_t Val) const {
1120   const auto& InstDesc = AsmParser->getMII()->get(Inst.getOpcode());
1121   auto OpNum = Inst.getNumOperands();
1122   // Check that this operand accepts literals
1123   assert(AMDGPU::isSISrcOperand(InstDesc, OpNum));
1124 
1125   auto OpSize = AMDGPU::getOperandSize(InstDesc, OpNum); // expected operand size
1126 
1127   if (Imm.IsFPImm) { // We got fp literal token
1128     APInt Literal(64, Val);
1129 
1130     switch (OpSize) {
1131     case 8:
1132       if (AMDGPU::isInlinableLiteral64(Literal.getZExtValue(),
1133                                        AsmParser->hasInv2PiInlineImm())) {
1134         Inst.addOperand(MCOperand::createImm(Literal.getZExtValue()));
1135         return;
1136       }
1137 
1138       // Non-inlineable
1139       if (AMDGPU::isSISrcFPOperand(InstDesc, OpNum)) { // Expected 64-bit fp operand
1140         // For fp operands we check if low 32 bits are zeros
1141         if (Literal.getLoBits(32) != 0) {
1142           const_cast<AMDGPUAsmParser *>(AsmParser)->Warning(Inst.getLoc(),
1143           "Can't encode literal as exact 64-bit floating-point operand. "
1144           "Low 32-bits will be set to zero");
1145         }
1146 
1147         Inst.addOperand(MCOperand::createImm(Literal.lshr(32).getZExtValue()));
1148         return;
1149       }
1150 
1151       // We don't allow fp literals in 64-bit integer instructions. It is
1152       // unclear how we should encode them. This case should be checked earlier
1153       // in predicate methods (isLiteralImm())
1154       llvm_unreachable("fp literal in 64-bit integer instruction.");
1155 
1156     case 4:
1157     case 2: {
1158       bool lost;
1159       APFloat FPLiteral(APFloat::IEEEdouble(), Literal);
1160       // Convert literal to single precision
1161       FPLiteral.convert(*getFltSemantics(OpSize),
1162                         APFloat::rmNearestTiesToEven, &lost);
1163       // We allow precision lost but not overflow or underflow. This should be
1164       // checked earlier in isLiteralImm()
1165       Inst.addOperand(MCOperand::createImm(FPLiteral.bitcastToAPInt().getZExtValue()));
1166       return;
1167     }
1168     default:
1169       llvm_unreachable("invalid operand size");
1170     }
1171 
1172     return;
1173   }
1174 
1175    // We got int literal token.
1176   // Only sign extend inline immediates.
1177   // FIXME: No errors on truncation
1178   switch (OpSize) {
1179   case 4:
1180     if (isInt<32>(Val) &&
1181         AMDGPU::isInlinableLiteral32(static_cast<int32_t>(Val),
1182                                      AsmParser->hasInv2PiInlineImm())) {
1183       Inst.addOperand(MCOperand::createImm(Val));
1184       return;
1185     }
1186 
1187     Inst.addOperand(MCOperand::createImm(Val & 0xffffffff));
1188     return;
1189 
1190   case 8:
1191     if (AMDGPU::isInlinableLiteral64(Val,
1192                                      AsmParser->hasInv2PiInlineImm())) {
1193       Inst.addOperand(MCOperand::createImm(Val));
1194       return;
1195     }
1196 
1197     Inst.addOperand(MCOperand::createImm(Lo_32(Val)));
1198     return;
1199 
1200   case 2:
1201     if (isInt<16>(Val) &&
1202         AMDGPU::isInlinableLiteral16(static_cast<int16_t>(Val),
1203                                      AsmParser->hasInv2PiInlineImm())) {
1204       Inst.addOperand(MCOperand::createImm(Val));
1205       return;
1206     }
1207 
1208     Inst.addOperand(MCOperand::createImm(Val & 0xffff));
1209     return;
1210 
1211   default:
1212     llvm_unreachable("invalid operand size");
1213   }
1214 }
1215 
1216 template <unsigned Bitwidth>
1217 void AMDGPUOperand::addKImmFPOperands(MCInst &Inst, unsigned N) const {
1218   APInt Literal(64, Imm.Val);
1219 
1220   if (!Imm.IsFPImm) {
1221     // We got int literal token.
1222     Inst.addOperand(MCOperand::createImm(Literal.getLoBits(Bitwidth).getZExtValue()));
1223     return;
1224   }
1225 
1226   bool Lost;
1227   APFloat FPLiteral(APFloat::IEEEdouble(), Literal);
1228   FPLiteral.convert(*getFltSemantics(Bitwidth / 8),
1229                     APFloat::rmNearestTiesToEven, &Lost);
1230   Inst.addOperand(MCOperand::createImm(FPLiteral.bitcastToAPInt().getZExtValue()));
1231 }
1232 
1233 void AMDGPUOperand::addRegOperands(MCInst &Inst, unsigned N) const {
1234   Inst.addOperand(MCOperand::createReg(AMDGPU::getMCReg(getReg(), AsmParser->getSTI())));
1235 }
1236 
1237 //===----------------------------------------------------------------------===//
1238 // AsmParser
1239 //===----------------------------------------------------------------------===//
1240 
1241 static int getRegClass(RegisterKind Is, unsigned RegWidth) {
1242   if (Is == IS_VGPR) {
1243     switch (RegWidth) {
1244       default: return -1;
1245       case 1: return AMDGPU::VGPR_32RegClassID;
1246       case 2: return AMDGPU::VReg_64RegClassID;
1247       case 3: return AMDGPU::VReg_96RegClassID;
1248       case 4: return AMDGPU::VReg_128RegClassID;
1249       case 8: return AMDGPU::VReg_256RegClassID;
1250       case 16: return AMDGPU::VReg_512RegClassID;
1251     }
1252   } else if (Is == IS_TTMP) {
1253     switch (RegWidth) {
1254       default: return -1;
1255       case 1: return AMDGPU::TTMP_32RegClassID;
1256       case 2: return AMDGPU::TTMP_64RegClassID;
1257       case 4: return AMDGPU::TTMP_128RegClassID;
1258     }
1259   } else if (Is == IS_SGPR) {
1260     switch (RegWidth) {
1261       default: return -1;
1262       case 1: return AMDGPU::SGPR_32RegClassID;
1263       case 2: return AMDGPU::SGPR_64RegClassID;
1264       case 4: return AMDGPU::SGPR_128RegClassID;
1265       case 8: return AMDGPU::SReg_256RegClassID;
1266       case 16: return AMDGPU::SReg_512RegClassID;
1267     }
1268   }
1269   return -1;
1270 }
1271 
1272 static unsigned getSpecialRegForName(StringRef RegName) {
1273   return StringSwitch<unsigned>(RegName)
1274     .Case("exec", AMDGPU::EXEC)
1275     .Case("vcc", AMDGPU::VCC)
1276     .Case("flat_scratch", AMDGPU::FLAT_SCR)
1277     .Case("m0", AMDGPU::M0)
1278     .Case("scc", AMDGPU::SCC)
1279     .Case("tba", AMDGPU::TBA)
1280     .Case("tma", AMDGPU::TMA)
1281     .Case("flat_scratch_lo", AMDGPU::FLAT_SCR_LO)
1282     .Case("flat_scratch_hi", AMDGPU::FLAT_SCR_HI)
1283     .Case("vcc_lo", AMDGPU::VCC_LO)
1284     .Case("vcc_hi", AMDGPU::VCC_HI)
1285     .Case("exec_lo", AMDGPU::EXEC_LO)
1286     .Case("exec_hi", AMDGPU::EXEC_HI)
1287     .Case("tma_lo", AMDGPU::TMA_LO)
1288     .Case("tma_hi", AMDGPU::TMA_HI)
1289     .Case("tba_lo", AMDGPU::TBA_LO)
1290     .Case("tba_hi", AMDGPU::TBA_HI)
1291     .Default(0);
1292 }
1293 
1294 bool AMDGPUAsmParser::ParseRegister(unsigned &RegNo, SMLoc &StartLoc,
1295                                     SMLoc &EndLoc) {
1296   auto R = parseRegister();
1297   if (!R) return true;
1298   assert(R->isReg());
1299   RegNo = R->getReg();
1300   StartLoc = R->getStartLoc();
1301   EndLoc = R->getEndLoc();
1302   return false;
1303 }
1304 
1305 bool AMDGPUAsmParser::AddNextRegisterToList(unsigned &Reg, unsigned &RegWidth,
1306                                             RegisterKind RegKind, unsigned Reg1,
1307                                             unsigned RegNum) {
1308   switch (RegKind) {
1309   case IS_SPECIAL:
1310     if (Reg == AMDGPU::EXEC_LO && Reg1 == AMDGPU::EXEC_HI) {
1311       Reg = AMDGPU::EXEC;
1312       RegWidth = 2;
1313       return true;
1314     }
1315     if (Reg == AMDGPU::FLAT_SCR_LO && Reg1 == AMDGPU::FLAT_SCR_HI) {
1316       Reg = AMDGPU::FLAT_SCR;
1317       RegWidth = 2;
1318       return true;
1319     }
1320     if (Reg == AMDGPU::VCC_LO && Reg1 == AMDGPU::VCC_HI) {
1321       Reg = AMDGPU::VCC;
1322       RegWidth = 2;
1323       return true;
1324     }
1325     if (Reg == AMDGPU::TBA_LO && Reg1 == AMDGPU::TBA_HI) {
1326       Reg = AMDGPU::TBA;
1327       RegWidth = 2;
1328       return true;
1329     }
1330     if (Reg == AMDGPU::TMA_LO && Reg1 == AMDGPU::TMA_HI) {
1331       Reg = AMDGPU::TMA;
1332       RegWidth = 2;
1333       return true;
1334     }
1335     return false;
1336   case IS_VGPR:
1337   case IS_SGPR:
1338   case IS_TTMP:
1339     if (Reg1 != Reg + RegWidth) {
1340       return false;
1341     }
1342     RegWidth++;
1343     return true;
1344   default:
1345     llvm_unreachable("unexpected register kind");
1346   }
1347 }
1348 
1349 bool AMDGPUAsmParser::ParseAMDGPURegister(RegisterKind &RegKind, unsigned &Reg,
1350                                           unsigned &RegNum, unsigned &RegWidth,
1351                                           unsigned *DwordRegIndex) {
1352   if (DwordRegIndex) { *DwordRegIndex = 0; }
1353   const MCRegisterInfo *TRI = getContext().getRegisterInfo();
1354   if (getLexer().is(AsmToken::Identifier)) {
1355     StringRef RegName = Parser.getTok().getString();
1356     if ((Reg = getSpecialRegForName(RegName))) {
1357       Parser.Lex();
1358       RegKind = IS_SPECIAL;
1359     } else {
1360       unsigned RegNumIndex = 0;
1361       if (RegName[0] == 'v') {
1362         RegNumIndex = 1;
1363         RegKind = IS_VGPR;
1364       } else if (RegName[0] == 's') {
1365         RegNumIndex = 1;
1366         RegKind = IS_SGPR;
1367       } else if (RegName.startswith("ttmp")) {
1368         RegNumIndex = strlen("ttmp");
1369         RegKind = IS_TTMP;
1370       } else {
1371         return false;
1372       }
1373       if (RegName.size() > RegNumIndex) {
1374         // Single 32-bit register: vXX.
1375         if (RegName.substr(RegNumIndex).getAsInteger(10, RegNum))
1376           return false;
1377         Parser.Lex();
1378         RegWidth = 1;
1379       } else {
1380         // Range of registers: v[XX:YY]. ":YY" is optional.
1381         Parser.Lex();
1382         int64_t RegLo, RegHi;
1383         if (getLexer().isNot(AsmToken::LBrac))
1384           return false;
1385         Parser.Lex();
1386 
1387         if (getParser().parseAbsoluteExpression(RegLo))
1388           return false;
1389 
1390         const bool isRBrace = getLexer().is(AsmToken::RBrac);
1391         if (!isRBrace && getLexer().isNot(AsmToken::Colon))
1392           return false;
1393         Parser.Lex();
1394 
1395         if (isRBrace) {
1396           RegHi = RegLo;
1397         } else {
1398           if (getParser().parseAbsoluteExpression(RegHi))
1399             return false;
1400 
1401           if (getLexer().isNot(AsmToken::RBrac))
1402             return false;
1403           Parser.Lex();
1404         }
1405         RegNum = (unsigned) RegLo;
1406         RegWidth = (RegHi - RegLo) + 1;
1407       }
1408     }
1409   } else if (getLexer().is(AsmToken::LBrac)) {
1410     // List of consecutive registers: [s0,s1,s2,s3]
1411     Parser.Lex();
1412     if (!ParseAMDGPURegister(RegKind, Reg, RegNum, RegWidth, nullptr))
1413       return false;
1414     if (RegWidth != 1)
1415       return false;
1416     RegisterKind RegKind1;
1417     unsigned Reg1, RegNum1, RegWidth1;
1418     do {
1419       if (getLexer().is(AsmToken::Comma)) {
1420         Parser.Lex();
1421       } else if (getLexer().is(AsmToken::RBrac)) {
1422         Parser.Lex();
1423         break;
1424       } else if (ParseAMDGPURegister(RegKind1, Reg1, RegNum1, RegWidth1, nullptr)) {
1425         if (RegWidth1 != 1) {
1426           return false;
1427         }
1428         if (RegKind1 != RegKind) {
1429           return false;
1430         }
1431         if (!AddNextRegisterToList(Reg, RegWidth, RegKind1, Reg1, RegNum1)) {
1432           return false;
1433         }
1434       } else {
1435         return false;
1436       }
1437     } while (true);
1438   } else {
1439     return false;
1440   }
1441   switch (RegKind) {
1442   case IS_SPECIAL:
1443     RegNum = 0;
1444     RegWidth = 1;
1445     break;
1446   case IS_VGPR:
1447   case IS_SGPR:
1448   case IS_TTMP:
1449   {
1450     unsigned Size = 1;
1451     if (RegKind == IS_SGPR || RegKind == IS_TTMP) {
1452       // SGPR and TTMP registers must be aligned. Max required alignment is 4 dwords.
1453       Size = std::min(RegWidth, 4u);
1454     }
1455     if (RegNum % Size != 0)
1456       return false;
1457     if (DwordRegIndex) { *DwordRegIndex = RegNum; }
1458     RegNum = RegNum / Size;
1459     int RCID = getRegClass(RegKind, RegWidth);
1460     if (RCID == -1)
1461       return false;
1462     const MCRegisterClass RC = TRI->getRegClass(RCID);
1463     if (RegNum >= RC.getNumRegs())
1464       return false;
1465     Reg = RC.getRegister(RegNum);
1466     break;
1467   }
1468 
1469   default:
1470     llvm_unreachable("unexpected register kind");
1471   }
1472 
1473   if (!subtargetHasRegister(*TRI, Reg))
1474     return false;
1475   return true;
1476 }
1477 
1478 std::unique_ptr<AMDGPUOperand> AMDGPUAsmParser::parseRegister() {
1479   const auto &Tok = Parser.getTok();
1480   SMLoc StartLoc = Tok.getLoc();
1481   SMLoc EndLoc = Tok.getEndLoc();
1482   RegisterKind RegKind;
1483   unsigned Reg, RegNum, RegWidth, DwordRegIndex;
1484 
1485   if (!ParseAMDGPURegister(RegKind, Reg, RegNum, RegWidth, &DwordRegIndex)) {
1486     return nullptr;
1487   }
1488   KernelScope.usesRegister(RegKind, DwordRegIndex, RegWidth);
1489   return AMDGPUOperand::CreateReg(this, Reg, StartLoc, EndLoc, false);
1490 }
1491 
1492 OperandMatchResultTy
1493 AMDGPUAsmParser::parseImm(OperandVector &Operands) {
1494   // TODO: add syntactic sugar for 1/(2*PI)
1495   bool Minus = false;
1496   if (getLexer().getKind() == AsmToken::Minus) {
1497     Minus = true;
1498     Parser.Lex();
1499   }
1500 
1501   SMLoc S = Parser.getTok().getLoc();
1502   switch(getLexer().getKind()) {
1503   case AsmToken::Integer: {
1504     int64_t IntVal;
1505     if (getParser().parseAbsoluteExpression(IntVal))
1506       return MatchOperand_ParseFail;
1507     if (Minus)
1508       IntVal *= -1;
1509     Operands.push_back(AMDGPUOperand::CreateImm(this, IntVal, S));
1510     return MatchOperand_Success;
1511   }
1512   case AsmToken::Real: {
1513     int64_t IntVal;
1514     if (getParser().parseAbsoluteExpression(IntVal))
1515       return MatchOperand_ParseFail;
1516 
1517     APFloat F(BitsToDouble(IntVal));
1518     if (Minus)
1519       F.changeSign();
1520     Operands.push_back(
1521         AMDGPUOperand::CreateImm(this, F.bitcastToAPInt().getZExtValue(), S,
1522                                  AMDGPUOperand::ImmTyNone, true));
1523     return MatchOperand_Success;
1524   }
1525   default:
1526     return Minus ? MatchOperand_ParseFail : MatchOperand_NoMatch;
1527   }
1528 }
1529 
1530 OperandMatchResultTy
1531 AMDGPUAsmParser::parseReg(OperandVector &Operands) {
1532   if (auto R = parseRegister()) {
1533     assert(R->isReg());
1534     R->Reg.IsForcedVOP3 = isForcedVOP3();
1535     Operands.push_back(std::move(R));
1536     return MatchOperand_Success;
1537   }
1538   return MatchOperand_NoMatch;
1539 }
1540 
1541 OperandMatchResultTy
1542 AMDGPUAsmParser::parseRegOrImm(OperandVector &Operands) {
1543   auto res = parseImm(Operands);
1544   if (res != MatchOperand_NoMatch) {
1545     return res;
1546   }
1547 
1548   return parseReg(Operands);
1549 }
1550 
1551 OperandMatchResultTy
1552 AMDGPUAsmParser::parseRegOrImmWithFPInputMods(OperandVector &Operands,
1553                                               bool AllowImm) {
1554   // XXX: During parsing we can't determine if minus sign means
1555   // negate-modifier or negative immediate value.
1556   // By default we suppose it is modifier.
1557   bool Negate = false, Abs = false, Abs2 = false;
1558 
1559   if (getLexer().getKind()== AsmToken::Minus) {
1560     Parser.Lex();
1561     Negate = true;
1562   }
1563 
1564   if (getLexer().getKind() == AsmToken::Identifier &&
1565       Parser.getTok().getString() == "abs") {
1566     Parser.Lex();
1567     Abs2 = true;
1568     if (getLexer().isNot(AsmToken::LParen)) {
1569       Error(Parser.getTok().getLoc(), "expected left paren after abs");
1570       return MatchOperand_ParseFail;
1571     }
1572     Parser.Lex();
1573   }
1574 
1575   if (getLexer().getKind() == AsmToken::Pipe) {
1576     if (Abs2) {
1577       Error(Parser.getTok().getLoc(), "expected register or immediate");
1578       return MatchOperand_ParseFail;
1579     }
1580     Parser.Lex();
1581     Abs = true;
1582   }
1583 
1584   OperandMatchResultTy Res;
1585   if (AllowImm) {
1586     Res = parseRegOrImm(Operands);
1587   } else {
1588     Res = parseReg(Operands);
1589   }
1590   if (Res != MatchOperand_Success) {
1591     return Res;
1592   }
1593 
1594   AMDGPUOperand::Modifiers Mods;
1595   if (Negate) {
1596     Mods.Neg = true;
1597   }
1598   if (Abs) {
1599     if (getLexer().getKind() != AsmToken::Pipe) {
1600       Error(Parser.getTok().getLoc(), "expected vertical bar");
1601       return MatchOperand_ParseFail;
1602     }
1603     Parser.Lex();
1604     Mods.Abs = true;
1605   }
1606   if (Abs2) {
1607     if (getLexer().isNot(AsmToken::RParen)) {
1608       Error(Parser.getTok().getLoc(), "expected closing parentheses");
1609       return MatchOperand_ParseFail;
1610     }
1611     Parser.Lex();
1612     Mods.Abs = true;
1613   }
1614 
1615   if (Mods.hasFPModifiers()) {
1616     AMDGPUOperand &Op = static_cast<AMDGPUOperand &>(*Operands.back());
1617     Op.setModifiers(Mods);
1618   }
1619   return MatchOperand_Success;
1620 }
1621 
1622 OperandMatchResultTy
1623 AMDGPUAsmParser::parseRegOrImmWithIntInputMods(OperandVector &Operands,
1624                                                bool AllowImm) {
1625   bool Sext = false;
1626 
1627   if (getLexer().getKind() == AsmToken::Identifier &&
1628       Parser.getTok().getString() == "sext") {
1629     Parser.Lex();
1630     Sext = true;
1631     if (getLexer().isNot(AsmToken::LParen)) {
1632       Error(Parser.getTok().getLoc(), "expected left paren after sext");
1633       return MatchOperand_ParseFail;
1634     }
1635     Parser.Lex();
1636   }
1637 
1638   OperandMatchResultTy Res;
1639   if (AllowImm) {
1640     Res = parseRegOrImm(Operands);
1641   } else {
1642     Res = parseReg(Operands);
1643   }
1644   if (Res != MatchOperand_Success) {
1645     return Res;
1646   }
1647 
1648   AMDGPUOperand::Modifiers Mods;
1649   if (Sext) {
1650     if (getLexer().isNot(AsmToken::RParen)) {
1651       Error(Parser.getTok().getLoc(), "expected closing parentheses");
1652       return MatchOperand_ParseFail;
1653     }
1654     Parser.Lex();
1655     Mods.Sext = true;
1656   }
1657 
1658   if (Mods.hasIntModifiers()) {
1659     AMDGPUOperand &Op = static_cast<AMDGPUOperand &>(*Operands.back());
1660     Op.setModifiers(Mods);
1661   }
1662 
1663   return MatchOperand_Success;
1664 }
1665 
1666 OperandMatchResultTy
1667 AMDGPUAsmParser::parseRegWithFPInputMods(OperandVector &Operands) {
1668   return parseRegOrImmWithFPInputMods(Operands, false);
1669 }
1670 
1671 OperandMatchResultTy
1672 AMDGPUAsmParser::parseRegWithIntInputMods(OperandVector &Operands) {
1673   return parseRegOrImmWithIntInputMods(Operands, false);
1674 }
1675 
1676 OperandMatchResultTy AMDGPUAsmParser::parseVReg32OrOff(OperandVector &Operands) {
1677   std::unique_ptr<AMDGPUOperand> Reg = parseRegister();
1678   if (Reg) {
1679     Operands.push_back(std::move(Reg));
1680     return MatchOperand_Success;
1681   }
1682 
1683   const AsmToken &Tok = Parser.getTok();
1684   if (Tok.getString() == "off") {
1685     Operands.push_back(AMDGPUOperand::CreateImm(this, 0, Tok.getLoc(),
1686                                                 AMDGPUOperand::ImmTyOff, false));
1687     Parser.Lex();
1688     return MatchOperand_Success;
1689   }
1690 
1691   return MatchOperand_NoMatch;
1692 }
1693 
1694 unsigned AMDGPUAsmParser::checkTargetMatchPredicate(MCInst &Inst) {
1695   uint64_t TSFlags = MII.get(Inst.getOpcode()).TSFlags;
1696 
1697   if ((getForcedEncodingSize() == 32 && (TSFlags & SIInstrFlags::VOP3)) ||
1698       (getForcedEncodingSize() == 64 && !(TSFlags & SIInstrFlags::VOP3)) ||
1699       (isForcedDPP() && !(TSFlags & SIInstrFlags::DPP)) ||
1700       (isForcedSDWA() && !(TSFlags & SIInstrFlags::SDWA)) )
1701     return Match_InvalidOperand;
1702 
1703   if ((TSFlags & SIInstrFlags::VOP3) &&
1704       (TSFlags & SIInstrFlags::VOPAsmPrefer32Bit) &&
1705       getForcedEncodingSize() != 64)
1706     return Match_PreferE32;
1707 
1708   if (Inst.getOpcode() == AMDGPU::V_MAC_F32_sdwa_vi ||
1709       Inst.getOpcode() == AMDGPU::V_MAC_F16_sdwa_vi) {
1710     // v_mac_f32/16 allow only dst_sel == DWORD;
1711     auto OpNum =
1712         AMDGPU::getNamedOperandIdx(Inst.getOpcode(), AMDGPU::OpName::dst_sel);
1713     const auto &Op = Inst.getOperand(OpNum);
1714     if (!Op.isImm() || Op.getImm() != AMDGPU::SDWA::SdwaSel::DWORD) {
1715       return Match_InvalidOperand;
1716     }
1717   }
1718 
1719   return Match_Success;
1720 }
1721 
1722 // What asm variants we should check
1723 ArrayRef<unsigned> AMDGPUAsmParser::getMatchedVariants() const {
1724   if (getForcedEncodingSize() == 32) {
1725     static const unsigned Variants[] = {AMDGPUAsmVariants::DEFAULT};
1726     return makeArrayRef(Variants);
1727   }
1728 
1729   if (isForcedVOP3()) {
1730     static const unsigned Variants[] = {AMDGPUAsmVariants::VOP3};
1731     return makeArrayRef(Variants);
1732   }
1733 
1734   if (isForcedSDWA()) {
1735     static const unsigned Variants[] = {AMDGPUAsmVariants::SDWA};
1736     return makeArrayRef(Variants);
1737   }
1738 
1739   if (isForcedDPP()) {
1740     static const unsigned Variants[] = {AMDGPUAsmVariants::DPP};
1741     return makeArrayRef(Variants);
1742   }
1743 
1744   static const unsigned Variants[] = {
1745     AMDGPUAsmVariants::DEFAULT, AMDGPUAsmVariants::VOP3,
1746     AMDGPUAsmVariants::SDWA, AMDGPUAsmVariants::DPP
1747   };
1748 
1749   return makeArrayRef(Variants);
1750 }
1751 
1752 bool AMDGPUAsmParser::MatchAndEmitInstruction(SMLoc IDLoc, unsigned &Opcode,
1753                                               OperandVector &Operands,
1754                                               MCStreamer &Out,
1755                                               uint64_t &ErrorInfo,
1756                                               bool MatchingInlineAsm) {
1757   MCInst Inst;
1758   unsigned Result = Match_Success;
1759   for (auto Variant : getMatchedVariants()) {
1760     uint64_t EI;
1761     auto R = MatchInstructionImpl(Operands, Inst, EI, MatchingInlineAsm,
1762                                   Variant);
1763     // We order match statuses from least to most specific. We use most specific
1764     // status as resulting
1765     // Match_MnemonicFail < Match_InvalidOperand < Match_MissingFeature < Match_PreferE32
1766     if ((R == Match_Success) ||
1767         (R == Match_PreferE32) ||
1768         (R == Match_MissingFeature && Result != Match_PreferE32) ||
1769         (R == Match_InvalidOperand && Result != Match_MissingFeature
1770                                    && Result != Match_PreferE32) ||
1771         (R == Match_MnemonicFail   && Result != Match_InvalidOperand
1772                                    && Result != Match_MissingFeature
1773                                    && Result != Match_PreferE32)) {
1774       Result = R;
1775       ErrorInfo = EI;
1776     }
1777     if (R == Match_Success)
1778       break;
1779   }
1780 
1781   switch (Result) {
1782   default: break;
1783   case Match_Success:
1784     Inst.setLoc(IDLoc);
1785     Out.EmitInstruction(Inst, getSTI());
1786     return false;
1787 
1788   case Match_MissingFeature:
1789     return Error(IDLoc, "instruction not supported on this GPU");
1790 
1791   case Match_MnemonicFail:
1792     return Error(IDLoc, "unrecognized instruction mnemonic");
1793 
1794   case Match_InvalidOperand: {
1795     SMLoc ErrorLoc = IDLoc;
1796     if (ErrorInfo != ~0ULL) {
1797       if (ErrorInfo >= Operands.size()) {
1798         return Error(IDLoc, "too few operands for instruction");
1799       }
1800       ErrorLoc = ((AMDGPUOperand &)*Operands[ErrorInfo]).getStartLoc();
1801       if (ErrorLoc == SMLoc())
1802         ErrorLoc = IDLoc;
1803     }
1804     return Error(ErrorLoc, "invalid operand for instruction");
1805   }
1806 
1807   case Match_PreferE32:
1808     return Error(IDLoc, "internal error: instruction without _e64 suffix "
1809                         "should be encoded as e32");
1810   }
1811   llvm_unreachable("Implement any new match types added!");
1812 }
1813 
1814 bool AMDGPUAsmParser::ParseAsAbsoluteExpression(uint32_t &Ret) {
1815   int64_t Tmp = -1;
1816   if (getLexer().isNot(AsmToken::Integer) && getLexer().isNot(AsmToken::Identifier)) {
1817     return true;
1818   }
1819   if (getParser().parseAbsoluteExpression(Tmp)) {
1820     return true;
1821   }
1822   Ret = static_cast<uint32_t>(Tmp);
1823   return false;
1824 }
1825 
1826 bool AMDGPUAsmParser::ParseDirectiveMajorMinor(uint32_t &Major,
1827                                                uint32_t &Minor) {
1828   if (ParseAsAbsoluteExpression(Major))
1829     return TokError("invalid major version");
1830 
1831   if (getLexer().isNot(AsmToken::Comma))
1832     return TokError("minor version number required, comma expected");
1833   Lex();
1834 
1835   if (ParseAsAbsoluteExpression(Minor))
1836     return TokError("invalid minor version");
1837 
1838   return false;
1839 }
1840 
1841 bool AMDGPUAsmParser::ParseDirectiveHSACodeObjectVersion() {
1842   uint32_t Major;
1843   uint32_t Minor;
1844 
1845   if (ParseDirectiveMajorMinor(Major, Minor))
1846     return true;
1847 
1848   getTargetStreamer().EmitDirectiveHSACodeObjectVersion(Major, Minor);
1849   return false;
1850 }
1851 
1852 bool AMDGPUAsmParser::ParseDirectiveHSACodeObjectISA() {
1853   uint32_t Major;
1854   uint32_t Minor;
1855   uint32_t Stepping;
1856   StringRef VendorName;
1857   StringRef ArchName;
1858 
1859   // If this directive has no arguments, then use the ISA version for the
1860   // targeted GPU.
1861   if (getLexer().is(AsmToken::EndOfStatement)) {
1862     AMDGPU::IsaVersion Isa = AMDGPU::getIsaVersion(getSTI().getFeatureBits());
1863     getTargetStreamer().EmitDirectiveHSACodeObjectISA(Isa.Major, Isa.Minor,
1864                                                       Isa.Stepping,
1865                                                       "AMD", "AMDGPU");
1866     return false;
1867   }
1868 
1869   if (ParseDirectiveMajorMinor(Major, Minor))
1870     return true;
1871 
1872   if (getLexer().isNot(AsmToken::Comma))
1873     return TokError("stepping version number required, comma expected");
1874   Lex();
1875 
1876   if (ParseAsAbsoluteExpression(Stepping))
1877     return TokError("invalid stepping version");
1878 
1879   if (getLexer().isNot(AsmToken::Comma))
1880     return TokError("vendor name required, comma expected");
1881   Lex();
1882 
1883   if (getLexer().isNot(AsmToken::String))
1884     return TokError("invalid vendor name");
1885 
1886   VendorName = getLexer().getTok().getStringContents();
1887   Lex();
1888 
1889   if (getLexer().isNot(AsmToken::Comma))
1890     return TokError("arch name required, comma expected");
1891   Lex();
1892 
1893   if (getLexer().isNot(AsmToken::String))
1894     return TokError("invalid arch name");
1895 
1896   ArchName = getLexer().getTok().getStringContents();
1897   Lex();
1898 
1899   getTargetStreamer().EmitDirectiveHSACodeObjectISA(Major, Minor, Stepping,
1900                                                     VendorName, ArchName);
1901   return false;
1902 }
1903 
1904 bool AMDGPUAsmParser::ParseDirectiveRuntimeMetadata() {
1905   std::string Metadata;
1906   raw_string_ostream MS(Metadata);
1907 
1908   getLexer().setSkipSpace(false);
1909 
1910   bool FoundEnd = false;
1911   while (!getLexer().is(AsmToken::Eof)) {
1912     while (getLexer().is(AsmToken::Space)) {
1913       MS << ' ';
1914       Lex();
1915     }
1916 
1917     if (getLexer().is(AsmToken::Identifier)) {
1918       StringRef ID = getLexer().getTok().getIdentifier();
1919       if (ID == ".end_amdgpu_runtime_metadata") {
1920         Lex();
1921         FoundEnd = true;
1922         break;
1923       }
1924     }
1925 
1926     MS << Parser.parseStringToEndOfStatement()
1927        << getContext().getAsmInfo()->getSeparatorString();
1928 
1929     Parser.eatToEndOfStatement();
1930   }
1931 
1932   getLexer().setSkipSpace(true);
1933 
1934   if (getLexer().is(AsmToken::Eof) && !FoundEnd)
1935     return TokError("expected directive .end_amdgpu_runtime_metadata not found");
1936 
1937   MS.flush();
1938 
1939   getTargetStreamer().EmitRuntimeMetadata(Metadata);
1940 
1941   return false;
1942 }
1943 
1944 bool AMDGPUAsmParser::ParseAMDKernelCodeTValue(StringRef ID,
1945                                                amd_kernel_code_t &Header) {
1946   SmallString<40> ErrStr;
1947   raw_svector_ostream Err(ErrStr);
1948   if (!parseAmdKernelCodeField(ID, getParser(), Header, Err)) {
1949     return TokError(Err.str());
1950   }
1951   Lex();
1952   return false;
1953 }
1954 
1955 bool AMDGPUAsmParser::ParseDirectiveAMDKernelCodeT() {
1956   amd_kernel_code_t Header;
1957   AMDGPU::initDefaultAMDKernelCodeT(Header, getSTI().getFeatureBits());
1958 
1959   while (true) {
1960     // Lex EndOfStatement.  This is in a while loop, because lexing a comment
1961     // will set the current token to EndOfStatement.
1962     while(getLexer().is(AsmToken::EndOfStatement))
1963       Lex();
1964 
1965     if (getLexer().isNot(AsmToken::Identifier))
1966       return TokError("expected value identifier or .end_amd_kernel_code_t");
1967 
1968     StringRef ID = getLexer().getTok().getIdentifier();
1969     Lex();
1970 
1971     if (ID == ".end_amd_kernel_code_t")
1972       break;
1973 
1974     if (ParseAMDKernelCodeTValue(ID, Header))
1975       return true;
1976   }
1977 
1978   getTargetStreamer().EmitAMDKernelCodeT(Header);
1979 
1980   return false;
1981 }
1982 
1983 bool AMDGPUAsmParser::ParseSectionDirectiveHSAText() {
1984   getParser().getStreamer().SwitchSection(
1985       AMDGPU::getHSATextSection(getContext()));
1986   return false;
1987 }
1988 
1989 bool AMDGPUAsmParser::ParseDirectiveAMDGPUHsaKernel() {
1990   if (getLexer().isNot(AsmToken::Identifier))
1991     return TokError("expected symbol name");
1992 
1993   StringRef KernelName = Parser.getTok().getString();
1994 
1995   getTargetStreamer().EmitAMDGPUSymbolType(KernelName,
1996                                            ELF::STT_AMDGPU_HSA_KERNEL);
1997   Lex();
1998   KernelScope.initialize(getContext());
1999   return false;
2000 }
2001 
2002 bool AMDGPUAsmParser::ParseDirectiveAMDGPUHsaModuleGlobal() {
2003   if (getLexer().isNot(AsmToken::Identifier))
2004     return TokError("expected symbol name");
2005 
2006   StringRef GlobalName = Parser.getTok().getIdentifier();
2007 
2008   getTargetStreamer().EmitAMDGPUHsaModuleScopeGlobal(GlobalName);
2009   Lex();
2010   return false;
2011 }
2012 
2013 bool AMDGPUAsmParser::ParseDirectiveAMDGPUHsaProgramGlobal() {
2014   if (getLexer().isNot(AsmToken::Identifier))
2015     return TokError("expected symbol name");
2016 
2017   StringRef GlobalName = Parser.getTok().getIdentifier();
2018 
2019   getTargetStreamer().EmitAMDGPUHsaProgramScopeGlobal(GlobalName);
2020   Lex();
2021   return false;
2022 }
2023 
2024 bool AMDGPUAsmParser::ParseSectionDirectiveHSADataGlobalAgent() {
2025   getParser().getStreamer().SwitchSection(
2026       AMDGPU::getHSADataGlobalAgentSection(getContext()));
2027   return false;
2028 }
2029 
2030 bool AMDGPUAsmParser::ParseSectionDirectiveHSADataGlobalProgram() {
2031   getParser().getStreamer().SwitchSection(
2032       AMDGPU::getHSADataGlobalProgramSection(getContext()));
2033   return false;
2034 }
2035 
2036 bool AMDGPUAsmParser::ParseSectionDirectiveHSARodataReadonlyAgent() {
2037   getParser().getStreamer().SwitchSection(
2038       AMDGPU::getHSARodataReadonlyAgentSection(getContext()));
2039   return false;
2040 }
2041 
2042 bool AMDGPUAsmParser::ParseDirective(AsmToken DirectiveID) {
2043   StringRef IDVal = DirectiveID.getString();
2044 
2045   if (IDVal == ".hsa_code_object_version")
2046     return ParseDirectiveHSACodeObjectVersion();
2047 
2048   if (IDVal == ".hsa_code_object_isa")
2049     return ParseDirectiveHSACodeObjectISA();
2050 
2051   if (IDVal == ".amdgpu_runtime_metadata")
2052     return ParseDirectiveRuntimeMetadata();
2053 
2054   if (IDVal == ".amd_kernel_code_t")
2055     return ParseDirectiveAMDKernelCodeT();
2056 
2057   if (IDVal == ".hsatext")
2058     return ParseSectionDirectiveHSAText();
2059 
2060   if (IDVal == ".amdgpu_hsa_kernel")
2061     return ParseDirectiveAMDGPUHsaKernel();
2062 
2063   if (IDVal == ".amdgpu_hsa_module_global")
2064     return ParseDirectiveAMDGPUHsaModuleGlobal();
2065 
2066   if (IDVal == ".amdgpu_hsa_program_global")
2067     return ParseDirectiveAMDGPUHsaProgramGlobal();
2068 
2069   if (IDVal == ".hsadata_global_agent")
2070     return ParseSectionDirectiveHSADataGlobalAgent();
2071 
2072   if (IDVal == ".hsadata_global_program")
2073     return ParseSectionDirectiveHSADataGlobalProgram();
2074 
2075   if (IDVal == ".hsarodata_readonly_agent")
2076     return ParseSectionDirectiveHSARodataReadonlyAgent();
2077 
2078   return true;
2079 }
2080 
2081 bool AMDGPUAsmParser::subtargetHasRegister(const MCRegisterInfo &MRI,
2082                                            unsigned RegNo) const {
2083   if (isCI())
2084     return true;
2085 
2086   if (isSI()) {
2087     // No flat_scr
2088     switch (RegNo) {
2089     case AMDGPU::FLAT_SCR:
2090     case AMDGPU::FLAT_SCR_LO:
2091     case AMDGPU::FLAT_SCR_HI:
2092       return false;
2093     default:
2094       return true;
2095     }
2096   }
2097 
2098   // VI only has 102 SGPRs, so make sure we aren't trying to use the 2 more that
2099   // SI/CI have.
2100   for (MCRegAliasIterator R(AMDGPU::SGPR102_SGPR103, &MRI, true);
2101        R.isValid(); ++R) {
2102     if (*R == RegNo)
2103       return false;
2104   }
2105 
2106   return true;
2107 }
2108 
2109 OperandMatchResultTy
2110 AMDGPUAsmParser::parseOperand(OperandVector &Operands, StringRef Mnemonic) {
2111   // Try to parse with a custom parser
2112   OperandMatchResultTy ResTy = MatchOperandParserImpl(Operands, Mnemonic);
2113 
2114   // If we successfully parsed the operand or if there as an error parsing,
2115   // we are done.
2116   //
2117   // If we are parsing after we reach EndOfStatement then this means we
2118   // are appending default values to the Operands list.  This is only done
2119   // by custom parser, so we shouldn't continue on to the generic parsing.
2120   if (ResTy == MatchOperand_Success || ResTy == MatchOperand_ParseFail ||
2121       getLexer().is(AsmToken::EndOfStatement))
2122     return ResTy;
2123 
2124   ResTy = parseRegOrImm(Operands);
2125 
2126   if (ResTy == MatchOperand_Success)
2127     return ResTy;
2128 
2129   if (getLexer().getKind() == AsmToken::Identifier) {
2130     // If this identifier is a symbol, we want to create an expression for it.
2131     // It is a little difficult to distinguish between a symbol name, and
2132     // an instruction flag like 'gds'.  In order to do this, we parse
2133     // all tokens as expressions and then treate the symbol name as the token
2134     // string when we want to interpret the operand as a token.
2135     const auto &Tok = Parser.getTok();
2136     SMLoc S = Tok.getLoc();
2137     const MCExpr *Expr = nullptr;
2138     if (!Parser.parseExpression(Expr)) {
2139       Operands.push_back(AMDGPUOperand::CreateExpr(this, Expr, S));
2140       return MatchOperand_Success;
2141     }
2142 
2143     Operands.push_back(AMDGPUOperand::CreateToken(this, Tok.getString(), Tok.getLoc()));
2144     Parser.Lex();
2145     return MatchOperand_Success;
2146   }
2147   return MatchOperand_NoMatch;
2148 }
2149 
2150 StringRef AMDGPUAsmParser::parseMnemonicSuffix(StringRef Name) {
2151   // Clear any forced encodings from the previous instruction.
2152   setForcedEncodingSize(0);
2153   setForcedDPP(false);
2154   setForcedSDWA(false);
2155 
2156   if (Name.endswith("_e64")) {
2157     setForcedEncodingSize(64);
2158     return Name.substr(0, Name.size() - 4);
2159   } else if (Name.endswith("_e32")) {
2160     setForcedEncodingSize(32);
2161     return Name.substr(0, Name.size() - 4);
2162   } else if (Name.endswith("_dpp")) {
2163     setForcedDPP(true);
2164     return Name.substr(0, Name.size() - 4);
2165   } else if (Name.endswith("_sdwa")) {
2166     setForcedSDWA(true);
2167     return Name.substr(0, Name.size() - 5);
2168   }
2169   return Name;
2170 }
2171 
2172 bool AMDGPUAsmParser::ParseInstruction(ParseInstructionInfo &Info,
2173                                        StringRef Name,
2174                                        SMLoc NameLoc, OperandVector &Operands) {
2175   // Add the instruction mnemonic
2176   Name = parseMnemonicSuffix(Name);
2177   Operands.push_back(AMDGPUOperand::CreateToken(this, Name, NameLoc));
2178 
2179   while (!getLexer().is(AsmToken::EndOfStatement)) {
2180     OperandMatchResultTy Res = parseOperand(Operands, Name);
2181 
2182     // Eat the comma or space if there is one.
2183     if (getLexer().is(AsmToken::Comma))
2184       Parser.Lex();
2185 
2186     switch (Res) {
2187       case MatchOperand_Success: break;
2188       case MatchOperand_ParseFail:
2189         Error(getLexer().getLoc(), "failed parsing operand.");
2190         while (!getLexer().is(AsmToken::EndOfStatement)) {
2191           Parser.Lex();
2192         }
2193         return true;
2194       case MatchOperand_NoMatch:
2195         Error(getLexer().getLoc(), "not a valid operand.");
2196         while (!getLexer().is(AsmToken::EndOfStatement)) {
2197           Parser.Lex();
2198         }
2199         return true;
2200     }
2201   }
2202 
2203   return false;
2204 }
2205 
2206 //===----------------------------------------------------------------------===//
2207 // Utility functions
2208 //===----------------------------------------------------------------------===//
2209 
2210 OperandMatchResultTy
2211 AMDGPUAsmParser::parseIntWithPrefix(const char *Prefix, int64_t &Int) {
2212   switch(getLexer().getKind()) {
2213     default: return MatchOperand_NoMatch;
2214     case AsmToken::Identifier: {
2215       StringRef Name = Parser.getTok().getString();
2216       if (!Name.equals(Prefix)) {
2217         return MatchOperand_NoMatch;
2218       }
2219 
2220       Parser.Lex();
2221       if (getLexer().isNot(AsmToken::Colon))
2222         return MatchOperand_ParseFail;
2223 
2224       Parser.Lex();
2225       if (getLexer().isNot(AsmToken::Integer))
2226         return MatchOperand_ParseFail;
2227 
2228       if (getParser().parseAbsoluteExpression(Int))
2229         return MatchOperand_ParseFail;
2230       break;
2231     }
2232   }
2233   return MatchOperand_Success;
2234 }
2235 
2236 OperandMatchResultTy
2237 AMDGPUAsmParser::parseIntWithPrefix(const char *Prefix, OperandVector &Operands,
2238                                     enum AMDGPUOperand::ImmTy ImmTy,
2239                                     bool (*ConvertResult)(int64_t&)) {
2240   SMLoc S = Parser.getTok().getLoc();
2241   int64_t Value = 0;
2242 
2243   OperandMatchResultTy Res = parseIntWithPrefix(Prefix, Value);
2244   if (Res != MatchOperand_Success)
2245     return Res;
2246 
2247   if (ConvertResult && !ConvertResult(Value)) {
2248     return MatchOperand_ParseFail;
2249   }
2250 
2251   Operands.push_back(AMDGPUOperand::CreateImm(this, Value, S, ImmTy));
2252   return MatchOperand_Success;
2253 }
2254 
2255 OperandMatchResultTy
2256 AMDGPUAsmParser::parseNamedBit(const char *Name, OperandVector &Operands,
2257                                enum AMDGPUOperand::ImmTy ImmTy) {
2258   int64_t Bit = 0;
2259   SMLoc S = Parser.getTok().getLoc();
2260 
2261   // We are at the end of the statement, and this is a default argument, so
2262   // use a default value.
2263   if (getLexer().isNot(AsmToken::EndOfStatement)) {
2264     switch(getLexer().getKind()) {
2265       case AsmToken::Identifier: {
2266         StringRef Tok = Parser.getTok().getString();
2267         if (Tok == Name) {
2268           Bit = 1;
2269           Parser.Lex();
2270         } else if (Tok.startswith("no") && Tok.endswith(Name)) {
2271           Bit = 0;
2272           Parser.Lex();
2273         } else {
2274           return MatchOperand_NoMatch;
2275         }
2276         break;
2277       }
2278       default:
2279         return MatchOperand_NoMatch;
2280     }
2281   }
2282 
2283   Operands.push_back(AMDGPUOperand::CreateImm(this, Bit, S, ImmTy));
2284   return MatchOperand_Success;
2285 }
2286 
2287 typedef std::map<enum AMDGPUOperand::ImmTy, unsigned> OptionalImmIndexMap;
2288 
2289 void addOptionalImmOperand(MCInst& Inst, const OperandVector& Operands,
2290                            OptionalImmIndexMap& OptionalIdx,
2291                            enum AMDGPUOperand::ImmTy ImmT, int64_t Default = 0) {
2292   auto i = OptionalIdx.find(ImmT);
2293   if (i != OptionalIdx.end()) {
2294     unsigned Idx = i->second;
2295     ((AMDGPUOperand &)*Operands[Idx]).addImmOperands(Inst, 1);
2296   } else {
2297     Inst.addOperand(MCOperand::createImm(Default));
2298   }
2299 }
2300 
2301 OperandMatchResultTy
2302 AMDGPUAsmParser::parseStringWithPrefix(StringRef Prefix, StringRef &Value) {
2303   if (getLexer().isNot(AsmToken::Identifier)) {
2304     return MatchOperand_NoMatch;
2305   }
2306   StringRef Tok = Parser.getTok().getString();
2307   if (Tok != Prefix) {
2308     return MatchOperand_NoMatch;
2309   }
2310 
2311   Parser.Lex();
2312   if (getLexer().isNot(AsmToken::Colon)) {
2313     return MatchOperand_ParseFail;
2314   }
2315 
2316   Parser.Lex();
2317   if (getLexer().isNot(AsmToken::Identifier)) {
2318     return MatchOperand_ParseFail;
2319   }
2320 
2321   Value = Parser.getTok().getString();
2322   return MatchOperand_Success;
2323 }
2324 
2325 //===----------------------------------------------------------------------===//
2326 // ds
2327 //===----------------------------------------------------------------------===//
2328 
2329 void AMDGPUAsmParser::cvtDSOffset01(MCInst &Inst,
2330                                     const OperandVector &Operands) {
2331   OptionalImmIndexMap OptionalIdx;
2332 
2333   for (unsigned i = 1, e = Operands.size(); i != e; ++i) {
2334     AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[i]);
2335 
2336     // Add the register arguments
2337     if (Op.isReg()) {
2338       Op.addRegOperands(Inst, 1);
2339       continue;
2340     }
2341 
2342     // Handle optional arguments
2343     OptionalIdx[Op.getImmTy()] = i;
2344   }
2345 
2346   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyOffset0);
2347   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyOffset1);
2348   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyGDS);
2349 
2350   Inst.addOperand(MCOperand::createReg(AMDGPU::M0)); // m0
2351 }
2352 
2353 void AMDGPUAsmParser::cvtDS(MCInst &Inst, const OperandVector &Operands) {
2354   std::map<enum AMDGPUOperand::ImmTy, unsigned> OptionalIdx;
2355   bool GDSOnly = false;
2356 
2357   for (unsigned i = 1, e = Operands.size(); i != e; ++i) {
2358     AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[i]);
2359 
2360     // Add the register arguments
2361     if (Op.isReg()) {
2362       Op.addRegOperands(Inst, 1);
2363       continue;
2364     }
2365 
2366     if (Op.isToken() && Op.getToken() == "gds") {
2367       GDSOnly = true;
2368       continue;
2369     }
2370 
2371     // Handle optional arguments
2372     OptionalIdx[Op.getImmTy()] = i;
2373   }
2374 
2375   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyOffset);
2376   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyGDS);
2377 
2378   if (!GDSOnly) {
2379     addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyGDS);
2380   }
2381   Inst.addOperand(MCOperand::createReg(AMDGPU::M0)); // m0
2382 }
2383 
2384 void AMDGPUAsmParser::cvtExp(MCInst &Inst, const OperandVector &Operands) {
2385   OptionalImmIndexMap OptionalIdx;
2386 
2387   unsigned EnMask = 0;
2388   int SrcIdx = 0;
2389 
2390   for (unsigned i = 1, e = Operands.size(); i != e; ++i) {
2391     AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[i]);
2392 
2393     // Add the register arguments
2394     if (Op.isReg()) {
2395       EnMask |= (1 << SrcIdx);
2396       Op.addRegOperands(Inst, 1);
2397       ++SrcIdx;
2398       continue;
2399     }
2400 
2401     if (Op.isOff()) {
2402       ++SrcIdx;
2403       Inst.addOperand(MCOperand::createReg(AMDGPU::NoRegister));
2404       continue;
2405     }
2406 
2407     if (Op.isImm() && Op.getImmTy() == AMDGPUOperand::ImmTyExpTgt) {
2408       Op.addImmOperands(Inst, 1);
2409       continue;
2410     }
2411 
2412     if (Op.isToken() && Op.getToken() == "done")
2413       continue;
2414 
2415     // Handle optional arguments
2416     OptionalIdx[Op.getImmTy()] = i;
2417   }
2418 
2419   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyExpVM);
2420   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyExpCompr);
2421 
2422   Inst.addOperand(MCOperand::createImm(EnMask));
2423 }
2424 
2425 //===----------------------------------------------------------------------===//
2426 // s_waitcnt
2427 //===----------------------------------------------------------------------===//
2428 
2429 bool AMDGPUAsmParser::parseCnt(int64_t &IntVal) {
2430   StringRef CntName = Parser.getTok().getString();
2431   int64_t CntVal;
2432 
2433   Parser.Lex();
2434   if (getLexer().isNot(AsmToken::LParen))
2435     return true;
2436 
2437   Parser.Lex();
2438   if (getLexer().isNot(AsmToken::Integer))
2439     return true;
2440 
2441   if (getParser().parseAbsoluteExpression(CntVal))
2442     return true;
2443 
2444   if (getLexer().isNot(AsmToken::RParen))
2445     return true;
2446 
2447   Parser.Lex();
2448   if (getLexer().is(AsmToken::Amp) || getLexer().is(AsmToken::Comma))
2449     Parser.Lex();
2450 
2451   IsaVersion IV = getIsaVersion(getSTI().getFeatureBits());
2452   if (CntName == "vmcnt")
2453     IntVal = encodeVmcnt(IV, IntVal, CntVal);
2454   else if (CntName == "expcnt")
2455     IntVal = encodeExpcnt(IV, IntVal, CntVal);
2456   else if (CntName == "lgkmcnt")
2457     IntVal = encodeLgkmcnt(IV, IntVal, CntVal);
2458   else
2459     return true;
2460 
2461   return false;
2462 }
2463 
2464 OperandMatchResultTy
2465 AMDGPUAsmParser::parseSWaitCntOps(OperandVector &Operands) {
2466   IsaVersion IV = getIsaVersion(getSTI().getFeatureBits());
2467   int64_t Waitcnt = getWaitcntBitMask(IV);
2468   SMLoc S = Parser.getTok().getLoc();
2469 
2470   switch(getLexer().getKind()) {
2471     default: return MatchOperand_ParseFail;
2472     case AsmToken::Integer:
2473       // The operand can be an integer value.
2474       if (getParser().parseAbsoluteExpression(Waitcnt))
2475         return MatchOperand_ParseFail;
2476       break;
2477 
2478     case AsmToken::Identifier:
2479       do {
2480         if (parseCnt(Waitcnt))
2481           return MatchOperand_ParseFail;
2482       } while(getLexer().isNot(AsmToken::EndOfStatement));
2483       break;
2484   }
2485   Operands.push_back(AMDGPUOperand::CreateImm(this, Waitcnt, S));
2486   return MatchOperand_Success;
2487 }
2488 
2489 bool AMDGPUAsmParser::parseHwregConstruct(OperandInfoTy &HwReg, int64_t &Offset, int64_t &Width) {
2490   using namespace llvm::AMDGPU::Hwreg;
2491 
2492   if (Parser.getTok().getString() != "hwreg")
2493     return true;
2494   Parser.Lex();
2495 
2496   if (getLexer().isNot(AsmToken::LParen))
2497     return true;
2498   Parser.Lex();
2499 
2500   if (getLexer().is(AsmToken::Identifier)) {
2501     HwReg.IsSymbolic = true;
2502     HwReg.Id = ID_UNKNOWN_;
2503     const StringRef tok = Parser.getTok().getString();
2504     for (int i = ID_SYMBOLIC_FIRST_; i < ID_SYMBOLIC_LAST_; ++i) {
2505       if (tok == IdSymbolic[i]) {
2506         HwReg.Id = i;
2507         break;
2508       }
2509     }
2510     Parser.Lex();
2511   } else {
2512     HwReg.IsSymbolic = false;
2513     if (getLexer().isNot(AsmToken::Integer))
2514       return true;
2515     if (getParser().parseAbsoluteExpression(HwReg.Id))
2516       return true;
2517   }
2518 
2519   if (getLexer().is(AsmToken::RParen)) {
2520     Parser.Lex();
2521     return false;
2522   }
2523 
2524   // optional params
2525   if (getLexer().isNot(AsmToken::Comma))
2526     return true;
2527   Parser.Lex();
2528 
2529   if (getLexer().isNot(AsmToken::Integer))
2530     return true;
2531   if (getParser().parseAbsoluteExpression(Offset))
2532     return true;
2533 
2534   if (getLexer().isNot(AsmToken::Comma))
2535     return true;
2536   Parser.Lex();
2537 
2538   if (getLexer().isNot(AsmToken::Integer))
2539     return true;
2540   if (getParser().parseAbsoluteExpression(Width))
2541     return true;
2542 
2543   if (getLexer().isNot(AsmToken::RParen))
2544     return true;
2545   Parser.Lex();
2546 
2547   return false;
2548 }
2549 
2550 OperandMatchResultTy
2551 AMDGPUAsmParser::parseHwreg(OperandVector &Operands) {
2552   using namespace llvm::AMDGPU::Hwreg;
2553 
2554   int64_t Imm16Val = 0;
2555   SMLoc S = Parser.getTok().getLoc();
2556 
2557   switch(getLexer().getKind()) {
2558     default: return MatchOperand_NoMatch;
2559     case AsmToken::Integer:
2560       // The operand can be an integer value.
2561       if (getParser().parseAbsoluteExpression(Imm16Val))
2562         return MatchOperand_NoMatch;
2563       if (Imm16Val < 0 || !isUInt<16>(Imm16Val)) {
2564         Error(S, "invalid immediate: only 16-bit values are legal");
2565         // Do not return error code, but create an imm operand anyway and proceed
2566         // to the next operand, if any. That avoids unneccessary error messages.
2567       }
2568       break;
2569 
2570     case AsmToken::Identifier: {
2571         OperandInfoTy HwReg(ID_UNKNOWN_);
2572         int64_t Offset = OFFSET_DEFAULT_;
2573         int64_t Width = WIDTH_M1_DEFAULT_ + 1;
2574         if (parseHwregConstruct(HwReg, Offset, Width))
2575           return MatchOperand_ParseFail;
2576         if (HwReg.Id < 0 || !isUInt<ID_WIDTH_>(HwReg.Id)) {
2577           if (HwReg.IsSymbolic)
2578             Error(S, "invalid symbolic name of hardware register");
2579           else
2580             Error(S, "invalid code of hardware register: only 6-bit values are legal");
2581         }
2582         if (Offset < 0 || !isUInt<OFFSET_WIDTH_>(Offset))
2583           Error(S, "invalid bit offset: only 5-bit values are legal");
2584         if ((Width-1) < 0 || !isUInt<WIDTH_M1_WIDTH_>(Width-1))
2585           Error(S, "invalid bitfield width: only values from 1 to 32 are legal");
2586         Imm16Val = (HwReg.Id << ID_SHIFT_) | (Offset << OFFSET_SHIFT_) | ((Width-1) << WIDTH_M1_SHIFT_);
2587       }
2588       break;
2589   }
2590   Operands.push_back(AMDGPUOperand::CreateImm(this, Imm16Val, S, AMDGPUOperand::ImmTyHwreg));
2591   return MatchOperand_Success;
2592 }
2593 
2594 bool AMDGPUOperand::isSWaitCnt() const {
2595   return isImm();
2596 }
2597 
2598 bool AMDGPUOperand::isHwreg() const {
2599   return isImmTy(ImmTyHwreg);
2600 }
2601 
2602 bool AMDGPUAsmParser::parseSendMsgConstruct(OperandInfoTy &Msg, OperandInfoTy &Operation, int64_t &StreamId) {
2603   using namespace llvm::AMDGPU::SendMsg;
2604 
2605   if (Parser.getTok().getString() != "sendmsg")
2606     return true;
2607   Parser.Lex();
2608 
2609   if (getLexer().isNot(AsmToken::LParen))
2610     return true;
2611   Parser.Lex();
2612 
2613   if (getLexer().is(AsmToken::Identifier)) {
2614     Msg.IsSymbolic = true;
2615     Msg.Id = ID_UNKNOWN_;
2616     const std::string tok = Parser.getTok().getString();
2617     for (int i = ID_GAPS_FIRST_; i < ID_GAPS_LAST_; ++i) {
2618       switch(i) {
2619         default: continue; // Omit gaps.
2620         case ID_INTERRUPT: case ID_GS: case ID_GS_DONE:  case ID_SYSMSG: break;
2621       }
2622       if (tok == IdSymbolic[i]) {
2623         Msg.Id = i;
2624         break;
2625       }
2626     }
2627     Parser.Lex();
2628   } else {
2629     Msg.IsSymbolic = false;
2630     if (getLexer().isNot(AsmToken::Integer))
2631       return true;
2632     if (getParser().parseAbsoluteExpression(Msg.Id))
2633       return true;
2634     if (getLexer().is(AsmToken::Integer))
2635       if (getParser().parseAbsoluteExpression(Msg.Id))
2636         Msg.Id = ID_UNKNOWN_;
2637   }
2638   if (Msg.Id == ID_UNKNOWN_) // Don't know how to parse the rest.
2639     return false;
2640 
2641   if (!(Msg.Id == ID_GS || Msg.Id == ID_GS_DONE || Msg.Id == ID_SYSMSG)) {
2642     if (getLexer().isNot(AsmToken::RParen))
2643       return true;
2644     Parser.Lex();
2645     return false;
2646   }
2647 
2648   if (getLexer().isNot(AsmToken::Comma))
2649     return true;
2650   Parser.Lex();
2651 
2652   assert(Msg.Id == ID_GS || Msg.Id == ID_GS_DONE || Msg.Id == ID_SYSMSG);
2653   Operation.Id = ID_UNKNOWN_;
2654   if (getLexer().is(AsmToken::Identifier)) {
2655     Operation.IsSymbolic = true;
2656     const char* const *S = (Msg.Id == ID_SYSMSG) ? OpSysSymbolic : OpGsSymbolic;
2657     const int F = (Msg.Id == ID_SYSMSG) ? OP_SYS_FIRST_ : OP_GS_FIRST_;
2658     const int L = (Msg.Id == ID_SYSMSG) ? OP_SYS_LAST_ : OP_GS_LAST_;
2659     const StringRef Tok = Parser.getTok().getString();
2660     for (int i = F; i < L; ++i) {
2661       if (Tok == S[i]) {
2662         Operation.Id = i;
2663         break;
2664       }
2665     }
2666     Parser.Lex();
2667   } else {
2668     Operation.IsSymbolic = false;
2669     if (getLexer().isNot(AsmToken::Integer))
2670       return true;
2671     if (getParser().parseAbsoluteExpression(Operation.Id))
2672       return true;
2673   }
2674 
2675   if ((Msg.Id == ID_GS || Msg.Id == ID_GS_DONE) && Operation.Id != OP_GS_NOP) {
2676     // Stream id is optional.
2677     if (getLexer().is(AsmToken::RParen)) {
2678       Parser.Lex();
2679       return false;
2680     }
2681 
2682     if (getLexer().isNot(AsmToken::Comma))
2683       return true;
2684     Parser.Lex();
2685 
2686     if (getLexer().isNot(AsmToken::Integer))
2687       return true;
2688     if (getParser().parseAbsoluteExpression(StreamId))
2689       return true;
2690   }
2691 
2692   if (getLexer().isNot(AsmToken::RParen))
2693     return true;
2694   Parser.Lex();
2695   return false;
2696 }
2697 
2698 OperandMatchResultTy AMDGPUAsmParser::parseInterpSlot(OperandVector &Operands) {
2699   if (getLexer().getKind() != AsmToken::Identifier)
2700     return MatchOperand_NoMatch;
2701 
2702   StringRef Str = Parser.getTok().getString();
2703   int Slot = StringSwitch<int>(Str)
2704     .Case("p10", 0)
2705     .Case("p20", 1)
2706     .Case("p0", 2)
2707     .Default(-1);
2708 
2709   SMLoc S = Parser.getTok().getLoc();
2710   if (Slot == -1)
2711     return MatchOperand_ParseFail;
2712 
2713   Parser.Lex();
2714   Operands.push_back(AMDGPUOperand::CreateImm(this, Slot, S,
2715                                               AMDGPUOperand::ImmTyInterpSlot));
2716   return MatchOperand_Success;
2717 }
2718 
2719 OperandMatchResultTy AMDGPUAsmParser::parseInterpAttr(OperandVector &Operands) {
2720   if (getLexer().getKind() != AsmToken::Identifier)
2721     return MatchOperand_NoMatch;
2722 
2723   StringRef Str = Parser.getTok().getString();
2724   if (!Str.startswith("attr"))
2725     return MatchOperand_NoMatch;
2726 
2727   StringRef Chan = Str.take_back(2);
2728   int AttrChan = StringSwitch<int>(Chan)
2729     .Case(".x", 0)
2730     .Case(".y", 1)
2731     .Case(".z", 2)
2732     .Case(".w", 3)
2733     .Default(-1);
2734   if (AttrChan == -1)
2735     return MatchOperand_ParseFail;
2736 
2737   Str = Str.drop_back(2).drop_front(4);
2738 
2739   uint8_t Attr;
2740   if (Str.getAsInteger(10, Attr))
2741     return MatchOperand_ParseFail;
2742 
2743   SMLoc S = Parser.getTok().getLoc();
2744   Parser.Lex();
2745   if (Attr > 63) {
2746     Error(S, "out of bounds attr");
2747     return MatchOperand_Success;
2748   }
2749 
2750   SMLoc SChan = SMLoc::getFromPointer(Chan.data());
2751 
2752   Operands.push_back(AMDGPUOperand::CreateImm(this, Attr, S,
2753                                               AMDGPUOperand::ImmTyInterpAttr));
2754   Operands.push_back(AMDGPUOperand::CreateImm(this, AttrChan, SChan,
2755                                               AMDGPUOperand::ImmTyAttrChan));
2756   return MatchOperand_Success;
2757 }
2758 
2759 void AMDGPUAsmParser::errorExpTgt() {
2760   Error(Parser.getTok().getLoc(), "invalid exp target");
2761 }
2762 
2763 OperandMatchResultTy AMDGPUAsmParser::parseExpTgtImpl(StringRef Str,
2764                                                       uint8_t &Val) {
2765   if (Str == "null") {
2766     Val = 9;
2767     return MatchOperand_Success;
2768   }
2769 
2770   if (Str.startswith("mrt")) {
2771     Str = Str.drop_front(3);
2772     if (Str == "z") { // == mrtz
2773       Val = 8;
2774       return MatchOperand_Success;
2775     }
2776 
2777     if (Str.getAsInteger(10, Val))
2778       return MatchOperand_ParseFail;
2779 
2780     if (Val > 7)
2781       errorExpTgt();
2782 
2783     return MatchOperand_Success;
2784   }
2785 
2786   if (Str.startswith("pos")) {
2787     Str = Str.drop_front(3);
2788     if (Str.getAsInteger(10, Val))
2789       return MatchOperand_ParseFail;
2790 
2791     if (Val > 3)
2792       errorExpTgt();
2793 
2794     Val += 12;
2795     return MatchOperand_Success;
2796   }
2797 
2798   if (Str.startswith("param")) {
2799     Str = Str.drop_front(5);
2800     if (Str.getAsInteger(10, Val))
2801       return MatchOperand_ParseFail;
2802 
2803     if (Val >= 32)
2804       errorExpTgt();
2805 
2806     Val += 32;
2807     return MatchOperand_Success;
2808   }
2809 
2810   if (Str.startswith("invalid_target_")) {
2811     Str = Str.drop_front(15);
2812     if (Str.getAsInteger(10, Val))
2813       return MatchOperand_ParseFail;
2814 
2815     errorExpTgt();
2816     return MatchOperand_Success;
2817   }
2818 
2819   return MatchOperand_NoMatch;
2820 }
2821 
2822 OperandMatchResultTy AMDGPUAsmParser::parseExpTgt(OperandVector &Operands) {
2823   uint8_t Val;
2824   StringRef Str = Parser.getTok().getString();
2825 
2826   auto Res = parseExpTgtImpl(Str, Val);
2827   if (Res != MatchOperand_Success)
2828     return Res;
2829 
2830   SMLoc S = Parser.getTok().getLoc();
2831   Parser.Lex();
2832 
2833   Operands.push_back(AMDGPUOperand::CreateImm(this, Val, S,
2834                                               AMDGPUOperand::ImmTyExpTgt));
2835   return MatchOperand_Success;
2836 }
2837 
2838 OperandMatchResultTy
2839 AMDGPUAsmParser::parseSendMsgOp(OperandVector &Operands) {
2840   using namespace llvm::AMDGPU::SendMsg;
2841 
2842   int64_t Imm16Val = 0;
2843   SMLoc S = Parser.getTok().getLoc();
2844 
2845   switch(getLexer().getKind()) {
2846   default:
2847     return MatchOperand_NoMatch;
2848   case AsmToken::Integer:
2849     // The operand can be an integer value.
2850     if (getParser().parseAbsoluteExpression(Imm16Val))
2851       return MatchOperand_NoMatch;
2852     if (Imm16Val < 0 || !isUInt<16>(Imm16Val)) {
2853       Error(S, "invalid immediate: only 16-bit values are legal");
2854       // Do not return error code, but create an imm operand anyway and proceed
2855       // to the next operand, if any. That avoids unneccessary error messages.
2856     }
2857     break;
2858   case AsmToken::Identifier: {
2859       OperandInfoTy Msg(ID_UNKNOWN_);
2860       OperandInfoTy Operation(OP_UNKNOWN_);
2861       int64_t StreamId = STREAM_ID_DEFAULT_;
2862       if (parseSendMsgConstruct(Msg, Operation, StreamId))
2863         return MatchOperand_ParseFail;
2864       do {
2865         // Validate and encode message ID.
2866         if (! ((ID_INTERRUPT <= Msg.Id && Msg.Id <= ID_GS_DONE)
2867                 || Msg.Id == ID_SYSMSG)) {
2868           if (Msg.IsSymbolic)
2869             Error(S, "invalid/unsupported symbolic name of message");
2870           else
2871             Error(S, "invalid/unsupported code of message");
2872           break;
2873         }
2874         Imm16Val = (Msg.Id << ID_SHIFT_);
2875         // Validate and encode operation ID.
2876         if (Msg.Id == ID_GS || Msg.Id == ID_GS_DONE) {
2877           if (! (OP_GS_FIRST_ <= Operation.Id && Operation.Id < OP_GS_LAST_)) {
2878             if (Operation.IsSymbolic)
2879               Error(S, "invalid symbolic name of GS_OP");
2880             else
2881               Error(S, "invalid code of GS_OP: only 2-bit values are legal");
2882             break;
2883           }
2884           if (Operation.Id == OP_GS_NOP
2885               && Msg.Id != ID_GS_DONE) {
2886             Error(S, "invalid GS_OP: NOP is for GS_DONE only");
2887             break;
2888           }
2889           Imm16Val |= (Operation.Id << OP_SHIFT_);
2890         }
2891         if (Msg.Id == ID_SYSMSG) {
2892           if (! (OP_SYS_FIRST_ <= Operation.Id && Operation.Id < OP_SYS_LAST_)) {
2893             if (Operation.IsSymbolic)
2894               Error(S, "invalid/unsupported symbolic name of SYSMSG_OP");
2895             else
2896               Error(S, "invalid/unsupported code of SYSMSG_OP");
2897             break;
2898           }
2899           Imm16Val |= (Operation.Id << OP_SHIFT_);
2900         }
2901         // Validate and encode stream ID.
2902         if ((Msg.Id == ID_GS || Msg.Id == ID_GS_DONE) && Operation.Id != OP_GS_NOP) {
2903           if (! (STREAM_ID_FIRST_ <= StreamId && StreamId < STREAM_ID_LAST_)) {
2904             Error(S, "invalid stream id: only 2-bit values are legal");
2905             break;
2906           }
2907           Imm16Val |= (StreamId << STREAM_ID_SHIFT_);
2908         }
2909       } while (false);
2910     }
2911     break;
2912   }
2913   Operands.push_back(AMDGPUOperand::CreateImm(this, Imm16Val, S, AMDGPUOperand::ImmTySendMsg));
2914   return MatchOperand_Success;
2915 }
2916 
2917 bool AMDGPUOperand::isSendMsg() const {
2918   return isImmTy(ImmTySendMsg);
2919 }
2920 
2921 //===----------------------------------------------------------------------===//
2922 // sopp branch targets
2923 //===----------------------------------------------------------------------===//
2924 
2925 OperandMatchResultTy
2926 AMDGPUAsmParser::parseSOppBrTarget(OperandVector &Operands) {
2927   SMLoc S = Parser.getTok().getLoc();
2928 
2929   switch (getLexer().getKind()) {
2930     default: return MatchOperand_ParseFail;
2931     case AsmToken::Integer: {
2932       int64_t Imm;
2933       if (getParser().parseAbsoluteExpression(Imm))
2934         return MatchOperand_ParseFail;
2935       Operands.push_back(AMDGPUOperand::CreateImm(this, Imm, S));
2936       return MatchOperand_Success;
2937     }
2938 
2939     case AsmToken::Identifier:
2940       Operands.push_back(AMDGPUOperand::CreateExpr(this,
2941           MCSymbolRefExpr::create(getContext().getOrCreateSymbol(
2942                                   Parser.getTok().getString()), getContext()), S));
2943       Parser.Lex();
2944       return MatchOperand_Success;
2945   }
2946 }
2947 
2948 //===----------------------------------------------------------------------===//
2949 // mubuf
2950 //===----------------------------------------------------------------------===//
2951 
2952 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultGLC() const {
2953   return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTyGLC);
2954 }
2955 
2956 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultSLC() const {
2957   return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTySLC);
2958 }
2959 
2960 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultTFE() const {
2961   return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTyTFE);
2962 }
2963 
2964 void AMDGPUAsmParser::cvtMubufImpl(MCInst &Inst,
2965                                const OperandVector &Operands,
2966                                bool IsAtomic, bool IsAtomicReturn) {
2967   OptionalImmIndexMap OptionalIdx;
2968   assert(IsAtomicReturn ? IsAtomic : true);
2969 
2970   for (unsigned i = 1, e = Operands.size(); i != e; ++i) {
2971     AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[i]);
2972 
2973     // Add the register arguments
2974     if (Op.isReg()) {
2975       Op.addRegOperands(Inst, 1);
2976       continue;
2977     }
2978 
2979     // Handle the case where soffset is an immediate
2980     if (Op.isImm() && Op.getImmTy() == AMDGPUOperand::ImmTyNone) {
2981       Op.addImmOperands(Inst, 1);
2982       continue;
2983     }
2984 
2985     // Handle tokens like 'offen' which are sometimes hard-coded into the
2986     // asm string.  There are no MCInst operands for these.
2987     if (Op.isToken()) {
2988       continue;
2989     }
2990     assert(Op.isImm());
2991 
2992     // Handle optional arguments
2993     OptionalIdx[Op.getImmTy()] = i;
2994   }
2995 
2996   // Copy $vdata_in operand and insert as $vdata for MUBUF_Atomic RTN insns.
2997   if (IsAtomicReturn) {
2998     MCInst::iterator I = Inst.begin(); // $vdata_in is always at the beginning.
2999     Inst.insert(I, *I);
3000   }
3001 
3002   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyOffset);
3003   if (!IsAtomic) { // glc is hard-coded.
3004     addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyGLC);
3005   }
3006   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySLC);
3007   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyTFE);
3008 }
3009 
3010 //===----------------------------------------------------------------------===//
3011 // mimg
3012 //===----------------------------------------------------------------------===//
3013 
3014 void AMDGPUAsmParser::cvtMIMG(MCInst &Inst, const OperandVector &Operands) {
3015   unsigned I = 1;
3016   const MCInstrDesc &Desc = MII.get(Inst.getOpcode());
3017   for (unsigned J = 0; J < Desc.getNumDefs(); ++J) {
3018     ((AMDGPUOperand &)*Operands[I++]).addRegOperands(Inst, 1);
3019   }
3020 
3021   OptionalImmIndexMap OptionalIdx;
3022 
3023   for (unsigned E = Operands.size(); I != E; ++I) {
3024     AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[I]);
3025 
3026     // Add the register arguments
3027     if (Op.isRegOrImm()) {
3028       Op.addRegOrImmOperands(Inst, 1);
3029       continue;
3030     } else if (Op.isImmModifier()) {
3031       OptionalIdx[Op.getImmTy()] = I;
3032     } else {
3033       llvm_unreachable("unexpected operand type");
3034     }
3035   }
3036 
3037   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyDMask);
3038   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyUNorm);
3039   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyGLC);
3040   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyDA);
3041   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyR128);
3042   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyTFE);
3043   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyLWE);
3044   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySLC);
3045 }
3046 
3047 void AMDGPUAsmParser::cvtMIMGAtomic(MCInst &Inst, const OperandVector &Operands) {
3048   unsigned I = 1;
3049   const MCInstrDesc &Desc = MII.get(Inst.getOpcode());
3050   for (unsigned J = 0; J < Desc.getNumDefs(); ++J) {
3051     ((AMDGPUOperand &)*Operands[I++]).addRegOperands(Inst, 1);
3052   }
3053 
3054   // Add src, same as dst
3055   ((AMDGPUOperand &)*Operands[I]).addRegOperands(Inst, 1);
3056 
3057   OptionalImmIndexMap OptionalIdx;
3058 
3059   for (unsigned E = Operands.size(); I != E; ++I) {
3060     AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[I]);
3061 
3062     // Add the register arguments
3063     if (Op.isRegOrImm()) {
3064       Op.addRegOrImmOperands(Inst, 1);
3065       continue;
3066     } else if (Op.isImmModifier()) {
3067       OptionalIdx[Op.getImmTy()] = I;
3068     } else {
3069       llvm_unreachable("unexpected operand type");
3070     }
3071   }
3072 
3073   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyDMask);
3074   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyUNorm);
3075   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyGLC);
3076   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyDA);
3077   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyR128);
3078   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyTFE);
3079   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyLWE);
3080   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySLC);
3081 }
3082 
3083 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultDMask() const {
3084   return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTyDMask);
3085 }
3086 
3087 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultUNorm() const {
3088   return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTyUNorm);
3089 }
3090 
3091 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultDA() const {
3092   return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTyDA);
3093 }
3094 
3095 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultR128() const {
3096   return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTyR128);
3097 }
3098 
3099 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultLWE() const {
3100   return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTyLWE);
3101 }
3102 
3103 //===----------------------------------------------------------------------===//
3104 // smrd
3105 //===----------------------------------------------------------------------===//
3106 
3107 bool AMDGPUOperand::isSMRDOffset8() const {
3108   return isImm() && isUInt<8>(getImm());
3109 }
3110 
3111 bool AMDGPUOperand::isSMRDOffset20() const {
3112   return isImm() && isUInt<20>(getImm());
3113 }
3114 
3115 bool AMDGPUOperand::isSMRDLiteralOffset() const {
3116   // 32-bit literals are only supported on CI and we only want to use them
3117   // when the offset is > 8-bits.
3118   return isImm() && !isUInt<8>(getImm()) && isUInt<32>(getImm());
3119 }
3120 
3121 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultSMRDOffset8() const {
3122   return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTyOffset);
3123 }
3124 
3125 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultSMRDOffset20() const {
3126   return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTyOffset);
3127 }
3128 
3129 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultSMRDLiteralOffset() const {
3130   return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTyOffset);
3131 }
3132 
3133 //===----------------------------------------------------------------------===//
3134 // vop3
3135 //===----------------------------------------------------------------------===//
3136 
3137 static bool ConvertOmodMul(int64_t &Mul) {
3138   if (Mul != 1 && Mul != 2 && Mul != 4)
3139     return false;
3140 
3141   Mul >>= 1;
3142   return true;
3143 }
3144 
3145 static bool ConvertOmodDiv(int64_t &Div) {
3146   if (Div == 1) {
3147     Div = 0;
3148     return true;
3149   }
3150 
3151   if (Div == 2) {
3152     Div = 3;
3153     return true;
3154   }
3155 
3156   return false;
3157 }
3158 
3159 static bool ConvertBoundCtrl(int64_t &BoundCtrl) {
3160   if (BoundCtrl == 0) {
3161     BoundCtrl = 1;
3162     return true;
3163   }
3164 
3165   if (BoundCtrl == -1) {
3166     BoundCtrl = 0;
3167     return true;
3168   }
3169 
3170   return false;
3171 }
3172 
3173 // Note: the order in this table matches the order of operands in AsmString.
3174 static const OptionalOperand AMDGPUOptionalOperandTable[] = {
3175   {"offen",   AMDGPUOperand::ImmTyOffen, true, nullptr},
3176   {"idxen",   AMDGPUOperand::ImmTyIdxen, true, nullptr},
3177   {"addr64",  AMDGPUOperand::ImmTyAddr64, true, nullptr},
3178   {"offset0", AMDGPUOperand::ImmTyOffset0, false, nullptr},
3179   {"offset1", AMDGPUOperand::ImmTyOffset1, false, nullptr},
3180   {"gds",     AMDGPUOperand::ImmTyGDS, true, nullptr},
3181   {"offset",  AMDGPUOperand::ImmTyOffset, false, nullptr},
3182   {"glc",     AMDGPUOperand::ImmTyGLC, true, nullptr},
3183   {"slc",     AMDGPUOperand::ImmTySLC, true, nullptr},
3184   {"tfe",     AMDGPUOperand::ImmTyTFE, true, nullptr},
3185   {"clamp",   AMDGPUOperand::ImmTyClampSI, true, nullptr},
3186   {"omod",    AMDGPUOperand::ImmTyOModSI, false, ConvertOmodMul},
3187   {"unorm",   AMDGPUOperand::ImmTyUNorm, true, nullptr},
3188   {"da",      AMDGPUOperand::ImmTyDA,    true, nullptr},
3189   {"r128",    AMDGPUOperand::ImmTyR128,  true, nullptr},
3190   {"lwe",     AMDGPUOperand::ImmTyLWE,   true, nullptr},
3191   {"dmask",   AMDGPUOperand::ImmTyDMask, false, nullptr},
3192   {"row_mask",   AMDGPUOperand::ImmTyDppRowMask, false, nullptr},
3193   {"bank_mask",  AMDGPUOperand::ImmTyDppBankMask, false, nullptr},
3194   {"bound_ctrl", AMDGPUOperand::ImmTyDppBoundCtrl, false, ConvertBoundCtrl},
3195   {"dst_sel",    AMDGPUOperand::ImmTySdwaDstSel, false, nullptr},
3196   {"src0_sel",   AMDGPUOperand::ImmTySdwaSrc0Sel, false, nullptr},
3197   {"src1_sel",   AMDGPUOperand::ImmTySdwaSrc1Sel, false, nullptr},
3198   {"dst_unused", AMDGPUOperand::ImmTySdwaDstUnused, false, nullptr},
3199   {"vm", AMDGPUOperand::ImmTyExpVM, true, nullptr},
3200 };
3201 
3202 OperandMatchResultTy AMDGPUAsmParser::parseOptionalOperand(OperandVector &Operands) {
3203   OperandMatchResultTy res;
3204   for (const OptionalOperand &Op : AMDGPUOptionalOperandTable) {
3205     // try to parse any optional operand here
3206     if (Op.IsBit) {
3207       res = parseNamedBit(Op.Name, Operands, Op.Type);
3208     } else if (Op.Type == AMDGPUOperand::ImmTyOModSI) {
3209       res = parseOModOperand(Operands);
3210     } else if (Op.Type == AMDGPUOperand::ImmTySdwaDstSel ||
3211                Op.Type == AMDGPUOperand::ImmTySdwaSrc0Sel ||
3212                Op.Type == AMDGPUOperand::ImmTySdwaSrc1Sel) {
3213       res = parseSDWASel(Operands, Op.Name, Op.Type);
3214     } else if (Op.Type == AMDGPUOperand::ImmTySdwaDstUnused) {
3215       res = parseSDWADstUnused(Operands);
3216     } else {
3217       res = parseIntWithPrefix(Op.Name, Operands, Op.Type, Op.ConvertResult);
3218     }
3219     if (res != MatchOperand_NoMatch) {
3220       return res;
3221     }
3222   }
3223   return MatchOperand_NoMatch;
3224 }
3225 
3226 OperandMatchResultTy AMDGPUAsmParser::parseOModOperand(OperandVector &Operands) {
3227   StringRef Name = Parser.getTok().getString();
3228   if (Name == "mul") {
3229     return parseIntWithPrefix("mul", Operands,
3230                               AMDGPUOperand::ImmTyOModSI, ConvertOmodMul);
3231   }
3232 
3233   if (Name == "div") {
3234     return parseIntWithPrefix("div", Operands,
3235                               AMDGPUOperand::ImmTyOModSI, ConvertOmodDiv);
3236   }
3237 
3238   return MatchOperand_NoMatch;
3239 }
3240 
3241 void AMDGPUAsmParser::cvtId(MCInst &Inst, const OperandVector &Operands) {
3242   unsigned I = 1;
3243   const MCInstrDesc &Desc = MII.get(Inst.getOpcode());
3244   for (unsigned J = 0; J < Desc.getNumDefs(); ++J) {
3245     ((AMDGPUOperand &)*Operands[I++]).addRegOperands(Inst, 1);
3246   }
3247   for (unsigned E = Operands.size(); I != E; ++I)
3248     ((AMDGPUOperand &)*Operands[I]).addRegOrImmOperands(Inst, 1);
3249 }
3250 
3251 void AMDGPUAsmParser::cvtVOP3_2_mod(MCInst &Inst, const OperandVector &Operands) {
3252   uint64_t TSFlags = MII.get(Inst.getOpcode()).TSFlags;
3253   if (TSFlags & SIInstrFlags::VOP3) {
3254     cvtVOP3(Inst, Operands);
3255   } else {
3256     cvtId(Inst, Operands);
3257   }
3258 }
3259 
3260 static bool isRegOrImmWithInputMods(const MCInstrDesc &Desc, unsigned OpNum) {
3261       // 1. This operand is input modifiers
3262   return Desc.OpInfo[OpNum].OperandType == AMDGPU::OPERAND_INPUT_MODS
3263       // 2. This is not last operand
3264       && Desc.NumOperands > (OpNum + 1)
3265       // 3. Next operand is register class
3266       && Desc.OpInfo[OpNum + 1].RegClass != -1
3267       // 4. Next register is not tied to any other operand
3268       && Desc.getOperandConstraint(OpNum + 1, MCOI::OperandConstraint::TIED_TO) == -1;
3269 }
3270 
3271 void AMDGPUAsmParser::cvtVOP3(MCInst &Inst, const OperandVector &Operands) {
3272   OptionalImmIndexMap OptionalIdx;
3273   unsigned I = 1;
3274   const MCInstrDesc &Desc = MII.get(Inst.getOpcode());
3275   for (unsigned J = 0; J < Desc.getNumDefs(); ++J) {
3276     ((AMDGPUOperand &)*Operands[I++]).addRegOperands(Inst, 1);
3277   }
3278 
3279   for (unsigned E = Operands.size(); I != E; ++I) {
3280     AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[I]);
3281     if (isRegOrImmWithInputMods(Desc, Inst.getNumOperands())) {
3282       Op.addRegOrImmWithFPInputModsOperands(Inst, 2);
3283     } else if (Op.isImm()) {
3284       OptionalIdx[Op.getImmTy()] = I;
3285     } else {
3286       llvm_unreachable("unhandled operand type");
3287     }
3288   }
3289 
3290   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyClampSI);
3291   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyOModSI);
3292 
3293   // special case v_mac_{f16, f32}:
3294   // it has src2 register operand that is tied to dst operand
3295   // we don't allow modifiers for this operand in assembler so src2_modifiers
3296   // should be 0
3297   if (Inst.getOpcode() == AMDGPU::V_MAC_F32_e64_si ||
3298       Inst.getOpcode() == AMDGPU::V_MAC_F32_e64_vi ||
3299       Inst.getOpcode() == AMDGPU::V_MAC_F16_e64_vi) {
3300     auto it = Inst.begin();
3301     std::advance(
3302       it,
3303       AMDGPU::getNamedOperandIdx(Inst.getOpcode() == AMDGPU::V_MAC_F16_e64_vi ?
3304                                      AMDGPU::V_MAC_F16_e64 :
3305                                      AMDGPU::V_MAC_F32_e64,
3306                                  AMDGPU::OpName::src2_modifiers));
3307     it = Inst.insert(it, MCOperand::createImm(0)); // no modifiers for src2
3308     ++it;
3309     Inst.insert(it, Inst.getOperand(0)); // src2 = dst
3310   }
3311 }
3312 
3313 //===----------------------------------------------------------------------===//
3314 // dpp
3315 //===----------------------------------------------------------------------===//
3316 
3317 bool AMDGPUOperand::isDPPCtrl() const {
3318   bool result = isImm() && getImmTy() == ImmTyDppCtrl && isUInt<9>(getImm());
3319   if (result) {
3320     int64_t Imm = getImm();
3321     return ((Imm >= 0x000) && (Imm <= 0x0ff)) ||
3322            ((Imm >= 0x101) && (Imm <= 0x10f)) ||
3323            ((Imm >= 0x111) && (Imm <= 0x11f)) ||
3324            ((Imm >= 0x121) && (Imm <= 0x12f)) ||
3325            (Imm == 0x130) ||
3326            (Imm == 0x134) ||
3327            (Imm == 0x138) ||
3328            (Imm == 0x13c) ||
3329            (Imm == 0x140) ||
3330            (Imm == 0x141) ||
3331            (Imm == 0x142) ||
3332            (Imm == 0x143);
3333   }
3334   return false;
3335 }
3336 
3337 bool AMDGPUOperand::isGPRIdxMode() const {
3338   return isImm() && isUInt<4>(getImm());
3339 }
3340 
3341 OperandMatchResultTy
3342 AMDGPUAsmParser::parseDPPCtrl(OperandVector &Operands) {
3343   SMLoc S = Parser.getTok().getLoc();
3344   StringRef Prefix;
3345   int64_t Int;
3346 
3347   if (getLexer().getKind() == AsmToken::Identifier) {
3348     Prefix = Parser.getTok().getString();
3349   } else {
3350     return MatchOperand_NoMatch;
3351   }
3352 
3353   if (Prefix == "row_mirror") {
3354     Int = 0x140;
3355     Parser.Lex();
3356   } else if (Prefix == "row_half_mirror") {
3357     Int = 0x141;
3358     Parser.Lex();
3359   } else {
3360     // Check to prevent parseDPPCtrlOps from eating invalid tokens
3361     if (Prefix != "quad_perm"
3362         && Prefix != "row_shl"
3363         && Prefix != "row_shr"
3364         && Prefix != "row_ror"
3365         && Prefix != "wave_shl"
3366         && Prefix != "wave_rol"
3367         && Prefix != "wave_shr"
3368         && Prefix != "wave_ror"
3369         && Prefix != "row_bcast") {
3370       return MatchOperand_NoMatch;
3371     }
3372 
3373     Parser.Lex();
3374     if (getLexer().isNot(AsmToken::Colon))
3375       return MatchOperand_ParseFail;
3376 
3377     if (Prefix == "quad_perm") {
3378       // quad_perm:[%d,%d,%d,%d]
3379       Parser.Lex();
3380       if (getLexer().isNot(AsmToken::LBrac))
3381         return MatchOperand_ParseFail;
3382       Parser.Lex();
3383 
3384       if (getParser().parseAbsoluteExpression(Int) || !(0 <= Int && Int <=3))
3385         return MatchOperand_ParseFail;
3386 
3387       for (int i = 0; i < 3; ++i) {
3388         if (getLexer().isNot(AsmToken::Comma))
3389           return MatchOperand_ParseFail;
3390         Parser.Lex();
3391 
3392         int64_t Temp;
3393         if (getParser().parseAbsoluteExpression(Temp) || !(0 <= Temp && Temp <=3))
3394           return MatchOperand_ParseFail;
3395         const int shift = i*2 + 2;
3396         Int += (Temp << shift);
3397       }
3398 
3399       if (getLexer().isNot(AsmToken::RBrac))
3400         return MatchOperand_ParseFail;
3401       Parser.Lex();
3402 
3403     } else {
3404       // sel:%d
3405       Parser.Lex();
3406       if (getParser().parseAbsoluteExpression(Int))
3407         return MatchOperand_ParseFail;
3408 
3409       if (Prefix == "row_shl" && 1 <= Int && Int <= 15) {
3410         Int |= 0x100;
3411       } else if (Prefix == "row_shr" && 1 <= Int && Int <= 15) {
3412         Int |= 0x110;
3413       } else if (Prefix == "row_ror" && 1 <= Int && Int <= 15) {
3414         Int |= 0x120;
3415       } else if (Prefix == "wave_shl" && 1 == Int) {
3416         Int = 0x130;
3417       } else if (Prefix == "wave_rol" && 1 == Int) {
3418         Int = 0x134;
3419       } else if (Prefix == "wave_shr" && 1 == Int) {
3420         Int = 0x138;
3421       } else if (Prefix == "wave_ror" && 1 == Int) {
3422         Int = 0x13C;
3423       } else if (Prefix == "row_bcast") {
3424         if (Int == 15) {
3425           Int = 0x142;
3426         } else if (Int == 31) {
3427           Int = 0x143;
3428         } else {
3429           return MatchOperand_ParseFail;
3430         }
3431       } else {
3432         return MatchOperand_ParseFail;
3433       }
3434     }
3435   }
3436 
3437   Operands.push_back(AMDGPUOperand::CreateImm(this, Int, S, AMDGPUOperand::ImmTyDppCtrl));
3438   return MatchOperand_Success;
3439 }
3440 
3441 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultRowMask() const {
3442   return AMDGPUOperand::CreateImm(this, 0xf, SMLoc(), AMDGPUOperand::ImmTyDppRowMask);
3443 }
3444 
3445 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultBankMask() const {
3446   return AMDGPUOperand::CreateImm(this, 0xf, SMLoc(), AMDGPUOperand::ImmTyDppBankMask);
3447 }
3448 
3449 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultBoundCtrl() const {
3450   return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTyDppBoundCtrl);
3451 }
3452 
3453 void AMDGPUAsmParser::cvtDPP(MCInst &Inst, const OperandVector &Operands) {
3454   OptionalImmIndexMap OptionalIdx;
3455 
3456   unsigned I = 1;
3457   const MCInstrDesc &Desc = MII.get(Inst.getOpcode());
3458   for (unsigned J = 0; J < Desc.getNumDefs(); ++J) {
3459     ((AMDGPUOperand &)*Operands[I++]).addRegOperands(Inst, 1);
3460   }
3461 
3462   for (unsigned E = Operands.size(); I != E; ++I) {
3463     AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[I]);
3464     // Add the register arguments
3465     if (Op.isReg() && Op.Reg.RegNo == AMDGPU::VCC) {
3466       // VOP2b (v_add_u32, v_sub_u32 ...) dpp use "vcc" token.
3467       // Skip it.
3468       continue;
3469     } if (isRegOrImmWithInputMods(Desc, Inst.getNumOperands())) {
3470       Op.addRegWithFPInputModsOperands(Inst, 2);
3471     } else if (Op.isDPPCtrl()) {
3472       Op.addImmOperands(Inst, 1);
3473     } else if (Op.isImm()) {
3474       // Handle optional arguments
3475       OptionalIdx[Op.getImmTy()] = I;
3476     } else {
3477       llvm_unreachable("Invalid operand type");
3478     }
3479   }
3480 
3481   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyDppRowMask, 0xf);
3482   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyDppBankMask, 0xf);
3483   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyDppBoundCtrl);
3484 
3485   // special case v_mac_{f16, f32}:
3486   // it has src2 register operand that is tied to dst operand
3487   if (Inst.getOpcode() == AMDGPU::V_MAC_F32_dpp ||
3488       Inst.getOpcode() == AMDGPU::V_MAC_F16_dpp) {
3489     auto it = Inst.begin();
3490     std::advance(
3491         it, AMDGPU::getNamedOperandIdx(Inst.getOpcode(), AMDGPU::OpName::src2));
3492     Inst.insert(it, Inst.getOperand(0)); // src2 = dst
3493   }
3494 }
3495 
3496 //===----------------------------------------------------------------------===//
3497 // sdwa
3498 //===----------------------------------------------------------------------===//
3499 
3500 OperandMatchResultTy
3501 AMDGPUAsmParser::parseSDWASel(OperandVector &Operands, StringRef Prefix,
3502                               AMDGPUOperand::ImmTy Type) {
3503   using namespace llvm::AMDGPU::SDWA;
3504 
3505   SMLoc S = Parser.getTok().getLoc();
3506   StringRef Value;
3507   OperandMatchResultTy res;
3508 
3509   res = parseStringWithPrefix(Prefix, Value);
3510   if (res != MatchOperand_Success) {
3511     return res;
3512   }
3513 
3514   int64_t Int;
3515   Int = StringSwitch<int64_t>(Value)
3516         .Case("BYTE_0", SdwaSel::BYTE_0)
3517         .Case("BYTE_1", SdwaSel::BYTE_1)
3518         .Case("BYTE_2", SdwaSel::BYTE_2)
3519         .Case("BYTE_3", SdwaSel::BYTE_3)
3520         .Case("WORD_0", SdwaSel::WORD_0)
3521         .Case("WORD_1", SdwaSel::WORD_1)
3522         .Case("DWORD", SdwaSel::DWORD)
3523         .Default(0xffffffff);
3524   Parser.Lex(); // eat last token
3525 
3526   if (Int == 0xffffffff) {
3527     return MatchOperand_ParseFail;
3528   }
3529 
3530   Operands.push_back(AMDGPUOperand::CreateImm(this, Int, S, Type));
3531   return MatchOperand_Success;
3532 }
3533 
3534 OperandMatchResultTy
3535 AMDGPUAsmParser::parseSDWADstUnused(OperandVector &Operands) {
3536   using namespace llvm::AMDGPU::SDWA;
3537 
3538   SMLoc S = Parser.getTok().getLoc();
3539   StringRef Value;
3540   OperandMatchResultTy res;
3541 
3542   res = parseStringWithPrefix("dst_unused", Value);
3543   if (res != MatchOperand_Success) {
3544     return res;
3545   }
3546 
3547   int64_t Int;
3548   Int = StringSwitch<int64_t>(Value)
3549         .Case("UNUSED_PAD", DstUnused::UNUSED_PAD)
3550         .Case("UNUSED_SEXT", DstUnused::UNUSED_SEXT)
3551         .Case("UNUSED_PRESERVE", DstUnused::UNUSED_PRESERVE)
3552         .Default(0xffffffff);
3553   Parser.Lex(); // eat last token
3554 
3555   if (Int == 0xffffffff) {
3556     return MatchOperand_ParseFail;
3557   }
3558 
3559   Operands.push_back(AMDGPUOperand::CreateImm(this, Int, S, AMDGPUOperand::ImmTySdwaDstUnused));
3560   return MatchOperand_Success;
3561 }
3562 
3563 void AMDGPUAsmParser::cvtSdwaVOP1(MCInst &Inst, const OperandVector &Operands) {
3564   cvtSDWA(Inst, Operands, SIInstrFlags::VOP1);
3565 }
3566 
3567 void AMDGPUAsmParser::cvtSdwaVOP2(MCInst &Inst, const OperandVector &Operands) {
3568   cvtSDWA(Inst, Operands, SIInstrFlags::VOP2);
3569 }
3570 
3571 void AMDGPUAsmParser::cvtSdwaVOPC(MCInst &Inst, const OperandVector &Operands) {
3572   cvtSDWA(Inst, Operands, SIInstrFlags::VOPC);
3573 }
3574 
3575 void AMDGPUAsmParser::cvtSDWA(MCInst &Inst, const OperandVector &Operands,
3576                               uint64_t BasicInstType) {
3577   using namespace llvm::AMDGPU::SDWA;
3578   OptionalImmIndexMap OptionalIdx;
3579 
3580   unsigned I = 1;
3581   const MCInstrDesc &Desc = MII.get(Inst.getOpcode());
3582   for (unsigned J = 0; J < Desc.getNumDefs(); ++J) {
3583     ((AMDGPUOperand &)*Operands[I++]).addRegOperands(Inst, 1);
3584   }
3585 
3586   for (unsigned E = Operands.size(); I != E; ++I) {
3587     AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[I]);
3588     // Add the register arguments
3589     if ((BasicInstType == SIInstrFlags::VOPC ||
3590          BasicInstType == SIInstrFlags::VOP2)&&
3591         Op.isReg() &&
3592         Op.Reg.RegNo == AMDGPU::VCC) {
3593       // VOPC and VOP2b (v_add_u32, v_sub_u32 ...) sdwa use "vcc" token as dst.
3594       // Skip it.
3595       continue;
3596     } else if (isRegOrImmWithInputMods(Desc, Inst.getNumOperands())) {
3597       Op.addRegWithInputModsOperands(Inst, 2);
3598     } else if (Op.isImm()) {
3599       // Handle optional arguments
3600       OptionalIdx[Op.getImmTy()] = I;
3601     } else {
3602       llvm_unreachable("Invalid operand type");
3603     }
3604   }
3605 
3606   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyClampSI, 0);
3607 
3608   if (Inst.getOpcode() != AMDGPU::V_NOP_sdwa_vi) {
3609     // V_NOP_sdwa_vi has no optional sdwa arguments
3610     switch (BasicInstType) {
3611     case SIInstrFlags::VOP1:
3612       addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySdwaDstSel, SdwaSel::DWORD);
3613       addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySdwaDstUnused, DstUnused::UNUSED_PRESERVE);
3614       addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySdwaSrc0Sel, SdwaSel::DWORD);
3615       break;
3616 
3617     case SIInstrFlags::VOP2:
3618       addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySdwaDstSel, SdwaSel::DWORD);
3619       addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySdwaDstUnused, DstUnused::UNUSED_PRESERVE);
3620       addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySdwaSrc0Sel, SdwaSel::DWORD);
3621       addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySdwaSrc1Sel, SdwaSel::DWORD);
3622       break;
3623 
3624     case SIInstrFlags::VOPC:
3625       addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySdwaSrc0Sel, SdwaSel::DWORD);
3626       addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySdwaSrc1Sel, SdwaSel::DWORD);
3627       break;
3628 
3629     default:
3630       llvm_unreachable("Invalid instruction type. Only VOP1, VOP2 and VOPC allowed");
3631     }
3632   }
3633 
3634   // special case v_mac_{f16, f32}:
3635   // it has src2 register operand that is tied to dst operand
3636   if (Inst.getOpcode() == AMDGPU::V_MAC_F32_sdwa_vi ||
3637       Inst.getOpcode() == AMDGPU::V_MAC_F16_sdwa_vi)  {
3638     auto it = Inst.begin();
3639     std::advance(
3640         it, AMDGPU::getNamedOperandIdx(Inst.getOpcode(), AMDGPU::OpName::src2));
3641     Inst.insert(it, Inst.getOperand(0)); // src2 = dst
3642   }
3643 
3644 }
3645 
3646 /// Force static initialization.
3647 extern "C" void LLVMInitializeAMDGPUAsmParser() {
3648   RegisterMCAsmParser<AMDGPUAsmParser> A(getTheAMDGPUTarget());
3649   RegisterMCAsmParser<AMDGPUAsmParser> B(getTheGCNTarget());
3650 }
3651 
3652 #define GET_REGISTER_MATCHER
3653 #define GET_MATCHER_IMPLEMENTATION
3654 #include "AMDGPUGenAsmMatcher.inc"
3655 
3656 // This fuction should be defined after auto-generated include so that we have
3657 // MatchClassKind enum defined
3658 unsigned AMDGPUAsmParser::validateTargetOperandClass(MCParsedAsmOperand &Op,
3659                                                      unsigned Kind) {
3660   // Tokens like "glc" would be parsed as immediate operands in ParseOperand().
3661   // But MatchInstructionImpl() expects to meet token and fails to validate
3662   // operand. This method checks if we are given immediate operand but expect to
3663   // get corresponding token.
3664   AMDGPUOperand &Operand = (AMDGPUOperand&)Op;
3665   switch (Kind) {
3666   case MCK_addr64:
3667     return Operand.isAddr64() ? Match_Success : Match_InvalidOperand;
3668   case MCK_gds:
3669     return Operand.isGDS() ? Match_Success : Match_InvalidOperand;
3670   case MCK_glc:
3671     return Operand.isGLC() ? Match_Success : Match_InvalidOperand;
3672   case MCK_idxen:
3673     return Operand.isIdxen() ? Match_Success : Match_InvalidOperand;
3674   case MCK_offen:
3675     return Operand.isOffen() ? Match_Success : Match_InvalidOperand;
3676   case MCK_SSrcB32:
3677     // When operands have expression values, they will return true for isToken,
3678     // because it is not possible to distinguish between a token and an
3679     // expression at parse time. MatchInstructionImpl() will always try to
3680     // match an operand as a token, when isToken returns true, and when the
3681     // name of the expression is not a valid token, the match will fail,
3682     // so we need to handle it here.
3683     return Operand.isSSrcB32() ? Match_Success : Match_InvalidOperand;
3684   case MCK_SSrcF32:
3685     return Operand.isSSrcF32() ? Match_Success : Match_InvalidOperand;
3686   case MCK_SoppBrTarget:
3687     return Operand.isSoppBrTarget() ? Match_Success : Match_InvalidOperand;
3688   case MCK_VReg32OrOff:
3689     return Operand.isVReg32OrOff() ? Match_Success : Match_InvalidOperand;
3690   case MCK_InterpSlot:
3691     return Operand.isInterpSlot() ? Match_Success : Match_InvalidOperand;
3692   case MCK_Attr:
3693     return Operand.isInterpAttr() ? Match_Success : Match_InvalidOperand;
3694   case MCK_AttrChan:
3695     return Operand.isAttrChan() ? Match_Success : Match_InvalidOperand;
3696   default:
3697     return Match_InvalidOperand;
3698   }
3699 }
3700