1 //===-- AMDGPUAsmParser.cpp - Parse SI asm to MCInst instructions ---------===//
2 //
3 //                     The LLVM Compiler Infrastructure
4 //
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
7 //
8 //===----------------------------------------------------------------------===//
9 
10 #include "AMDKernelCodeT.h"
11 #include "MCTargetDesc/AMDGPUMCTargetDesc.h"
12 #include "MCTargetDesc/AMDGPUTargetStreamer.h"
13 #include "SIDefines.h"
14 #include "Utils/AMDGPUAsmUtils.h"
15 #include "Utils/AMDGPUBaseInfo.h"
16 #include "Utils/AMDKernelCodeTUtils.h"
17 #include "llvm/ADT/APFloat.h"
18 #include "llvm/ADT/APInt.h"
19 #include "llvm/ADT/ArrayRef.h"
20 #include "llvm/ADT/STLExtras.h"
21 #include "llvm/ADT/SmallBitVector.h"
22 #include "llvm/ADT/SmallString.h"
23 #include "llvm/ADT/StringRef.h"
24 #include "llvm/ADT/StringSwitch.h"
25 #include "llvm/ADT/Twine.h"
26 #include "llvm/BinaryFormat/ELF.h"
27 #include "llvm/CodeGen/MachineValueType.h"
28 #include "llvm/MC/MCAsmInfo.h"
29 #include "llvm/MC/MCContext.h"
30 #include "llvm/MC/MCExpr.h"
31 #include "llvm/MC/MCInst.h"
32 #include "llvm/MC/MCInstrDesc.h"
33 #include "llvm/MC/MCInstrInfo.h"
34 #include "llvm/MC/MCParser/MCAsmLexer.h"
35 #include "llvm/MC/MCParser/MCAsmParser.h"
36 #include "llvm/MC/MCParser/MCAsmParserExtension.h"
37 #include "llvm/MC/MCParser/MCParsedAsmOperand.h"
38 #include "llvm/MC/MCParser/MCTargetAsmParser.h"
39 #include "llvm/MC/MCRegisterInfo.h"
40 #include "llvm/MC/MCStreamer.h"
41 #include "llvm/MC/MCSubtargetInfo.h"
42 #include "llvm/MC/MCSymbol.h"
43 #include "llvm/Support/Casting.h"
44 #include "llvm/Support/ErrorHandling.h"
45 #include "llvm/Support/MathExtras.h"
46 #include "llvm/Support/SMLoc.h"
47 #include "llvm/Support/TargetRegistry.h"
48 #include "llvm/Support/raw_ostream.h"
49 #include <algorithm>
50 #include <cassert>
51 #include <cstdint>
52 #include <cstring>
53 #include <iterator>
54 #include <map>
55 #include <memory>
56 #include <string>
57 
58 using namespace llvm;
59 using namespace llvm::AMDGPU;
60 
61 namespace {
62 
63 class AMDGPUAsmParser;
64 
65 enum RegisterKind { IS_UNKNOWN, IS_VGPR, IS_SGPR, IS_TTMP, IS_SPECIAL };
66 
67 //===----------------------------------------------------------------------===//
68 // Operand
69 //===----------------------------------------------------------------------===//
70 
71 class AMDGPUOperand : public MCParsedAsmOperand {
72   enum KindTy {
73     Token,
74     Immediate,
75     Register,
76     Expression
77   } Kind;
78 
79   SMLoc StartLoc, EndLoc;
80   const AMDGPUAsmParser *AsmParser;
81 
82 public:
83   AMDGPUOperand(KindTy Kind_, const AMDGPUAsmParser *AsmParser_)
84     : MCParsedAsmOperand(), Kind(Kind_), AsmParser(AsmParser_) {}
85 
86   typedef std::unique_ptr<AMDGPUOperand> Ptr;
87 
88   struct Modifiers {
89     bool Abs = false;
90     bool Neg = false;
91     bool Sext = false;
92 
93     bool hasFPModifiers() const { return Abs || Neg; }
94     bool hasIntModifiers() const { return Sext; }
95     bool hasModifiers() const { return hasFPModifiers() || hasIntModifiers(); }
96 
97     int64_t getFPModifiersOperand() const {
98       int64_t Operand = 0;
99       Operand |= Abs ? SISrcMods::ABS : 0;
100       Operand |= Neg ? SISrcMods::NEG : 0;
101       return Operand;
102     }
103 
104     int64_t getIntModifiersOperand() const {
105       int64_t Operand = 0;
106       Operand |= Sext ? SISrcMods::SEXT : 0;
107       return Operand;
108     }
109 
110     int64_t getModifiersOperand() const {
111       assert(!(hasFPModifiers() && hasIntModifiers())
112            && "fp and int modifiers should not be used simultaneously");
113       if (hasFPModifiers()) {
114         return getFPModifiersOperand();
115       } else if (hasIntModifiers()) {
116         return getIntModifiersOperand();
117       } else {
118         return 0;
119       }
120     }
121 
122     friend raw_ostream &operator <<(raw_ostream &OS, AMDGPUOperand::Modifiers Mods);
123   };
124 
125   enum ImmTy {
126     ImmTyNone,
127     ImmTyGDS,
128     ImmTyOffen,
129     ImmTyIdxen,
130     ImmTyAddr64,
131     ImmTyOffset,
132     ImmTyOffset0,
133     ImmTyOffset1,
134     ImmTyGLC,
135     ImmTySLC,
136     ImmTyTFE,
137     ImmTyClampSI,
138     ImmTyOModSI,
139     ImmTyDppCtrl,
140     ImmTyDppRowMask,
141     ImmTyDppBankMask,
142     ImmTyDppBoundCtrl,
143     ImmTySdwaDstSel,
144     ImmTySdwaSrc0Sel,
145     ImmTySdwaSrc1Sel,
146     ImmTySdwaDstUnused,
147     ImmTyDMask,
148     ImmTyUNorm,
149     ImmTyDA,
150     ImmTyR128,
151     ImmTyLWE,
152     ImmTyExpTgt,
153     ImmTyExpCompr,
154     ImmTyExpVM,
155     ImmTyDFMT,
156     ImmTyNFMT,
157     ImmTyHwreg,
158     ImmTyOff,
159     ImmTySendMsg,
160     ImmTyInterpSlot,
161     ImmTyInterpAttr,
162     ImmTyAttrChan,
163     ImmTyOpSel,
164     ImmTyOpSelHi,
165     ImmTyNegLo,
166     ImmTyNegHi,
167     ImmTySwizzle
168   };
169 
170   struct TokOp {
171     const char *Data;
172     unsigned Length;
173   };
174 
175   struct ImmOp {
176     int64_t Val;
177     ImmTy Type;
178     bool IsFPImm;
179     Modifiers Mods;
180   };
181 
182   struct RegOp {
183     unsigned RegNo;
184     bool IsForcedVOP3;
185     Modifiers Mods;
186   };
187 
188   union {
189     TokOp Tok;
190     ImmOp Imm;
191     RegOp Reg;
192     const MCExpr *Expr;
193   };
194 
195   bool isToken() const override {
196     if (Kind == Token)
197       return true;
198 
199     if (Kind != Expression || !Expr)
200       return false;
201 
202     // When parsing operands, we can't always tell if something was meant to be
203     // a token, like 'gds', or an expression that references a global variable.
204     // In this case, we assume the string is an expression, and if we need to
205     // interpret is a token, then we treat the symbol name as the token.
206     return isa<MCSymbolRefExpr>(Expr);
207   }
208 
209   bool isImm() const override {
210     return Kind == Immediate;
211   }
212 
213   bool isInlinableImm(MVT type) const;
214   bool isLiteralImm(MVT type) const;
215 
216   bool isRegKind() const {
217     return Kind == Register;
218   }
219 
220   bool isReg() const override {
221     return isRegKind() && !hasModifiers();
222   }
223 
224   bool isRegOrImmWithInputMods(MVT type) const {
225     return isRegKind() || isInlinableImm(type);
226   }
227 
228   bool isRegOrImmWithInt16InputMods() const {
229     return isRegOrImmWithInputMods(MVT::i16);
230   }
231 
232   bool isRegOrImmWithInt32InputMods() const {
233     return isRegOrImmWithInputMods(MVT::i32);
234   }
235 
236   bool isRegOrImmWithInt64InputMods() const {
237     return isRegOrImmWithInputMods(MVT::i64);
238   }
239 
240   bool isRegOrImmWithFP16InputMods() const {
241     return isRegOrImmWithInputMods(MVT::f16);
242   }
243 
244   bool isRegOrImmWithFP32InputMods() const {
245     return isRegOrImmWithInputMods(MVT::f32);
246   }
247 
248   bool isRegOrImmWithFP64InputMods() const {
249     return isRegOrImmWithInputMods(MVT::f64);
250   }
251 
252   bool isVReg() const {
253     return isRegClass(AMDGPU::VGPR_32RegClassID) ||
254            isRegClass(AMDGPU::VReg_64RegClassID) ||
255            isRegClass(AMDGPU::VReg_96RegClassID) ||
256            isRegClass(AMDGPU::VReg_128RegClassID) ||
257            isRegClass(AMDGPU::VReg_256RegClassID) ||
258            isRegClass(AMDGPU::VReg_512RegClassID);
259   }
260 
261   bool isVReg32OrOff() const {
262     return isOff() || isRegClass(AMDGPU::VGPR_32RegClassID);
263   }
264 
265   bool isSDWARegKind() const;
266 
267   bool isImmTy(ImmTy ImmT) const {
268     return isImm() && Imm.Type == ImmT;
269   }
270 
271   bool isImmModifier() const {
272     return isImm() && Imm.Type != ImmTyNone;
273   }
274 
275   bool isClampSI() const { return isImmTy(ImmTyClampSI); }
276   bool isOModSI() const { return isImmTy(ImmTyOModSI); }
277   bool isDMask() const { return isImmTy(ImmTyDMask); }
278   bool isUNorm() const { return isImmTy(ImmTyUNorm); }
279   bool isDA() const { return isImmTy(ImmTyDA); }
280   bool isR128() const { return isImmTy(ImmTyUNorm); }
281   bool isLWE() const { return isImmTy(ImmTyLWE); }
282   bool isOff() const { return isImmTy(ImmTyOff); }
283   bool isExpTgt() const { return isImmTy(ImmTyExpTgt); }
284   bool isExpVM() const { return isImmTy(ImmTyExpVM); }
285   bool isExpCompr() const { return isImmTy(ImmTyExpCompr); }
286   bool isOffen() const { return isImmTy(ImmTyOffen); }
287   bool isIdxen() const { return isImmTy(ImmTyIdxen); }
288   bool isAddr64() const { return isImmTy(ImmTyAddr64); }
289   bool isOffset() const { return isImmTy(ImmTyOffset) && isUInt<16>(getImm()); }
290   bool isOffset0() const { return isImmTy(ImmTyOffset0) && isUInt<16>(getImm()); }
291   bool isOffset1() const { return isImmTy(ImmTyOffset1) && isUInt<8>(getImm()); }
292 
293   bool isOffsetU12() const { return isImmTy(ImmTyOffset) && isUInt<12>(getImm()); }
294   bool isOffsetS13() const { return isImmTy(ImmTyOffset) && isInt<13>(getImm()); }
295   bool isGDS() const { return isImmTy(ImmTyGDS); }
296   bool isGLC() const { return isImmTy(ImmTyGLC); }
297   bool isSLC() const { return isImmTy(ImmTySLC); }
298   bool isTFE() const { return isImmTy(ImmTyTFE); }
299   bool isDFMT() const { return isImmTy(ImmTyDFMT) && isUInt<8>(getImm()); }
300   bool isNFMT() const { return isImmTy(ImmTyNFMT) && isUInt<8>(getImm()); }
301   bool isBankMask() const { return isImmTy(ImmTyDppBankMask); }
302   bool isRowMask() const { return isImmTy(ImmTyDppRowMask); }
303   bool isBoundCtrl() const { return isImmTy(ImmTyDppBoundCtrl); }
304   bool isSDWADstSel() const { return isImmTy(ImmTySdwaDstSel); }
305   bool isSDWASrc0Sel() const { return isImmTy(ImmTySdwaSrc0Sel); }
306   bool isSDWASrc1Sel() const { return isImmTy(ImmTySdwaSrc1Sel); }
307   bool isSDWADstUnused() const { return isImmTy(ImmTySdwaDstUnused); }
308   bool isInterpSlot() const { return isImmTy(ImmTyInterpSlot); }
309   bool isInterpAttr() const { return isImmTy(ImmTyInterpAttr); }
310   bool isAttrChan() const { return isImmTy(ImmTyAttrChan); }
311   bool isOpSel() const { return isImmTy(ImmTyOpSel); }
312   bool isOpSelHi() const { return isImmTy(ImmTyOpSelHi); }
313   bool isNegLo() const { return isImmTy(ImmTyNegLo); }
314   bool isNegHi() const { return isImmTy(ImmTyNegHi); }
315 
316   bool isMod() const {
317     return isClampSI() || isOModSI();
318   }
319 
320   bool isRegOrImm() const {
321     return isReg() || isImm();
322   }
323 
324   bool isRegClass(unsigned RCID) const;
325 
326   bool isRegOrInlineNoMods(unsigned RCID, MVT type) const {
327     return (isRegClass(RCID) || isInlinableImm(type)) && !hasModifiers();
328   }
329 
330   bool isSCSrcB16() const {
331     return isRegOrInlineNoMods(AMDGPU::SReg_32RegClassID, MVT::i16);
332   }
333 
334   bool isSCSrcV2B16() const {
335     return isSCSrcB16();
336   }
337 
338   bool isSCSrcB32() const {
339     return isRegOrInlineNoMods(AMDGPU::SReg_32RegClassID, MVT::i32);
340   }
341 
342   bool isSCSrcB64() const {
343     return isRegOrInlineNoMods(AMDGPU::SReg_64RegClassID, MVT::i64);
344   }
345 
346   bool isSCSrcF16() const {
347     return isRegOrInlineNoMods(AMDGPU::SReg_32RegClassID, MVT::f16);
348   }
349 
350   bool isSCSrcV2F16() const {
351     return isSCSrcF16();
352   }
353 
354   bool isSCSrcF32() const {
355     return isRegOrInlineNoMods(AMDGPU::SReg_32RegClassID, MVT::f32);
356   }
357 
358   bool isSCSrcF64() const {
359     return isRegOrInlineNoMods(AMDGPU::SReg_64RegClassID, MVT::f64);
360   }
361 
362   bool isSSrcB32() const {
363     return isSCSrcB32() || isLiteralImm(MVT::i32) || isExpr();
364   }
365 
366   bool isSSrcB16() const {
367     return isSCSrcB16() || isLiteralImm(MVT::i16);
368   }
369 
370   bool isSSrcV2B16() const {
371     llvm_unreachable("cannot happen");
372     return isSSrcB16();
373   }
374 
375   bool isSSrcB64() const {
376     // TODO: Find out how SALU supports extension of 32-bit literals to 64 bits.
377     // See isVSrc64().
378     return isSCSrcB64() || isLiteralImm(MVT::i64);
379   }
380 
381   bool isSSrcF32() const {
382     return isSCSrcB32() || isLiteralImm(MVT::f32) || isExpr();
383   }
384 
385   bool isSSrcF64() const {
386     return isSCSrcB64() || isLiteralImm(MVT::f64);
387   }
388 
389   bool isSSrcF16() const {
390     return isSCSrcB16() || isLiteralImm(MVT::f16);
391   }
392 
393   bool isSSrcV2F16() const {
394     llvm_unreachable("cannot happen");
395     return isSSrcF16();
396   }
397 
398   bool isVCSrcB32() const {
399     return isRegOrInlineNoMods(AMDGPU::VS_32RegClassID, MVT::i32);
400   }
401 
402   bool isVCSrcB64() const {
403     return isRegOrInlineNoMods(AMDGPU::VS_64RegClassID, MVT::i64);
404   }
405 
406   bool isVCSrcB16() const {
407     return isRegOrInlineNoMods(AMDGPU::VS_32RegClassID, MVT::i16);
408   }
409 
410   bool isVCSrcV2B16() const {
411     return isVCSrcB16();
412   }
413 
414   bool isVCSrcF32() const {
415     return isRegOrInlineNoMods(AMDGPU::VS_32RegClassID, MVT::f32);
416   }
417 
418   bool isVCSrcF64() const {
419     return isRegOrInlineNoMods(AMDGPU::VS_64RegClassID, MVT::f64);
420   }
421 
422   bool isVCSrcF16() const {
423     return isRegOrInlineNoMods(AMDGPU::VS_32RegClassID, MVT::f16);
424   }
425 
426   bool isVCSrcV2F16() const {
427     return isVCSrcF16();
428   }
429 
430   bool isVSrcB32() const {
431     return isVCSrcF32() || isLiteralImm(MVT::i32);
432   }
433 
434   bool isVSrcB64() const {
435     return isVCSrcF64() || isLiteralImm(MVT::i64);
436   }
437 
438   bool isVSrcB16() const {
439     return isVCSrcF16() || isLiteralImm(MVT::i16);
440   }
441 
442   bool isVSrcV2B16() const {
443     llvm_unreachable("cannot happen");
444     return isVSrcB16();
445   }
446 
447   bool isVSrcF32() const {
448     return isVCSrcF32() || isLiteralImm(MVT::f32);
449   }
450 
451   bool isVSrcF64() const {
452     return isVCSrcF64() || isLiteralImm(MVT::f64);
453   }
454 
455   bool isVSrcF16() const {
456     return isVCSrcF16() || isLiteralImm(MVT::f16);
457   }
458 
459   bool isVSrcV2F16() const {
460     llvm_unreachable("cannot happen");
461     return isVSrcF16();
462   }
463 
464   bool isKImmFP32() const {
465     return isLiteralImm(MVT::f32);
466   }
467 
468   bool isKImmFP16() const {
469     return isLiteralImm(MVT::f16);
470   }
471 
472   bool isMem() const override {
473     return false;
474   }
475 
476   bool isExpr() const {
477     return Kind == Expression;
478   }
479 
480   bool isSoppBrTarget() const {
481     return isExpr() || isImm();
482   }
483 
484   bool isSWaitCnt() const;
485   bool isHwreg() const;
486   bool isSendMsg() const;
487   bool isSwizzle() const;
488   bool isSMRDOffset8() const;
489   bool isSMRDOffset20() const;
490   bool isSMRDLiteralOffset() const;
491   bool isDPPCtrl() const;
492   bool isGPRIdxMode() const;
493   bool isS16Imm() const;
494   bool isU16Imm() const;
495 
496   StringRef getExpressionAsToken() const {
497     assert(isExpr());
498     const MCSymbolRefExpr *S = cast<MCSymbolRefExpr>(Expr);
499     return S->getSymbol().getName();
500   }
501 
502   StringRef getToken() const {
503     assert(isToken());
504 
505     if (Kind == Expression)
506       return getExpressionAsToken();
507 
508     return StringRef(Tok.Data, Tok.Length);
509   }
510 
511   int64_t getImm() const {
512     assert(isImm());
513     return Imm.Val;
514   }
515 
516   ImmTy getImmTy() const {
517     assert(isImm());
518     return Imm.Type;
519   }
520 
521   unsigned getReg() const override {
522     return Reg.RegNo;
523   }
524 
525   SMLoc getStartLoc() const override {
526     return StartLoc;
527   }
528 
529   SMLoc getEndLoc() const override {
530     return EndLoc;
531   }
532 
533   Modifiers getModifiers() const {
534     assert(isRegKind() || isImmTy(ImmTyNone));
535     return isRegKind() ? Reg.Mods : Imm.Mods;
536   }
537 
538   void setModifiers(Modifiers Mods) {
539     assert(isRegKind() || isImmTy(ImmTyNone));
540     if (isRegKind())
541       Reg.Mods = Mods;
542     else
543       Imm.Mods = Mods;
544   }
545 
546   bool hasModifiers() const {
547     return getModifiers().hasModifiers();
548   }
549 
550   bool hasFPModifiers() const {
551     return getModifiers().hasFPModifiers();
552   }
553 
554   bool hasIntModifiers() const {
555     return getModifiers().hasIntModifiers();
556   }
557 
558   uint64_t applyInputFPModifiers(uint64_t Val, unsigned Size) const;
559 
560   void addImmOperands(MCInst &Inst, unsigned N, bool ApplyModifiers = true) const;
561 
562   void addLiteralImmOperand(MCInst &Inst, int64_t Val, bool ApplyModifiers) const;
563 
564   template <unsigned Bitwidth>
565   void addKImmFPOperands(MCInst &Inst, unsigned N) const;
566 
567   void addKImmFP16Operands(MCInst &Inst, unsigned N) const {
568     addKImmFPOperands<16>(Inst, N);
569   }
570 
571   void addKImmFP32Operands(MCInst &Inst, unsigned N) const {
572     addKImmFPOperands<32>(Inst, N);
573   }
574 
575   void addRegOperands(MCInst &Inst, unsigned N) const;
576 
577   void addRegOrImmOperands(MCInst &Inst, unsigned N) const {
578     if (isRegKind())
579       addRegOperands(Inst, N);
580     else if (isExpr())
581       Inst.addOperand(MCOperand::createExpr(Expr));
582     else
583       addImmOperands(Inst, N);
584   }
585 
586   void addRegOrImmWithInputModsOperands(MCInst &Inst, unsigned N) const {
587     Modifiers Mods = getModifiers();
588     Inst.addOperand(MCOperand::createImm(Mods.getModifiersOperand()));
589     if (isRegKind()) {
590       addRegOperands(Inst, N);
591     } else {
592       addImmOperands(Inst, N, false);
593     }
594   }
595 
596   void addRegOrImmWithFPInputModsOperands(MCInst &Inst, unsigned N) const {
597     assert(!hasIntModifiers());
598     addRegOrImmWithInputModsOperands(Inst, N);
599   }
600 
601   void addRegOrImmWithIntInputModsOperands(MCInst &Inst, unsigned N) const {
602     assert(!hasFPModifiers());
603     addRegOrImmWithInputModsOperands(Inst, N);
604   }
605 
606   void addRegWithInputModsOperands(MCInst &Inst, unsigned N) const {
607     Modifiers Mods = getModifiers();
608     Inst.addOperand(MCOperand::createImm(Mods.getModifiersOperand()));
609     assert(isRegKind());
610     addRegOperands(Inst, N);
611   }
612 
613   void addRegWithFPInputModsOperands(MCInst &Inst, unsigned N) const {
614     assert(!hasIntModifiers());
615     addRegWithInputModsOperands(Inst, N);
616   }
617 
618   void addRegWithIntInputModsOperands(MCInst &Inst, unsigned N) const {
619     assert(!hasFPModifiers());
620     addRegWithInputModsOperands(Inst, N);
621   }
622 
623   void addSoppBrTargetOperands(MCInst &Inst, unsigned N) const {
624     if (isImm())
625       addImmOperands(Inst, N);
626     else {
627       assert(isExpr());
628       Inst.addOperand(MCOperand::createExpr(Expr));
629     }
630   }
631 
632   static void printImmTy(raw_ostream& OS, ImmTy Type) {
633     switch (Type) {
634     case ImmTyNone: OS << "None"; break;
635     case ImmTyGDS: OS << "GDS"; break;
636     case ImmTyOffen: OS << "Offen"; break;
637     case ImmTyIdxen: OS << "Idxen"; break;
638     case ImmTyAddr64: OS << "Addr64"; break;
639     case ImmTyOffset: OS << "Offset"; break;
640     case ImmTyOffset0: OS << "Offset0"; break;
641     case ImmTyOffset1: OS << "Offset1"; break;
642     case ImmTyGLC: OS << "GLC"; break;
643     case ImmTySLC: OS << "SLC"; break;
644     case ImmTyTFE: OS << "TFE"; break;
645     case ImmTyDFMT: OS << "DFMT"; break;
646     case ImmTyNFMT: OS << "NFMT"; break;
647     case ImmTyClampSI: OS << "ClampSI"; break;
648     case ImmTyOModSI: OS << "OModSI"; break;
649     case ImmTyDppCtrl: OS << "DppCtrl"; break;
650     case ImmTyDppRowMask: OS << "DppRowMask"; break;
651     case ImmTyDppBankMask: OS << "DppBankMask"; break;
652     case ImmTyDppBoundCtrl: OS << "DppBoundCtrl"; break;
653     case ImmTySdwaDstSel: OS << "SdwaDstSel"; break;
654     case ImmTySdwaSrc0Sel: OS << "SdwaSrc0Sel"; break;
655     case ImmTySdwaSrc1Sel: OS << "SdwaSrc1Sel"; break;
656     case ImmTySdwaDstUnused: OS << "SdwaDstUnused"; break;
657     case ImmTyDMask: OS << "DMask"; break;
658     case ImmTyUNorm: OS << "UNorm"; break;
659     case ImmTyDA: OS << "DA"; break;
660     case ImmTyR128: OS << "R128"; break;
661     case ImmTyLWE: OS << "LWE"; break;
662     case ImmTyOff: OS << "Off"; break;
663     case ImmTyExpTgt: OS << "ExpTgt"; break;
664     case ImmTyExpCompr: OS << "ExpCompr"; break;
665     case ImmTyExpVM: OS << "ExpVM"; break;
666     case ImmTyHwreg: OS << "Hwreg"; break;
667     case ImmTySendMsg: OS << "SendMsg"; break;
668     case ImmTyInterpSlot: OS << "InterpSlot"; break;
669     case ImmTyInterpAttr: OS << "InterpAttr"; break;
670     case ImmTyAttrChan: OS << "AttrChan"; break;
671     case ImmTyOpSel: OS << "OpSel"; break;
672     case ImmTyOpSelHi: OS << "OpSelHi"; break;
673     case ImmTyNegLo: OS << "NegLo"; break;
674     case ImmTyNegHi: OS << "NegHi"; break;
675     case ImmTySwizzle: OS << "Swizzle"; break;
676     }
677   }
678 
679   void print(raw_ostream &OS) const override {
680     switch (Kind) {
681     case Register:
682       OS << "<register " << getReg() << " mods: " << Reg.Mods << '>';
683       break;
684     case Immediate:
685       OS << '<' << getImm();
686       if (getImmTy() != ImmTyNone) {
687         OS << " type: "; printImmTy(OS, getImmTy());
688       }
689       OS << " mods: " << Imm.Mods << '>';
690       break;
691     case Token:
692       OS << '\'' << getToken() << '\'';
693       break;
694     case Expression:
695       OS << "<expr " << *Expr << '>';
696       break;
697     }
698   }
699 
700   static AMDGPUOperand::Ptr CreateImm(const AMDGPUAsmParser *AsmParser,
701                                       int64_t Val, SMLoc Loc,
702                                       ImmTy Type = ImmTyNone,
703                                       bool IsFPImm = false) {
704     auto Op = llvm::make_unique<AMDGPUOperand>(Immediate, AsmParser);
705     Op->Imm.Val = Val;
706     Op->Imm.IsFPImm = IsFPImm;
707     Op->Imm.Type = Type;
708     Op->Imm.Mods = Modifiers();
709     Op->StartLoc = Loc;
710     Op->EndLoc = Loc;
711     return Op;
712   }
713 
714   static AMDGPUOperand::Ptr CreateToken(const AMDGPUAsmParser *AsmParser,
715                                         StringRef Str, SMLoc Loc,
716                                         bool HasExplicitEncodingSize = true) {
717     auto Res = llvm::make_unique<AMDGPUOperand>(Token, AsmParser);
718     Res->Tok.Data = Str.data();
719     Res->Tok.Length = Str.size();
720     Res->StartLoc = Loc;
721     Res->EndLoc = Loc;
722     return Res;
723   }
724 
725   static AMDGPUOperand::Ptr CreateReg(const AMDGPUAsmParser *AsmParser,
726                                       unsigned RegNo, SMLoc S,
727                                       SMLoc E,
728                                       bool ForceVOP3) {
729     auto Op = llvm::make_unique<AMDGPUOperand>(Register, AsmParser);
730     Op->Reg.RegNo = RegNo;
731     Op->Reg.Mods = Modifiers();
732     Op->Reg.IsForcedVOP3 = ForceVOP3;
733     Op->StartLoc = S;
734     Op->EndLoc = E;
735     return Op;
736   }
737 
738   static AMDGPUOperand::Ptr CreateExpr(const AMDGPUAsmParser *AsmParser,
739                                        const class MCExpr *Expr, SMLoc S) {
740     auto Op = llvm::make_unique<AMDGPUOperand>(Expression, AsmParser);
741     Op->Expr = Expr;
742     Op->StartLoc = S;
743     Op->EndLoc = S;
744     return Op;
745   }
746 };
747 
748 raw_ostream &operator <<(raw_ostream &OS, AMDGPUOperand::Modifiers Mods) {
749   OS << "abs:" << Mods.Abs << " neg: " << Mods.Neg << " sext:" << Mods.Sext;
750   return OS;
751 }
752 
753 //===----------------------------------------------------------------------===//
754 // AsmParser
755 //===----------------------------------------------------------------------===//
756 
757 // Holds info related to the current kernel, e.g. count of SGPRs used.
758 // Kernel scope begins at .amdgpu_hsa_kernel directive, ends at next
759 // .amdgpu_hsa_kernel or at EOF.
760 class KernelScopeInfo {
761   int SgprIndexUnusedMin = -1;
762   int VgprIndexUnusedMin = -1;
763   MCContext *Ctx = nullptr;
764 
765   void usesSgprAt(int i) {
766     if (i >= SgprIndexUnusedMin) {
767       SgprIndexUnusedMin = ++i;
768       if (Ctx) {
769         MCSymbol * const Sym = Ctx->getOrCreateSymbol(Twine(".kernel.sgpr_count"));
770         Sym->setVariableValue(MCConstantExpr::create(SgprIndexUnusedMin, *Ctx));
771       }
772     }
773   }
774 
775   void usesVgprAt(int i) {
776     if (i >= VgprIndexUnusedMin) {
777       VgprIndexUnusedMin = ++i;
778       if (Ctx) {
779         MCSymbol * const Sym = Ctx->getOrCreateSymbol(Twine(".kernel.vgpr_count"));
780         Sym->setVariableValue(MCConstantExpr::create(VgprIndexUnusedMin, *Ctx));
781       }
782     }
783   }
784 
785 public:
786   KernelScopeInfo() = default;
787 
788   void initialize(MCContext &Context) {
789     Ctx = &Context;
790     usesSgprAt(SgprIndexUnusedMin = -1);
791     usesVgprAt(VgprIndexUnusedMin = -1);
792   }
793 
794   void usesRegister(RegisterKind RegKind, unsigned DwordRegIndex, unsigned RegWidth) {
795     switch (RegKind) {
796       case IS_SGPR: usesSgprAt(DwordRegIndex + RegWidth - 1); break;
797       case IS_VGPR: usesVgprAt(DwordRegIndex + RegWidth - 1); break;
798       default: break;
799     }
800   }
801 };
802 
803 class AMDGPUAsmParser : public MCTargetAsmParser {
804   const MCInstrInfo &MII;
805   MCAsmParser &Parser;
806 
807   unsigned ForcedEncodingSize = 0;
808   bool ForcedDPP = false;
809   bool ForcedSDWA = false;
810   KernelScopeInfo KernelScope;
811 
812   /// @name Auto-generated Match Functions
813   /// {
814 
815 #define GET_ASSEMBLER_HEADER
816 #include "AMDGPUGenAsmMatcher.inc"
817 
818   /// }
819 
820 private:
821   bool ParseAsAbsoluteExpression(uint32_t &Ret);
822   bool ParseDirectiveMajorMinor(uint32_t &Major, uint32_t &Minor);
823   bool ParseDirectiveHSACodeObjectVersion();
824   bool ParseDirectiveHSACodeObjectISA();
825   bool ParseDirectiveCodeObjectMetadata();
826   bool ParseAMDKernelCodeTValue(StringRef ID, amd_kernel_code_t &Header);
827   bool ParseDirectiveAMDKernelCodeT();
828   bool subtargetHasRegister(const MCRegisterInfo &MRI, unsigned RegNo) const;
829   bool ParseDirectiveAMDGPUHsaKernel();
830   bool AddNextRegisterToList(unsigned& Reg, unsigned& RegWidth,
831                              RegisterKind RegKind, unsigned Reg1,
832                              unsigned RegNum);
833   bool ParseAMDGPURegister(RegisterKind& RegKind, unsigned& Reg,
834                            unsigned& RegNum, unsigned& RegWidth,
835                            unsigned *DwordRegIndex);
836   void cvtMubufImpl(MCInst &Inst, const OperandVector &Operands,
837                     bool IsAtomic, bool IsAtomicReturn);
838   void cvtDSImpl(MCInst &Inst, const OperandVector &Operands,
839                  bool IsGdsHardcoded);
840 
841 public:
842   enum AMDGPUMatchResultTy {
843     Match_PreferE32 = FIRST_TARGET_MATCH_RESULT_TY
844   };
845 
846   typedef std::map<AMDGPUOperand::ImmTy, unsigned> OptionalImmIndexMap;
847 
848   AMDGPUAsmParser(const MCSubtargetInfo &STI, MCAsmParser &_Parser,
849                const MCInstrInfo &MII,
850                const MCTargetOptions &Options)
851       : MCTargetAsmParser(Options, STI), MII(MII), Parser(_Parser) {
852     MCAsmParserExtension::Initialize(Parser);
853 
854     if (getFeatureBits().none()) {
855       // Set default features.
856       copySTI().ToggleFeature("SOUTHERN_ISLANDS");
857     }
858 
859     setAvailableFeatures(ComputeAvailableFeatures(getFeatureBits()));
860 
861     {
862       // TODO: make those pre-defined variables read-only.
863       // Currently there is none suitable machinery in the core llvm-mc for this.
864       // MCSymbol::isRedefinable is intended for another purpose, and
865       // AsmParser::parseDirectiveSet() cannot be specialized for specific target.
866       AMDGPU::IsaInfo::IsaVersion ISA =
867           AMDGPU::IsaInfo::getIsaVersion(getFeatureBits());
868       MCContext &Ctx = getContext();
869       MCSymbol *Sym =
870           Ctx.getOrCreateSymbol(Twine(".option.machine_version_major"));
871       Sym->setVariableValue(MCConstantExpr::create(ISA.Major, Ctx));
872       Sym = Ctx.getOrCreateSymbol(Twine(".option.machine_version_minor"));
873       Sym->setVariableValue(MCConstantExpr::create(ISA.Minor, Ctx));
874       Sym = Ctx.getOrCreateSymbol(Twine(".option.machine_version_stepping"));
875       Sym->setVariableValue(MCConstantExpr::create(ISA.Stepping, Ctx));
876     }
877     KernelScope.initialize(getContext());
878   }
879 
880   bool isSI() const {
881     return AMDGPU::isSI(getSTI());
882   }
883 
884   bool isCI() const {
885     return AMDGPU::isCI(getSTI());
886   }
887 
888   bool isVI() const {
889     return AMDGPU::isVI(getSTI());
890   }
891 
892   bool isGFX9() const {
893     return AMDGPU::isGFX9(getSTI());
894   }
895 
896   bool hasInv2PiInlineImm() const {
897     return getFeatureBits()[AMDGPU::FeatureInv2PiInlineImm];
898   }
899 
900   bool hasFlatOffsets() const {
901     return getFeatureBits()[AMDGPU::FeatureFlatInstOffsets];
902   }
903 
904   bool hasSGPR102_SGPR103() const {
905     return !isVI();
906   }
907 
908   AMDGPUTargetStreamer &getTargetStreamer() {
909     MCTargetStreamer &TS = *getParser().getStreamer().getTargetStreamer();
910     return static_cast<AMDGPUTargetStreamer &>(TS);
911   }
912 
913   const MCRegisterInfo *getMRI() const {
914     // We need this const_cast because for some reason getContext() is not const
915     // in MCAsmParser.
916     return const_cast<AMDGPUAsmParser*>(this)->getContext().getRegisterInfo();
917   }
918 
919   const MCInstrInfo *getMII() const {
920     return &MII;
921   }
922 
923   const FeatureBitset &getFeatureBits() const {
924     return getSTI().getFeatureBits();
925   }
926 
927   void setForcedEncodingSize(unsigned Size) { ForcedEncodingSize = Size; }
928   void setForcedDPP(bool ForceDPP_) { ForcedDPP = ForceDPP_; }
929   void setForcedSDWA(bool ForceSDWA_) { ForcedSDWA = ForceSDWA_; }
930 
931   unsigned getForcedEncodingSize() const { return ForcedEncodingSize; }
932   bool isForcedVOP3() const { return ForcedEncodingSize == 64; }
933   bool isForcedDPP() const { return ForcedDPP; }
934   bool isForcedSDWA() const { return ForcedSDWA; }
935   ArrayRef<unsigned> getMatchedVariants() const;
936 
937   std::unique_ptr<AMDGPUOperand> parseRegister();
938   bool ParseRegister(unsigned &RegNo, SMLoc &StartLoc, SMLoc &EndLoc) override;
939   unsigned checkTargetMatchPredicate(MCInst &Inst) override;
940   unsigned validateTargetOperandClass(MCParsedAsmOperand &Op,
941                                       unsigned Kind) override;
942   bool MatchAndEmitInstruction(SMLoc IDLoc, unsigned &Opcode,
943                                OperandVector &Operands, MCStreamer &Out,
944                                uint64_t &ErrorInfo,
945                                bool MatchingInlineAsm) override;
946   bool ParseDirective(AsmToken DirectiveID) override;
947   OperandMatchResultTy parseOperand(OperandVector &Operands, StringRef Mnemonic);
948   StringRef parseMnemonicSuffix(StringRef Name);
949   bool ParseInstruction(ParseInstructionInfo &Info, StringRef Name,
950                         SMLoc NameLoc, OperandVector &Operands) override;
951   //bool ProcessInstruction(MCInst &Inst);
952 
953   OperandMatchResultTy parseIntWithPrefix(const char *Prefix, int64_t &Int);
954 
955   OperandMatchResultTy
956   parseIntWithPrefix(const char *Prefix, OperandVector &Operands,
957                      AMDGPUOperand::ImmTy ImmTy = AMDGPUOperand::ImmTyNone,
958                      bool (*ConvertResult)(int64_t &) = nullptr);
959 
960   OperandMatchResultTy parseOperandArrayWithPrefix(
961     const char *Prefix,
962     OperandVector &Operands,
963     AMDGPUOperand::ImmTy ImmTy = AMDGPUOperand::ImmTyNone,
964     bool (*ConvertResult)(int64_t&) = nullptr);
965 
966   OperandMatchResultTy
967   parseNamedBit(const char *Name, OperandVector &Operands,
968                 AMDGPUOperand::ImmTy ImmTy = AMDGPUOperand::ImmTyNone);
969   OperandMatchResultTy parseStringWithPrefix(StringRef Prefix,
970                                              StringRef &Value);
971 
972   bool parseAbsoluteExpr(int64_t &Val, bool AbsMod = false);
973   OperandMatchResultTy parseImm(OperandVector &Operands, bool AbsMod = false);
974   OperandMatchResultTy parseReg(OperandVector &Operands);
975   OperandMatchResultTy parseRegOrImm(OperandVector &Operands, bool AbsMod = false);
976   OperandMatchResultTy parseRegOrImmWithFPInputMods(OperandVector &Operands, bool AllowImm = true);
977   OperandMatchResultTy parseRegOrImmWithIntInputMods(OperandVector &Operands, bool AllowImm = true);
978   OperandMatchResultTy parseRegWithFPInputMods(OperandVector &Operands);
979   OperandMatchResultTy parseRegWithIntInputMods(OperandVector &Operands);
980   OperandMatchResultTy parseVReg32OrOff(OperandVector &Operands);
981 
982   void cvtDSOffset01(MCInst &Inst, const OperandVector &Operands);
983   void cvtDS(MCInst &Inst, const OperandVector &Operands) { cvtDSImpl(Inst, Operands, false); }
984   void cvtDSGds(MCInst &Inst, const OperandVector &Operands) { cvtDSImpl(Inst, Operands, true); }
985   void cvtExp(MCInst &Inst, const OperandVector &Operands);
986 
987   bool parseCnt(int64_t &IntVal);
988   OperandMatchResultTy parseSWaitCntOps(OperandVector &Operands);
989   OperandMatchResultTy parseHwreg(OperandVector &Operands);
990 
991 private:
992   struct OperandInfoTy {
993     int64_t Id;
994     bool IsSymbolic;
995     OperandInfoTy(int64_t Id_) : Id(Id_), IsSymbolic(false) { }
996   };
997 
998   bool parseSendMsgConstruct(OperandInfoTy &Msg, OperandInfoTy &Operation, int64_t &StreamId);
999   bool parseHwregConstruct(OperandInfoTy &HwReg, int64_t &Offset, int64_t &Width);
1000 
1001   void errorExpTgt();
1002   OperandMatchResultTy parseExpTgtImpl(StringRef Str, uint8_t &Val);
1003 
1004   bool validateInstruction(const MCInst &Inst, const SMLoc &IDLoc);
1005   bool validateConstantBusLimitations(const MCInst &Inst);
1006   bool validateEarlyClobberLimitations(const MCInst &Inst);
1007   bool usesConstantBus(const MCInst &Inst, unsigned OpIdx);
1008   bool isInlineConstant(const MCInst &Inst, unsigned OpIdx) const;
1009   unsigned findImplicitSGPRReadInVOP(const MCInst &Inst) const;
1010 
1011   bool trySkipId(const StringRef Id);
1012   bool trySkipToken(const AsmToken::TokenKind Kind);
1013   bool skipToken(const AsmToken::TokenKind Kind, const StringRef ErrMsg);
1014   bool parseString(StringRef &Val, const StringRef ErrMsg = "expected a string");
1015   bool parseExpr(int64_t &Imm);
1016 
1017 public:
1018   OperandMatchResultTy parseOptionalOperand(OperandVector &Operands);
1019 
1020   OperandMatchResultTy parseExpTgt(OperandVector &Operands);
1021   OperandMatchResultTy parseSendMsgOp(OperandVector &Operands);
1022   OperandMatchResultTy parseInterpSlot(OperandVector &Operands);
1023   OperandMatchResultTy parseInterpAttr(OperandVector &Operands);
1024   OperandMatchResultTy parseSOppBrTarget(OperandVector &Operands);
1025 
1026   bool parseSwizzleOperands(const unsigned OpNum, int64_t* Op,
1027                             const unsigned MinVal,
1028                             const unsigned MaxVal,
1029                             const StringRef ErrMsg);
1030   OperandMatchResultTy parseSwizzleOp(OperandVector &Operands);
1031   bool parseSwizzleOffset(int64_t &Imm);
1032   bool parseSwizzleMacro(int64_t &Imm);
1033   bool parseSwizzleQuadPerm(int64_t &Imm);
1034   bool parseSwizzleBitmaskPerm(int64_t &Imm);
1035   bool parseSwizzleBroadcast(int64_t &Imm);
1036   bool parseSwizzleSwap(int64_t &Imm);
1037   bool parseSwizzleReverse(int64_t &Imm);
1038 
1039   void cvtMubuf(MCInst &Inst, const OperandVector &Operands) { cvtMubufImpl(Inst, Operands, false, false); }
1040   void cvtMubufAtomic(MCInst &Inst, const OperandVector &Operands) { cvtMubufImpl(Inst, Operands, true, false); }
1041   void cvtMubufAtomicReturn(MCInst &Inst, const OperandVector &Operands) { cvtMubufImpl(Inst, Operands, true, true); }
1042   void cvtMtbuf(MCInst &Inst, const OperandVector &Operands);
1043 
1044   AMDGPUOperand::Ptr defaultGLC() const;
1045   AMDGPUOperand::Ptr defaultSLC() const;
1046   AMDGPUOperand::Ptr defaultTFE() const;
1047 
1048   AMDGPUOperand::Ptr defaultDMask() const;
1049   AMDGPUOperand::Ptr defaultUNorm() const;
1050   AMDGPUOperand::Ptr defaultDA() const;
1051   AMDGPUOperand::Ptr defaultR128() const;
1052   AMDGPUOperand::Ptr defaultLWE() const;
1053   AMDGPUOperand::Ptr defaultSMRDOffset8() const;
1054   AMDGPUOperand::Ptr defaultSMRDOffset20() const;
1055   AMDGPUOperand::Ptr defaultSMRDLiteralOffset() const;
1056   AMDGPUOperand::Ptr defaultOffsetU12() const;
1057   AMDGPUOperand::Ptr defaultOffsetS13() const;
1058 
1059   OperandMatchResultTy parseOModOperand(OperandVector &Operands);
1060 
1061   void cvtId(MCInst &Inst, const OperandVector &Operands);
1062   void cvtVOP3_2_mod(MCInst &Inst, const OperandVector &Operands);
1063 
1064   void cvtVOP3Impl(MCInst &Inst,
1065                    const OperandVector &Operands,
1066                    OptionalImmIndexMap &OptionalIdx);
1067   void cvtVOP3(MCInst &Inst, const OperandVector &Operands);
1068   void cvtVOP3OMod(MCInst &Inst, const OperandVector &Operands);
1069   void cvtVOP3P(MCInst &Inst, const OperandVector &Operands);
1070 
1071   void cvtMIMG(MCInst &Inst, const OperandVector &Operands);
1072   void cvtMIMGAtomic(MCInst &Inst, const OperandVector &Operands);
1073 
1074   OperandMatchResultTy parseDPPCtrl(OperandVector &Operands);
1075   AMDGPUOperand::Ptr defaultRowMask() const;
1076   AMDGPUOperand::Ptr defaultBankMask() const;
1077   AMDGPUOperand::Ptr defaultBoundCtrl() const;
1078   void cvtDPP(MCInst &Inst, const OperandVector &Operands);
1079 
1080   OperandMatchResultTy parseSDWASel(OperandVector &Operands, StringRef Prefix,
1081                                     AMDGPUOperand::ImmTy Type);
1082   OperandMatchResultTy parseSDWADstUnused(OperandVector &Operands);
1083   void cvtSdwaVOP1(MCInst &Inst, const OperandVector &Operands);
1084   void cvtSdwaVOP2(MCInst &Inst, const OperandVector &Operands);
1085   void cvtSdwaVOP2b(MCInst &Inst, const OperandVector &Operands);
1086   void cvtSdwaVOPC(MCInst &Inst, const OperandVector &Operands);
1087   void cvtSDWA(MCInst &Inst, const OperandVector &Operands,
1088                 uint64_t BasicInstType, bool skipVcc = false);
1089 };
1090 
1091 struct OptionalOperand {
1092   const char *Name;
1093   AMDGPUOperand::ImmTy Type;
1094   bool IsBit;
1095   bool (*ConvertResult)(int64_t&);
1096 };
1097 
1098 } // end anonymous namespace
1099 
1100 // May be called with integer type with equivalent bitwidth.
1101 static const fltSemantics *getFltSemantics(unsigned Size) {
1102   switch (Size) {
1103   case 4:
1104     return &APFloat::IEEEsingle();
1105   case 8:
1106     return &APFloat::IEEEdouble();
1107   case 2:
1108     return &APFloat::IEEEhalf();
1109   default:
1110     llvm_unreachable("unsupported fp type");
1111   }
1112 }
1113 
1114 static const fltSemantics *getFltSemantics(MVT VT) {
1115   return getFltSemantics(VT.getSizeInBits() / 8);
1116 }
1117 
1118 static const fltSemantics *getOpFltSemantics(uint8_t OperandType) {
1119   switch (OperandType) {
1120   case AMDGPU::OPERAND_REG_IMM_INT32:
1121   case AMDGPU::OPERAND_REG_IMM_FP32:
1122   case AMDGPU::OPERAND_REG_INLINE_C_INT32:
1123   case AMDGPU::OPERAND_REG_INLINE_C_FP32:
1124     return &APFloat::IEEEsingle();
1125   case AMDGPU::OPERAND_REG_IMM_INT64:
1126   case AMDGPU::OPERAND_REG_IMM_FP64:
1127   case AMDGPU::OPERAND_REG_INLINE_C_INT64:
1128   case AMDGPU::OPERAND_REG_INLINE_C_FP64:
1129     return &APFloat::IEEEdouble();
1130   case AMDGPU::OPERAND_REG_IMM_INT16:
1131   case AMDGPU::OPERAND_REG_IMM_FP16:
1132   case AMDGPU::OPERAND_REG_INLINE_C_INT16:
1133   case AMDGPU::OPERAND_REG_INLINE_C_FP16:
1134   case AMDGPU::OPERAND_REG_INLINE_C_V2INT16:
1135   case AMDGPU::OPERAND_REG_INLINE_C_V2FP16:
1136     return &APFloat::IEEEhalf();
1137   default:
1138     llvm_unreachable("unsupported fp type");
1139   }
1140 }
1141 
1142 //===----------------------------------------------------------------------===//
1143 // Operand
1144 //===----------------------------------------------------------------------===//
1145 
1146 static bool canLosslesslyConvertToFPType(APFloat &FPLiteral, MVT VT) {
1147   bool Lost;
1148 
1149   // Convert literal to single precision
1150   APFloat::opStatus Status = FPLiteral.convert(*getFltSemantics(VT),
1151                                                APFloat::rmNearestTiesToEven,
1152                                                &Lost);
1153   // We allow precision lost but not overflow or underflow
1154   if (Status != APFloat::opOK &&
1155       Lost &&
1156       ((Status & APFloat::opOverflow)  != 0 ||
1157        (Status & APFloat::opUnderflow) != 0)) {
1158     return false;
1159   }
1160 
1161   return true;
1162 }
1163 
1164 bool AMDGPUOperand::isInlinableImm(MVT type) const {
1165   if (!isImmTy(ImmTyNone)) {
1166     // Only plain immediates are inlinable (e.g. "clamp" attribute is not)
1167     return false;
1168   }
1169   // TODO: We should avoid using host float here. It would be better to
1170   // check the float bit values which is what a few other places do.
1171   // We've had bot failures before due to weird NaN support on mips hosts.
1172 
1173   APInt Literal(64, Imm.Val);
1174 
1175   if (Imm.IsFPImm) { // We got fp literal token
1176     if (type == MVT::f64 || type == MVT::i64) { // Expected 64-bit operand
1177       return AMDGPU::isInlinableLiteral64(Imm.Val,
1178                                           AsmParser->hasInv2PiInlineImm());
1179     }
1180 
1181     APFloat FPLiteral(APFloat::IEEEdouble(), APInt(64, Imm.Val));
1182     if (!canLosslesslyConvertToFPType(FPLiteral, type))
1183       return false;
1184 
1185     if (type.getScalarSizeInBits() == 16) {
1186       return AMDGPU::isInlinableLiteral16(
1187         static_cast<int16_t>(FPLiteral.bitcastToAPInt().getZExtValue()),
1188         AsmParser->hasInv2PiInlineImm());
1189     }
1190 
1191     // Check if single precision literal is inlinable
1192     return AMDGPU::isInlinableLiteral32(
1193       static_cast<int32_t>(FPLiteral.bitcastToAPInt().getZExtValue()),
1194       AsmParser->hasInv2PiInlineImm());
1195   }
1196 
1197   // We got int literal token.
1198   if (type == MVT::f64 || type == MVT::i64) { // Expected 64-bit operand
1199     return AMDGPU::isInlinableLiteral64(Imm.Val,
1200                                         AsmParser->hasInv2PiInlineImm());
1201   }
1202 
1203   if (type.getScalarSizeInBits() == 16) {
1204     return AMDGPU::isInlinableLiteral16(
1205       static_cast<int16_t>(Literal.getLoBits(16).getSExtValue()),
1206       AsmParser->hasInv2PiInlineImm());
1207   }
1208 
1209   return AMDGPU::isInlinableLiteral32(
1210     static_cast<int32_t>(Literal.getLoBits(32).getZExtValue()),
1211     AsmParser->hasInv2PiInlineImm());
1212 }
1213 
1214 bool AMDGPUOperand::isLiteralImm(MVT type) const {
1215   // Check that this imediate can be added as literal
1216   if (!isImmTy(ImmTyNone)) {
1217     return false;
1218   }
1219 
1220   if (!Imm.IsFPImm) {
1221     // We got int literal token.
1222 
1223     if (type == MVT::f64 && hasFPModifiers()) {
1224       // Cannot apply fp modifiers to int literals preserving the same semantics
1225       // for VOP1/2/C and VOP3 because of integer truncation. To avoid ambiguity,
1226       // disable these cases.
1227       return false;
1228     }
1229 
1230     unsigned Size = type.getSizeInBits();
1231     if (Size == 64)
1232       Size = 32;
1233 
1234     // FIXME: 64-bit operands can zero extend, sign extend, or pad zeroes for FP
1235     // types.
1236     return isUIntN(Size, Imm.Val) || isIntN(Size, Imm.Val);
1237   }
1238 
1239   // We got fp literal token
1240   if (type == MVT::f64) { // Expected 64-bit fp operand
1241     // We would set low 64-bits of literal to zeroes but we accept this literals
1242     return true;
1243   }
1244 
1245   if (type == MVT::i64) { // Expected 64-bit int operand
1246     // We don't allow fp literals in 64-bit integer instructions. It is
1247     // unclear how we should encode them.
1248     return false;
1249   }
1250 
1251   APFloat FPLiteral(APFloat::IEEEdouble(), APInt(64, Imm.Val));
1252   return canLosslesslyConvertToFPType(FPLiteral, type);
1253 }
1254 
1255 bool AMDGPUOperand::isRegClass(unsigned RCID) const {
1256   return isRegKind() && AsmParser->getMRI()->getRegClass(RCID).contains(getReg());
1257 }
1258 
1259 bool AMDGPUOperand::isSDWARegKind() const {
1260   if (AsmParser->isVI())
1261     return isVReg();
1262   else if (AsmParser->isGFX9())
1263     return isRegKind();
1264   else
1265     return false;
1266 }
1267 
1268 uint64_t AMDGPUOperand::applyInputFPModifiers(uint64_t Val, unsigned Size) const
1269 {
1270   assert(isImmTy(ImmTyNone) && Imm.Mods.hasFPModifiers());
1271   assert(Size == 2 || Size == 4 || Size == 8);
1272 
1273   const uint64_t FpSignMask = (1ULL << (Size * 8 - 1));
1274 
1275   if (Imm.Mods.Abs) {
1276     Val &= ~FpSignMask;
1277   }
1278   if (Imm.Mods.Neg) {
1279     Val ^= FpSignMask;
1280   }
1281 
1282   return Val;
1283 }
1284 
1285 void AMDGPUOperand::addImmOperands(MCInst &Inst, unsigned N, bool ApplyModifiers) const {
1286 
1287   if (AMDGPU::isSISrcOperand(AsmParser->getMII()->get(Inst.getOpcode()),
1288                              Inst.getNumOperands())) {
1289     addLiteralImmOperand(Inst, Imm.Val,
1290                          ApplyModifiers &
1291                          isImmTy(ImmTyNone) && Imm.Mods.hasFPModifiers());
1292   } else {
1293     assert(!isImmTy(ImmTyNone) || !hasModifiers());
1294     Inst.addOperand(MCOperand::createImm(Imm.Val));
1295   }
1296 }
1297 
1298 void AMDGPUOperand::addLiteralImmOperand(MCInst &Inst, int64_t Val, bool ApplyModifiers) const {
1299   const auto& InstDesc = AsmParser->getMII()->get(Inst.getOpcode());
1300   auto OpNum = Inst.getNumOperands();
1301   // Check that this operand accepts literals
1302   assert(AMDGPU::isSISrcOperand(InstDesc, OpNum));
1303 
1304   if (ApplyModifiers) {
1305     assert(AMDGPU::isSISrcFPOperand(InstDesc, OpNum));
1306     const unsigned Size = Imm.IsFPImm ? sizeof(double) : getOperandSize(InstDesc, OpNum);
1307     Val = applyInputFPModifiers(Val, Size);
1308   }
1309 
1310   APInt Literal(64, Val);
1311   uint8_t OpTy = InstDesc.OpInfo[OpNum].OperandType;
1312 
1313   if (Imm.IsFPImm) { // We got fp literal token
1314     switch (OpTy) {
1315     case AMDGPU::OPERAND_REG_IMM_INT64:
1316     case AMDGPU::OPERAND_REG_IMM_FP64:
1317     case AMDGPU::OPERAND_REG_INLINE_C_INT64:
1318     case AMDGPU::OPERAND_REG_INLINE_C_FP64: {
1319       if (AMDGPU::isInlinableLiteral64(Literal.getZExtValue(),
1320                                        AsmParser->hasInv2PiInlineImm())) {
1321         Inst.addOperand(MCOperand::createImm(Literal.getZExtValue()));
1322         return;
1323       }
1324 
1325       // Non-inlineable
1326       if (AMDGPU::isSISrcFPOperand(InstDesc, OpNum)) { // Expected 64-bit fp operand
1327         // For fp operands we check if low 32 bits are zeros
1328         if (Literal.getLoBits(32) != 0) {
1329           const_cast<AMDGPUAsmParser *>(AsmParser)->Warning(Inst.getLoc(),
1330           "Can't encode literal as exact 64-bit floating-point operand. "
1331           "Low 32-bits will be set to zero");
1332         }
1333 
1334         Inst.addOperand(MCOperand::createImm(Literal.lshr(32).getZExtValue()));
1335         return;
1336       }
1337 
1338       // We don't allow fp literals in 64-bit integer instructions. It is
1339       // unclear how we should encode them. This case should be checked earlier
1340       // in predicate methods (isLiteralImm())
1341       llvm_unreachable("fp literal in 64-bit integer instruction.");
1342     }
1343     case AMDGPU::OPERAND_REG_IMM_INT32:
1344     case AMDGPU::OPERAND_REG_IMM_FP32:
1345     case AMDGPU::OPERAND_REG_INLINE_C_INT32:
1346     case AMDGPU::OPERAND_REG_INLINE_C_FP32:
1347     case AMDGPU::OPERAND_REG_IMM_INT16:
1348     case AMDGPU::OPERAND_REG_IMM_FP16:
1349     case AMDGPU::OPERAND_REG_INLINE_C_INT16:
1350     case AMDGPU::OPERAND_REG_INLINE_C_FP16:
1351     case AMDGPU::OPERAND_REG_INLINE_C_V2INT16:
1352     case AMDGPU::OPERAND_REG_INLINE_C_V2FP16: {
1353       bool lost;
1354       APFloat FPLiteral(APFloat::IEEEdouble(), Literal);
1355       // Convert literal to single precision
1356       FPLiteral.convert(*getOpFltSemantics(OpTy),
1357                         APFloat::rmNearestTiesToEven, &lost);
1358       // We allow precision lost but not overflow or underflow. This should be
1359       // checked earlier in isLiteralImm()
1360 
1361       uint64_t ImmVal = FPLiteral.bitcastToAPInt().getZExtValue();
1362       if (OpTy == AMDGPU::OPERAND_REG_INLINE_C_V2INT16 ||
1363           OpTy == AMDGPU::OPERAND_REG_INLINE_C_V2FP16) {
1364         ImmVal |= (ImmVal << 16);
1365       }
1366 
1367       Inst.addOperand(MCOperand::createImm(ImmVal));
1368       return;
1369     }
1370     default:
1371       llvm_unreachable("invalid operand size");
1372     }
1373 
1374     return;
1375   }
1376 
1377    // We got int literal token.
1378   // Only sign extend inline immediates.
1379   // FIXME: No errors on truncation
1380   switch (OpTy) {
1381   case AMDGPU::OPERAND_REG_IMM_INT32:
1382   case AMDGPU::OPERAND_REG_IMM_FP32:
1383   case AMDGPU::OPERAND_REG_INLINE_C_INT32:
1384   case AMDGPU::OPERAND_REG_INLINE_C_FP32: {
1385     if (isInt<32>(Val) &&
1386         AMDGPU::isInlinableLiteral32(static_cast<int32_t>(Val),
1387                                      AsmParser->hasInv2PiInlineImm())) {
1388       Inst.addOperand(MCOperand::createImm(Val));
1389       return;
1390     }
1391 
1392     Inst.addOperand(MCOperand::createImm(Val & 0xffffffff));
1393     return;
1394   }
1395   case AMDGPU::OPERAND_REG_IMM_INT64:
1396   case AMDGPU::OPERAND_REG_IMM_FP64:
1397   case AMDGPU::OPERAND_REG_INLINE_C_INT64:
1398   case AMDGPU::OPERAND_REG_INLINE_C_FP64: {
1399     if (AMDGPU::isInlinableLiteral64(Val, AsmParser->hasInv2PiInlineImm())) {
1400       Inst.addOperand(MCOperand::createImm(Val));
1401       return;
1402     }
1403 
1404     Inst.addOperand(MCOperand::createImm(Lo_32(Val)));
1405     return;
1406   }
1407   case AMDGPU::OPERAND_REG_IMM_INT16:
1408   case AMDGPU::OPERAND_REG_IMM_FP16:
1409   case AMDGPU::OPERAND_REG_INLINE_C_INT16:
1410   case AMDGPU::OPERAND_REG_INLINE_C_FP16: {
1411     if (isInt<16>(Val) &&
1412         AMDGPU::isInlinableLiteral16(static_cast<int16_t>(Val),
1413                                      AsmParser->hasInv2PiInlineImm())) {
1414       Inst.addOperand(MCOperand::createImm(Val));
1415       return;
1416     }
1417 
1418     Inst.addOperand(MCOperand::createImm(Val & 0xffff));
1419     return;
1420   }
1421   case AMDGPU::OPERAND_REG_INLINE_C_V2INT16:
1422   case AMDGPU::OPERAND_REG_INLINE_C_V2FP16: {
1423     auto LiteralVal = static_cast<uint16_t>(Literal.getLoBits(16).getZExtValue());
1424     assert(AMDGPU::isInlinableLiteral16(LiteralVal,
1425                                         AsmParser->hasInv2PiInlineImm()));
1426 
1427     uint32_t ImmVal = static_cast<uint32_t>(LiteralVal) << 16 |
1428                       static_cast<uint32_t>(LiteralVal);
1429     Inst.addOperand(MCOperand::createImm(ImmVal));
1430     return;
1431   }
1432   default:
1433     llvm_unreachable("invalid operand size");
1434   }
1435 }
1436 
1437 template <unsigned Bitwidth>
1438 void AMDGPUOperand::addKImmFPOperands(MCInst &Inst, unsigned N) const {
1439   APInt Literal(64, Imm.Val);
1440 
1441   if (!Imm.IsFPImm) {
1442     // We got int literal token.
1443     Inst.addOperand(MCOperand::createImm(Literal.getLoBits(Bitwidth).getZExtValue()));
1444     return;
1445   }
1446 
1447   bool Lost;
1448   APFloat FPLiteral(APFloat::IEEEdouble(), Literal);
1449   FPLiteral.convert(*getFltSemantics(Bitwidth / 8),
1450                     APFloat::rmNearestTiesToEven, &Lost);
1451   Inst.addOperand(MCOperand::createImm(FPLiteral.bitcastToAPInt().getZExtValue()));
1452 }
1453 
1454 void AMDGPUOperand::addRegOperands(MCInst &Inst, unsigned N) const {
1455   Inst.addOperand(MCOperand::createReg(AMDGPU::getMCReg(getReg(), AsmParser->getSTI())));
1456 }
1457 
1458 //===----------------------------------------------------------------------===//
1459 // AsmParser
1460 //===----------------------------------------------------------------------===//
1461 
1462 static int getRegClass(RegisterKind Is, unsigned RegWidth) {
1463   if (Is == IS_VGPR) {
1464     switch (RegWidth) {
1465       default: return -1;
1466       case 1: return AMDGPU::VGPR_32RegClassID;
1467       case 2: return AMDGPU::VReg_64RegClassID;
1468       case 3: return AMDGPU::VReg_96RegClassID;
1469       case 4: return AMDGPU::VReg_128RegClassID;
1470       case 8: return AMDGPU::VReg_256RegClassID;
1471       case 16: return AMDGPU::VReg_512RegClassID;
1472     }
1473   } else if (Is == IS_TTMP) {
1474     switch (RegWidth) {
1475       default: return -1;
1476       case 1: return AMDGPU::TTMP_32RegClassID;
1477       case 2: return AMDGPU::TTMP_64RegClassID;
1478       case 4: return AMDGPU::TTMP_128RegClassID;
1479     }
1480   } else if (Is == IS_SGPR) {
1481     switch (RegWidth) {
1482       default: return -1;
1483       case 1: return AMDGPU::SGPR_32RegClassID;
1484       case 2: return AMDGPU::SGPR_64RegClassID;
1485       case 4: return AMDGPU::SGPR_128RegClassID;
1486       case 8: return AMDGPU::SReg_256RegClassID;
1487       case 16: return AMDGPU::SReg_512RegClassID;
1488     }
1489   }
1490   return -1;
1491 }
1492 
1493 static unsigned getSpecialRegForName(StringRef RegName) {
1494   return StringSwitch<unsigned>(RegName)
1495     .Case("exec", AMDGPU::EXEC)
1496     .Case("vcc", AMDGPU::VCC)
1497     .Case("flat_scratch", AMDGPU::FLAT_SCR)
1498     .Case("m0", AMDGPU::M0)
1499     .Case("scc", AMDGPU::SCC)
1500     .Case("tba", AMDGPU::TBA)
1501     .Case("tma", AMDGPU::TMA)
1502     .Case("flat_scratch_lo", AMDGPU::FLAT_SCR_LO)
1503     .Case("flat_scratch_hi", AMDGPU::FLAT_SCR_HI)
1504     .Case("vcc_lo", AMDGPU::VCC_LO)
1505     .Case("vcc_hi", AMDGPU::VCC_HI)
1506     .Case("exec_lo", AMDGPU::EXEC_LO)
1507     .Case("exec_hi", AMDGPU::EXEC_HI)
1508     .Case("tma_lo", AMDGPU::TMA_LO)
1509     .Case("tma_hi", AMDGPU::TMA_HI)
1510     .Case("tba_lo", AMDGPU::TBA_LO)
1511     .Case("tba_hi", AMDGPU::TBA_HI)
1512     .Default(0);
1513 }
1514 
1515 bool AMDGPUAsmParser::ParseRegister(unsigned &RegNo, SMLoc &StartLoc,
1516                                     SMLoc &EndLoc) {
1517   auto R = parseRegister();
1518   if (!R) return true;
1519   assert(R->isReg());
1520   RegNo = R->getReg();
1521   StartLoc = R->getStartLoc();
1522   EndLoc = R->getEndLoc();
1523   return false;
1524 }
1525 
1526 bool AMDGPUAsmParser::AddNextRegisterToList(unsigned &Reg, unsigned &RegWidth,
1527                                             RegisterKind RegKind, unsigned Reg1,
1528                                             unsigned RegNum) {
1529   switch (RegKind) {
1530   case IS_SPECIAL:
1531     if (Reg == AMDGPU::EXEC_LO && Reg1 == AMDGPU::EXEC_HI) {
1532       Reg = AMDGPU::EXEC;
1533       RegWidth = 2;
1534       return true;
1535     }
1536     if (Reg == AMDGPU::FLAT_SCR_LO && Reg1 == AMDGPU::FLAT_SCR_HI) {
1537       Reg = AMDGPU::FLAT_SCR;
1538       RegWidth = 2;
1539       return true;
1540     }
1541     if (Reg == AMDGPU::VCC_LO && Reg1 == AMDGPU::VCC_HI) {
1542       Reg = AMDGPU::VCC;
1543       RegWidth = 2;
1544       return true;
1545     }
1546     if (Reg == AMDGPU::TBA_LO && Reg1 == AMDGPU::TBA_HI) {
1547       Reg = AMDGPU::TBA;
1548       RegWidth = 2;
1549       return true;
1550     }
1551     if (Reg == AMDGPU::TMA_LO && Reg1 == AMDGPU::TMA_HI) {
1552       Reg = AMDGPU::TMA;
1553       RegWidth = 2;
1554       return true;
1555     }
1556     return false;
1557   case IS_VGPR:
1558   case IS_SGPR:
1559   case IS_TTMP:
1560     if (Reg1 != Reg + RegWidth) {
1561       return false;
1562     }
1563     RegWidth++;
1564     return true;
1565   default:
1566     llvm_unreachable("unexpected register kind");
1567   }
1568 }
1569 
1570 bool AMDGPUAsmParser::ParseAMDGPURegister(RegisterKind &RegKind, unsigned &Reg,
1571                                           unsigned &RegNum, unsigned &RegWidth,
1572                                           unsigned *DwordRegIndex) {
1573   if (DwordRegIndex) { *DwordRegIndex = 0; }
1574   const MCRegisterInfo *TRI = getContext().getRegisterInfo();
1575   if (getLexer().is(AsmToken::Identifier)) {
1576     StringRef RegName = Parser.getTok().getString();
1577     if ((Reg = getSpecialRegForName(RegName))) {
1578       Parser.Lex();
1579       RegKind = IS_SPECIAL;
1580     } else {
1581       unsigned RegNumIndex = 0;
1582       if (RegName[0] == 'v') {
1583         RegNumIndex = 1;
1584         RegKind = IS_VGPR;
1585       } else if (RegName[0] == 's') {
1586         RegNumIndex = 1;
1587         RegKind = IS_SGPR;
1588       } else if (RegName.startswith("ttmp")) {
1589         RegNumIndex = strlen("ttmp");
1590         RegKind = IS_TTMP;
1591       } else {
1592         return false;
1593       }
1594       if (RegName.size() > RegNumIndex) {
1595         // Single 32-bit register: vXX.
1596         if (RegName.substr(RegNumIndex).getAsInteger(10, RegNum))
1597           return false;
1598         Parser.Lex();
1599         RegWidth = 1;
1600       } else {
1601         // Range of registers: v[XX:YY]. ":YY" is optional.
1602         Parser.Lex();
1603         int64_t RegLo, RegHi;
1604         if (getLexer().isNot(AsmToken::LBrac))
1605           return false;
1606         Parser.Lex();
1607 
1608         if (getParser().parseAbsoluteExpression(RegLo))
1609           return false;
1610 
1611         const bool isRBrace = getLexer().is(AsmToken::RBrac);
1612         if (!isRBrace && getLexer().isNot(AsmToken::Colon))
1613           return false;
1614         Parser.Lex();
1615 
1616         if (isRBrace) {
1617           RegHi = RegLo;
1618         } else {
1619           if (getParser().parseAbsoluteExpression(RegHi))
1620             return false;
1621 
1622           if (getLexer().isNot(AsmToken::RBrac))
1623             return false;
1624           Parser.Lex();
1625         }
1626         RegNum = (unsigned) RegLo;
1627         RegWidth = (RegHi - RegLo) + 1;
1628       }
1629     }
1630   } else if (getLexer().is(AsmToken::LBrac)) {
1631     // List of consecutive registers: [s0,s1,s2,s3]
1632     Parser.Lex();
1633     if (!ParseAMDGPURegister(RegKind, Reg, RegNum, RegWidth, nullptr))
1634       return false;
1635     if (RegWidth != 1)
1636       return false;
1637     RegisterKind RegKind1;
1638     unsigned Reg1, RegNum1, RegWidth1;
1639     do {
1640       if (getLexer().is(AsmToken::Comma)) {
1641         Parser.Lex();
1642       } else if (getLexer().is(AsmToken::RBrac)) {
1643         Parser.Lex();
1644         break;
1645       } else if (ParseAMDGPURegister(RegKind1, Reg1, RegNum1, RegWidth1, nullptr)) {
1646         if (RegWidth1 != 1) {
1647           return false;
1648         }
1649         if (RegKind1 != RegKind) {
1650           return false;
1651         }
1652         if (!AddNextRegisterToList(Reg, RegWidth, RegKind1, Reg1, RegNum1)) {
1653           return false;
1654         }
1655       } else {
1656         return false;
1657       }
1658     } while (true);
1659   } else {
1660     return false;
1661   }
1662   switch (RegKind) {
1663   case IS_SPECIAL:
1664     RegNum = 0;
1665     RegWidth = 1;
1666     break;
1667   case IS_VGPR:
1668   case IS_SGPR:
1669   case IS_TTMP:
1670   {
1671     unsigned Size = 1;
1672     if (RegKind == IS_SGPR || RegKind == IS_TTMP) {
1673       // SGPR and TTMP registers must be aligned. Max required alignment is 4 dwords.
1674       Size = std::min(RegWidth, 4u);
1675     }
1676     if (RegNum % Size != 0)
1677       return false;
1678     if (DwordRegIndex) { *DwordRegIndex = RegNum; }
1679     RegNum = RegNum / Size;
1680     int RCID = getRegClass(RegKind, RegWidth);
1681     if (RCID == -1)
1682       return false;
1683     const MCRegisterClass RC = TRI->getRegClass(RCID);
1684     if (RegNum >= RC.getNumRegs())
1685       return false;
1686     Reg = RC.getRegister(RegNum);
1687     break;
1688   }
1689 
1690   default:
1691     llvm_unreachable("unexpected register kind");
1692   }
1693 
1694   if (!subtargetHasRegister(*TRI, Reg))
1695     return false;
1696   return true;
1697 }
1698 
1699 std::unique_ptr<AMDGPUOperand> AMDGPUAsmParser::parseRegister() {
1700   const auto &Tok = Parser.getTok();
1701   SMLoc StartLoc = Tok.getLoc();
1702   SMLoc EndLoc = Tok.getEndLoc();
1703   RegisterKind RegKind;
1704   unsigned Reg, RegNum, RegWidth, DwordRegIndex;
1705 
1706   if (!ParseAMDGPURegister(RegKind, Reg, RegNum, RegWidth, &DwordRegIndex)) {
1707     return nullptr;
1708   }
1709   KernelScope.usesRegister(RegKind, DwordRegIndex, RegWidth);
1710   return AMDGPUOperand::CreateReg(this, Reg, StartLoc, EndLoc, false);
1711 }
1712 
1713 bool
1714 AMDGPUAsmParser::parseAbsoluteExpr(int64_t &Val, bool AbsMod) {
1715   if (AbsMod && getLexer().peekTok().is(AsmToken::Pipe) &&
1716       (getLexer().getKind() == AsmToken::Integer ||
1717        getLexer().getKind() == AsmToken::Real)) {
1718 
1719     // This is a workaround for handling operands like these:
1720     //     |1.0|
1721     //     |-1|
1722     // This syntax is not compatible with syntax of standard
1723     // MC expressions (due to the trailing '|').
1724 
1725     SMLoc EndLoc;
1726     const MCExpr *Expr;
1727 
1728     if (getParser().parsePrimaryExpr(Expr, EndLoc)) {
1729       return true;
1730     }
1731 
1732     return !Expr->evaluateAsAbsolute(Val);
1733   }
1734 
1735   return getParser().parseAbsoluteExpression(Val);
1736 }
1737 
1738 OperandMatchResultTy
1739 AMDGPUAsmParser::parseImm(OperandVector &Operands, bool AbsMod) {
1740   // TODO: add syntactic sugar for 1/(2*PI)
1741   bool Minus = false;
1742   if (getLexer().getKind() == AsmToken::Minus) {
1743     Minus = true;
1744     Parser.Lex();
1745   }
1746 
1747   SMLoc S = Parser.getTok().getLoc();
1748   switch(getLexer().getKind()) {
1749   case AsmToken::Integer: {
1750     int64_t IntVal;
1751     if (parseAbsoluteExpr(IntVal, AbsMod))
1752       return MatchOperand_ParseFail;
1753     if (Minus)
1754       IntVal *= -1;
1755     Operands.push_back(AMDGPUOperand::CreateImm(this, IntVal, S));
1756     return MatchOperand_Success;
1757   }
1758   case AsmToken::Real: {
1759     int64_t IntVal;
1760     if (parseAbsoluteExpr(IntVal, AbsMod))
1761       return MatchOperand_ParseFail;
1762 
1763     APFloat F(BitsToDouble(IntVal));
1764     if (Minus)
1765       F.changeSign();
1766     Operands.push_back(
1767         AMDGPUOperand::CreateImm(this, F.bitcastToAPInt().getZExtValue(), S,
1768                                  AMDGPUOperand::ImmTyNone, true));
1769     return MatchOperand_Success;
1770   }
1771   default:
1772     return Minus ? MatchOperand_ParseFail : MatchOperand_NoMatch;
1773   }
1774 }
1775 
1776 OperandMatchResultTy
1777 AMDGPUAsmParser::parseReg(OperandVector &Operands) {
1778   if (auto R = parseRegister()) {
1779     assert(R->isReg());
1780     R->Reg.IsForcedVOP3 = isForcedVOP3();
1781     Operands.push_back(std::move(R));
1782     return MatchOperand_Success;
1783   }
1784   return MatchOperand_NoMatch;
1785 }
1786 
1787 OperandMatchResultTy
1788 AMDGPUAsmParser::parseRegOrImm(OperandVector &Operands, bool AbsMod) {
1789   auto res = parseImm(Operands, AbsMod);
1790   if (res != MatchOperand_NoMatch) {
1791     return res;
1792   }
1793 
1794   return parseReg(Operands);
1795 }
1796 
1797 OperandMatchResultTy
1798 AMDGPUAsmParser::parseRegOrImmWithFPInputMods(OperandVector &Operands,
1799                                               bool AllowImm) {
1800   bool Negate = false, Negate2 = false, Abs = false, Abs2 = false;
1801 
1802   if (getLexer().getKind()== AsmToken::Minus) {
1803     const AsmToken NextToken = getLexer().peekTok();
1804 
1805     // Disable ambiguous constructs like '--1' etc. Should use neg(-1) instead.
1806     if (NextToken.is(AsmToken::Minus)) {
1807       Error(Parser.getTok().getLoc(), "invalid syntax, expected 'neg' modifier");
1808       return MatchOperand_ParseFail;
1809     }
1810 
1811     // '-' followed by an integer literal N should be interpreted as integer
1812     // negation rather than a floating-point NEG modifier applied to N.
1813     // Beside being contr-intuitive, such use of floating-point NEG modifier
1814     // results in different meaning of integer literals used with VOP1/2/C
1815     // and VOP3, for example:
1816     //    v_exp_f32_e32 v5, -1 // VOP1: src0 = 0xFFFFFFFF
1817     //    v_exp_f32_e64 v5, -1 // VOP3: src0 = 0x80000001
1818     // Negative fp literals should be handled likewise for unifomtity
1819     if (!NextToken.is(AsmToken::Integer) && !NextToken.is(AsmToken::Real)) {
1820       Parser.Lex();
1821       Negate = true;
1822     }
1823   }
1824 
1825   if (getLexer().getKind() == AsmToken::Identifier &&
1826       Parser.getTok().getString() == "neg") {
1827     if (Negate) {
1828       Error(Parser.getTok().getLoc(), "expected register or immediate");
1829       return MatchOperand_ParseFail;
1830     }
1831     Parser.Lex();
1832     Negate2 = true;
1833     if (getLexer().isNot(AsmToken::LParen)) {
1834       Error(Parser.getTok().getLoc(), "expected left paren after neg");
1835       return MatchOperand_ParseFail;
1836     }
1837     Parser.Lex();
1838   }
1839 
1840   if (getLexer().getKind() == AsmToken::Identifier &&
1841       Parser.getTok().getString() == "abs") {
1842     Parser.Lex();
1843     Abs2 = true;
1844     if (getLexer().isNot(AsmToken::LParen)) {
1845       Error(Parser.getTok().getLoc(), "expected left paren after abs");
1846       return MatchOperand_ParseFail;
1847     }
1848     Parser.Lex();
1849   }
1850 
1851   if (getLexer().getKind() == AsmToken::Pipe) {
1852     if (Abs2) {
1853       Error(Parser.getTok().getLoc(), "expected register or immediate");
1854       return MatchOperand_ParseFail;
1855     }
1856     Parser.Lex();
1857     Abs = true;
1858   }
1859 
1860   OperandMatchResultTy Res;
1861   if (AllowImm) {
1862     Res = parseRegOrImm(Operands, Abs);
1863   } else {
1864     Res = parseReg(Operands);
1865   }
1866   if (Res != MatchOperand_Success) {
1867     return Res;
1868   }
1869 
1870   AMDGPUOperand::Modifiers Mods;
1871   if (Abs) {
1872     if (getLexer().getKind() != AsmToken::Pipe) {
1873       Error(Parser.getTok().getLoc(), "expected vertical bar");
1874       return MatchOperand_ParseFail;
1875     }
1876     Parser.Lex();
1877     Mods.Abs = true;
1878   }
1879   if (Abs2) {
1880     if (getLexer().isNot(AsmToken::RParen)) {
1881       Error(Parser.getTok().getLoc(), "expected closing parentheses");
1882       return MatchOperand_ParseFail;
1883     }
1884     Parser.Lex();
1885     Mods.Abs = true;
1886   }
1887 
1888   if (Negate) {
1889     Mods.Neg = true;
1890   } else if (Negate2) {
1891     if (getLexer().isNot(AsmToken::RParen)) {
1892       Error(Parser.getTok().getLoc(), "expected closing parentheses");
1893       return MatchOperand_ParseFail;
1894     }
1895     Parser.Lex();
1896     Mods.Neg = true;
1897   }
1898 
1899   if (Mods.hasFPModifiers()) {
1900     AMDGPUOperand &Op = static_cast<AMDGPUOperand &>(*Operands.back());
1901     Op.setModifiers(Mods);
1902   }
1903   return MatchOperand_Success;
1904 }
1905 
1906 OperandMatchResultTy
1907 AMDGPUAsmParser::parseRegOrImmWithIntInputMods(OperandVector &Operands,
1908                                                bool AllowImm) {
1909   bool Sext = false;
1910 
1911   if (getLexer().getKind() == AsmToken::Identifier &&
1912       Parser.getTok().getString() == "sext") {
1913     Parser.Lex();
1914     Sext = true;
1915     if (getLexer().isNot(AsmToken::LParen)) {
1916       Error(Parser.getTok().getLoc(), "expected left paren after sext");
1917       return MatchOperand_ParseFail;
1918     }
1919     Parser.Lex();
1920   }
1921 
1922   OperandMatchResultTy Res;
1923   if (AllowImm) {
1924     Res = parseRegOrImm(Operands);
1925   } else {
1926     Res = parseReg(Operands);
1927   }
1928   if (Res != MatchOperand_Success) {
1929     return Res;
1930   }
1931 
1932   AMDGPUOperand::Modifiers Mods;
1933   if (Sext) {
1934     if (getLexer().isNot(AsmToken::RParen)) {
1935       Error(Parser.getTok().getLoc(), "expected closing parentheses");
1936       return MatchOperand_ParseFail;
1937     }
1938     Parser.Lex();
1939     Mods.Sext = true;
1940   }
1941 
1942   if (Mods.hasIntModifiers()) {
1943     AMDGPUOperand &Op = static_cast<AMDGPUOperand &>(*Operands.back());
1944     Op.setModifiers(Mods);
1945   }
1946 
1947   return MatchOperand_Success;
1948 }
1949 
1950 OperandMatchResultTy
1951 AMDGPUAsmParser::parseRegWithFPInputMods(OperandVector &Operands) {
1952   return parseRegOrImmWithFPInputMods(Operands, false);
1953 }
1954 
1955 OperandMatchResultTy
1956 AMDGPUAsmParser::parseRegWithIntInputMods(OperandVector &Operands) {
1957   return parseRegOrImmWithIntInputMods(Operands, false);
1958 }
1959 
1960 OperandMatchResultTy AMDGPUAsmParser::parseVReg32OrOff(OperandVector &Operands) {
1961   std::unique_ptr<AMDGPUOperand> Reg = parseRegister();
1962   if (Reg) {
1963     Operands.push_back(std::move(Reg));
1964     return MatchOperand_Success;
1965   }
1966 
1967   const AsmToken &Tok = Parser.getTok();
1968   if (Tok.getString() == "off") {
1969     Operands.push_back(AMDGPUOperand::CreateImm(this, 0, Tok.getLoc(),
1970                                                 AMDGPUOperand::ImmTyOff, false));
1971     Parser.Lex();
1972     return MatchOperand_Success;
1973   }
1974 
1975   return MatchOperand_NoMatch;
1976 }
1977 
1978 unsigned AMDGPUAsmParser::checkTargetMatchPredicate(MCInst &Inst) {
1979   uint64_t TSFlags = MII.get(Inst.getOpcode()).TSFlags;
1980 
1981   if ((getForcedEncodingSize() == 32 && (TSFlags & SIInstrFlags::VOP3)) ||
1982       (getForcedEncodingSize() == 64 && !(TSFlags & SIInstrFlags::VOP3)) ||
1983       (isForcedDPP() && !(TSFlags & SIInstrFlags::DPP)) ||
1984       (isForcedSDWA() && !(TSFlags & SIInstrFlags::SDWA)) )
1985     return Match_InvalidOperand;
1986 
1987   if ((TSFlags & SIInstrFlags::VOP3) &&
1988       (TSFlags & SIInstrFlags::VOPAsmPrefer32Bit) &&
1989       getForcedEncodingSize() != 64)
1990     return Match_PreferE32;
1991 
1992   if (Inst.getOpcode() == AMDGPU::V_MAC_F32_sdwa_vi ||
1993       Inst.getOpcode() == AMDGPU::V_MAC_F16_sdwa_vi) {
1994     // v_mac_f32/16 allow only dst_sel == DWORD;
1995     auto OpNum =
1996         AMDGPU::getNamedOperandIdx(Inst.getOpcode(), AMDGPU::OpName::dst_sel);
1997     const auto &Op = Inst.getOperand(OpNum);
1998     if (!Op.isImm() || Op.getImm() != AMDGPU::SDWA::SdwaSel::DWORD) {
1999       return Match_InvalidOperand;
2000     }
2001   }
2002 
2003   if ((TSFlags & SIInstrFlags::FLAT) && !hasFlatOffsets()) {
2004     // FIXME: Produces error without correct column reported.
2005     auto OpNum =
2006         AMDGPU::getNamedOperandIdx(Inst.getOpcode(), AMDGPU::OpName::offset);
2007     const auto &Op = Inst.getOperand(OpNum);
2008     if (Op.getImm() != 0)
2009       return Match_InvalidOperand;
2010   }
2011 
2012   return Match_Success;
2013 }
2014 
2015 // What asm variants we should check
2016 ArrayRef<unsigned> AMDGPUAsmParser::getMatchedVariants() const {
2017   if (getForcedEncodingSize() == 32) {
2018     static const unsigned Variants[] = {AMDGPUAsmVariants::DEFAULT};
2019     return makeArrayRef(Variants);
2020   }
2021 
2022   if (isForcedVOP3()) {
2023     static const unsigned Variants[] = {AMDGPUAsmVariants::VOP3};
2024     return makeArrayRef(Variants);
2025   }
2026 
2027   if (isForcedSDWA()) {
2028     static const unsigned Variants[] = {AMDGPUAsmVariants::SDWA,
2029                                         AMDGPUAsmVariants::SDWA9};
2030     return makeArrayRef(Variants);
2031   }
2032 
2033   if (isForcedDPP()) {
2034     static const unsigned Variants[] = {AMDGPUAsmVariants::DPP};
2035     return makeArrayRef(Variants);
2036   }
2037 
2038   static const unsigned Variants[] = {
2039     AMDGPUAsmVariants::DEFAULT, AMDGPUAsmVariants::VOP3,
2040     AMDGPUAsmVariants::SDWA, AMDGPUAsmVariants::SDWA9, AMDGPUAsmVariants::DPP
2041   };
2042 
2043   return makeArrayRef(Variants);
2044 }
2045 
2046 unsigned AMDGPUAsmParser::findImplicitSGPRReadInVOP(const MCInst &Inst) const {
2047   const MCInstrDesc &Desc = MII.get(Inst.getOpcode());
2048   const unsigned Num = Desc.getNumImplicitUses();
2049   for (unsigned i = 0; i < Num; ++i) {
2050     unsigned Reg = Desc.ImplicitUses[i];
2051     switch (Reg) {
2052     case AMDGPU::FLAT_SCR:
2053     case AMDGPU::VCC:
2054     case AMDGPU::M0:
2055       return Reg;
2056     default:
2057       break;
2058     }
2059   }
2060   return AMDGPU::NoRegister;
2061 }
2062 
2063 // NB: This code is correct only when used to check constant
2064 // bus limitations because GFX7 support no f16 inline constants.
2065 // Note that there are no cases when a GFX7 opcode violates
2066 // constant bus limitations due to the use of an f16 constant.
2067 bool AMDGPUAsmParser::isInlineConstant(const MCInst &Inst,
2068                                        unsigned OpIdx) const {
2069   const MCInstrDesc &Desc = MII.get(Inst.getOpcode());
2070 
2071   if (!AMDGPU::isSISrcOperand(Desc, OpIdx)) {
2072     return false;
2073   }
2074 
2075   const MCOperand &MO = Inst.getOperand(OpIdx);
2076 
2077   int64_t Val = MO.getImm();
2078   auto OpSize = AMDGPU::getOperandSize(Desc, OpIdx);
2079 
2080   switch (OpSize) { // expected operand size
2081   case 8:
2082     return AMDGPU::isInlinableLiteral64(Val, hasInv2PiInlineImm());
2083   case 4:
2084     return AMDGPU::isInlinableLiteral32(Val, hasInv2PiInlineImm());
2085   case 2: {
2086     const unsigned OperandType = Desc.OpInfo[OpIdx].OperandType;
2087     if (OperandType == AMDGPU::OPERAND_REG_INLINE_C_V2INT16 ||
2088         OperandType == AMDGPU::OPERAND_REG_INLINE_C_V2FP16) {
2089       return AMDGPU::isInlinableLiteralV216(Val, hasInv2PiInlineImm());
2090     } else {
2091       return AMDGPU::isInlinableLiteral16(Val, hasInv2PiInlineImm());
2092     }
2093   }
2094   default:
2095     llvm_unreachable("invalid operand size");
2096   }
2097 }
2098 
2099 bool AMDGPUAsmParser::usesConstantBus(const MCInst &Inst, unsigned OpIdx) {
2100   const MCOperand &MO = Inst.getOperand(OpIdx);
2101   if (MO.isImm()) {
2102     return !isInlineConstant(Inst, OpIdx);
2103   }
2104   return !MO.isReg() ||
2105          isSGPR(mc2PseudoReg(MO.getReg()), getContext().getRegisterInfo());
2106 }
2107 
2108 bool AMDGPUAsmParser::validateConstantBusLimitations(const MCInst &Inst) {
2109   const unsigned Opcode = Inst.getOpcode();
2110   const MCInstrDesc &Desc = MII.get(Opcode);
2111   unsigned ConstantBusUseCount = 0;
2112 
2113   if (Desc.TSFlags &
2114       (SIInstrFlags::VOPC |
2115        SIInstrFlags::VOP1 | SIInstrFlags::VOP2 |
2116        SIInstrFlags::VOP3 | SIInstrFlags::VOP3P |
2117        SIInstrFlags::SDWA)) {
2118 
2119     // Check special imm operands (used by madmk, etc)
2120     if (AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::imm) != -1) {
2121       ++ConstantBusUseCount;
2122     }
2123 
2124     unsigned SGPRUsed = findImplicitSGPRReadInVOP(Inst);
2125     if (SGPRUsed != AMDGPU::NoRegister) {
2126       ++ConstantBusUseCount;
2127     }
2128 
2129     const int Src0Idx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::src0);
2130     const int Src1Idx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::src1);
2131     const int Src2Idx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::src2);
2132 
2133     const int OpIndices[] = { Src0Idx, Src1Idx, Src2Idx };
2134 
2135     for (int OpIdx : OpIndices) {
2136       if (OpIdx == -1) break;
2137 
2138       const MCOperand &MO = Inst.getOperand(OpIdx);
2139       if (usesConstantBus(Inst, OpIdx)) {
2140         if (MO.isReg()) {
2141           const unsigned Reg = mc2PseudoReg(MO.getReg());
2142           // Pairs of registers with a partial intersections like these
2143           //   s0, s[0:1]
2144           //   flat_scratch_lo, flat_scratch
2145           //   flat_scratch_lo, flat_scratch_hi
2146           // are theoretically valid but they are disabled anyway.
2147           // Note that this code mimics SIInstrInfo::verifyInstruction
2148           if (Reg != SGPRUsed) {
2149             ++ConstantBusUseCount;
2150           }
2151           SGPRUsed = Reg;
2152         } else { // Expression or a literal
2153           ++ConstantBusUseCount;
2154         }
2155       }
2156     }
2157   }
2158 
2159   return ConstantBusUseCount <= 1;
2160 }
2161 
2162 bool AMDGPUAsmParser::validateEarlyClobberLimitations(const MCInst &Inst) {
2163 
2164   const unsigned Opcode = Inst.getOpcode();
2165   const MCInstrDesc &Desc = MII.get(Opcode);
2166 
2167   const int DstIdx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::vdst);
2168   if (DstIdx == -1 ||
2169       Desc.getOperandConstraint(DstIdx, MCOI::EARLY_CLOBBER) == -1) {
2170     return true;
2171   }
2172 
2173   const MCRegisterInfo *TRI = getContext().getRegisterInfo();
2174 
2175   const int Src0Idx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::src0);
2176   const int Src1Idx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::src1);
2177   const int Src2Idx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::src2);
2178 
2179   assert(DstIdx != -1);
2180   const MCOperand &Dst = Inst.getOperand(DstIdx);
2181   assert(Dst.isReg());
2182   const unsigned DstReg = mc2PseudoReg(Dst.getReg());
2183 
2184   const int SrcIndices[] = { Src0Idx, Src1Idx, Src2Idx };
2185 
2186   for (int SrcIdx : SrcIndices) {
2187     if (SrcIdx == -1) break;
2188     const MCOperand &Src = Inst.getOperand(SrcIdx);
2189     if (Src.isReg()) {
2190       const unsigned SrcReg = mc2PseudoReg(Src.getReg());
2191       if (isRegIntersect(DstReg, SrcReg, TRI)) {
2192         return false;
2193       }
2194     }
2195   }
2196 
2197   return true;
2198 }
2199 
2200 bool AMDGPUAsmParser::validateInstruction(const MCInst &Inst,
2201                                           const SMLoc &IDLoc) {
2202   if (!validateConstantBusLimitations(Inst)) {
2203     Error(IDLoc,
2204       "invalid operand (violates constant bus restrictions)");
2205     return false;
2206   }
2207   if (!validateEarlyClobberLimitations(Inst)) {
2208     Error(IDLoc,
2209       "destination must be different than all sources");
2210     return false;
2211   }
2212 
2213   return true;
2214 }
2215 
2216 bool AMDGPUAsmParser::MatchAndEmitInstruction(SMLoc IDLoc, unsigned &Opcode,
2217                                               OperandVector &Operands,
2218                                               MCStreamer &Out,
2219                                               uint64_t &ErrorInfo,
2220                                               bool MatchingInlineAsm) {
2221   MCInst Inst;
2222   unsigned Result = Match_Success;
2223   for (auto Variant : getMatchedVariants()) {
2224     uint64_t EI;
2225     auto R = MatchInstructionImpl(Operands, Inst, EI, MatchingInlineAsm,
2226                                   Variant);
2227     // We order match statuses from least to most specific. We use most specific
2228     // status as resulting
2229     // Match_MnemonicFail < Match_InvalidOperand < Match_MissingFeature < Match_PreferE32
2230     if ((R == Match_Success) ||
2231         (R == Match_PreferE32) ||
2232         (R == Match_MissingFeature && Result != Match_PreferE32) ||
2233         (R == Match_InvalidOperand && Result != Match_MissingFeature
2234                                    && Result != Match_PreferE32) ||
2235         (R == Match_MnemonicFail   && Result != Match_InvalidOperand
2236                                    && Result != Match_MissingFeature
2237                                    && Result != Match_PreferE32)) {
2238       Result = R;
2239       ErrorInfo = EI;
2240     }
2241     if (R == Match_Success)
2242       break;
2243   }
2244 
2245   switch (Result) {
2246   default: break;
2247   case Match_Success:
2248     if (!validateInstruction(Inst, IDLoc)) {
2249       return true;
2250     }
2251     Inst.setLoc(IDLoc);
2252     Out.EmitInstruction(Inst, getSTI());
2253     return false;
2254 
2255   case Match_MissingFeature:
2256     return Error(IDLoc, "instruction not supported on this GPU");
2257 
2258   case Match_MnemonicFail:
2259     return Error(IDLoc, "unrecognized instruction mnemonic");
2260 
2261   case Match_InvalidOperand: {
2262     SMLoc ErrorLoc = IDLoc;
2263     if (ErrorInfo != ~0ULL) {
2264       if (ErrorInfo >= Operands.size()) {
2265         return Error(IDLoc, "too few operands for instruction");
2266       }
2267       ErrorLoc = ((AMDGPUOperand &)*Operands[ErrorInfo]).getStartLoc();
2268       if (ErrorLoc == SMLoc())
2269         ErrorLoc = IDLoc;
2270     }
2271     return Error(ErrorLoc, "invalid operand for instruction");
2272   }
2273 
2274   case Match_PreferE32:
2275     return Error(IDLoc, "internal error: instruction without _e64 suffix "
2276                         "should be encoded as e32");
2277   }
2278   llvm_unreachable("Implement any new match types added!");
2279 }
2280 
2281 bool AMDGPUAsmParser::ParseAsAbsoluteExpression(uint32_t &Ret) {
2282   int64_t Tmp = -1;
2283   if (getLexer().isNot(AsmToken::Integer) && getLexer().isNot(AsmToken::Identifier)) {
2284     return true;
2285   }
2286   if (getParser().parseAbsoluteExpression(Tmp)) {
2287     return true;
2288   }
2289   Ret = static_cast<uint32_t>(Tmp);
2290   return false;
2291 }
2292 
2293 bool AMDGPUAsmParser::ParseDirectiveMajorMinor(uint32_t &Major,
2294                                                uint32_t &Minor) {
2295   if (ParseAsAbsoluteExpression(Major))
2296     return TokError("invalid major version");
2297 
2298   if (getLexer().isNot(AsmToken::Comma))
2299     return TokError("minor version number required, comma expected");
2300   Lex();
2301 
2302   if (ParseAsAbsoluteExpression(Minor))
2303     return TokError("invalid minor version");
2304 
2305   return false;
2306 }
2307 
2308 bool AMDGPUAsmParser::ParseDirectiveHSACodeObjectVersion() {
2309   uint32_t Major;
2310   uint32_t Minor;
2311 
2312   if (ParseDirectiveMajorMinor(Major, Minor))
2313     return true;
2314 
2315   getTargetStreamer().EmitDirectiveHSACodeObjectVersion(Major, Minor);
2316   return false;
2317 }
2318 
2319 bool AMDGPUAsmParser::ParseDirectiveHSACodeObjectISA() {
2320   uint32_t Major;
2321   uint32_t Minor;
2322   uint32_t Stepping;
2323   StringRef VendorName;
2324   StringRef ArchName;
2325 
2326   // If this directive has no arguments, then use the ISA version for the
2327   // targeted GPU.
2328   if (getLexer().is(AsmToken::EndOfStatement)) {
2329     AMDGPU::IsaInfo::IsaVersion ISA =
2330         AMDGPU::IsaInfo::getIsaVersion(getFeatureBits());
2331     getTargetStreamer().EmitDirectiveHSACodeObjectISA(ISA.Major, ISA.Minor,
2332                                                       ISA.Stepping,
2333                                                       "AMD", "AMDGPU");
2334     return false;
2335   }
2336 
2337   if (ParseDirectiveMajorMinor(Major, Minor))
2338     return true;
2339 
2340   if (getLexer().isNot(AsmToken::Comma))
2341     return TokError("stepping version number required, comma expected");
2342   Lex();
2343 
2344   if (ParseAsAbsoluteExpression(Stepping))
2345     return TokError("invalid stepping version");
2346 
2347   if (getLexer().isNot(AsmToken::Comma))
2348     return TokError("vendor name required, comma expected");
2349   Lex();
2350 
2351   if (getLexer().isNot(AsmToken::String))
2352     return TokError("invalid vendor name");
2353 
2354   VendorName = getLexer().getTok().getStringContents();
2355   Lex();
2356 
2357   if (getLexer().isNot(AsmToken::Comma))
2358     return TokError("arch name required, comma expected");
2359   Lex();
2360 
2361   if (getLexer().isNot(AsmToken::String))
2362     return TokError("invalid arch name");
2363 
2364   ArchName = getLexer().getTok().getStringContents();
2365   Lex();
2366 
2367   getTargetStreamer().EmitDirectiveHSACodeObjectISA(Major, Minor, Stepping,
2368                                                     VendorName, ArchName);
2369   return false;
2370 }
2371 
2372 bool AMDGPUAsmParser::ParseDirectiveCodeObjectMetadata() {
2373   std::string YamlString;
2374   raw_string_ostream YamlStream(YamlString);
2375 
2376   getLexer().setSkipSpace(false);
2377 
2378   bool FoundEnd = false;
2379   while (!getLexer().is(AsmToken::Eof)) {
2380     while (getLexer().is(AsmToken::Space)) {
2381       YamlStream << getLexer().getTok().getString();
2382       Lex();
2383     }
2384 
2385     if (getLexer().is(AsmToken::Identifier)) {
2386       StringRef ID = getLexer().getTok().getIdentifier();
2387       if (ID == AMDGPU::CodeObject::MetadataAssemblerDirectiveEnd) {
2388         Lex();
2389         FoundEnd = true;
2390         break;
2391       }
2392     }
2393 
2394     YamlStream << Parser.parseStringToEndOfStatement()
2395                << getContext().getAsmInfo()->getSeparatorString();
2396 
2397     Parser.eatToEndOfStatement();
2398   }
2399 
2400   getLexer().setSkipSpace(true);
2401 
2402   if (getLexer().is(AsmToken::Eof) && !FoundEnd) {
2403     return TokError(
2404         "expected directive .end_amdgpu_code_object_metadata not found");
2405   }
2406 
2407   YamlStream.flush();
2408 
2409   if (!getTargetStreamer().EmitCodeObjectMetadata(YamlString))
2410     return Error(getParser().getTok().getLoc(), "invalid code object metadata");
2411 
2412   return false;
2413 }
2414 
2415 bool AMDGPUAsmParser::ParseAMDKernelCodeTValue(StringRef ID,
2416                                                amd_kernel_code_t &Header) {
2417   SmallString<40> ErrStr;
2418   raw_svector_ostream Err(ErrStr);
2419   if (!parseAmdKernelCodeField(ID, getParser(), Header, Err)) {
2420     return TokError(Err.str());
2421   }
2422   Lex();
2423   return false;
2424 }
2425 
2426 bool AMDGPUAsmParser::ParseDirectiveAMDKernelCodeT() {
2427   amd_kernel_code_t Header;
2428   AMDGPU::initDefaultAMDKernelCodeT(Header, getFeatureBits());
2429 
2430   while (true) {
2431     // Lex EndOfStatement.  This is in a while loop, because lexing a comment
2432     // will set the current token to EndOfStatement.
2433     while(getLexer().is(AsmToken::EndOfStatement))
2434       Lex();
2435 
2436     if (getLexer().isNot(AsmToken::Identifier))
2437       return TokError("expected value identifier or .end_amd_kernel_code_t");
2438 
2439     StringRef ID = getLexer().getTok().getIdentifier();
2440     Lex();
2441 
2442     if (ID == ".end_amd_kernel_code_t")
2443       break;
2444 
2445     if (ParseAMDKernelCodeTValue(ID, Header))
2446       return true;
2447   }
2448 
2449   getTargetStreamer().EmitAMDKernelCodeT(Header);
2450 
2451   return false;
2452 }
2453 
2454 bool AMDGPUAsmParser::ParseDirectiveAMDGPUHsaKernel() {
2455   if (getLexer().isNot(AsmToken::Identifier))
2456     return TokError("expected symbol name");
2457 
2458   StringRef KernelName = Parser.getTok().getString();
2459 
2460   getTargetStreamer().EmitAMDGPUSymbolType(KernelName,
2461                                            ELF::STT_AMDGPU_HSA_KERNEL);
2462   Lex();
2463   KernelScope.initialize(getContext());
2464   return false;
2465 }
2466 
2467 bool AMDGPUAsmParser::ParseDirective(AsmToken DirectiveID) {
2468   StringRef IDVal = DirectiveID.getString();
2469 
2470   if (IDVal == ".hsa_code_object_version")
2471     return ParseDirectiveHSACodeObjectVersion();
2472 
2473   if (IDVal == ".hsa_code_object_isa")
2474     return ParseDirectiveHSACodeObjectISA();
2475 
2476   if (IDVal == AMDGPU::CodeObject::MetadataAssemblerDirectiveBegin)
2477     return ParseDirectiveCodeObjectMetadata();
2478 
2479   if (IDVal == ".amd_kernel_code_t")
2480     return ParseDirectiveAMDKernelCodeT();
2481 
2482   if (IDVal == ".amdgpu_hsa_kernel")
2483     return ParseDirectiveAMDGPUHsaKernel();
2484 
2485   return true;
2486 }
2487 
2488 bool AMDGPUAsmParser::subtargetHasRegister(const MCRegisterInfo &MRI,
2489                                            unsigned RegNo) const {
2490   if (isCI())
2491     return true;
2492 
2493   if (isSI()) {
2494     // No flat_scr
2495     switch (RegNo) {
2496     case AMDGPU::FLAT_SCR:
2497     case AMDGPU::FLAT_SCR_LO:
2498     case AMDGPU::FLAT_SCR_HI:
2499       return false;
2500     default:
2501       return true;
2502     }
2503   }
2504 
2505   // VI only has 102 SGPRs, so make sure we aren't trying to use the 2 more that
2506   // SI/CI have.
2507   for (MCRegAliasIterator R(AMDGPU::SGPR102_SGPR103, &MRI, true);
2508        R.isValid(); ++R) {
2509     if (*R == RegNo)
2510       return false;
2511   }
2512 
2513   return true;
2514 }
2515 
2516 OperandMatchResultTy
2517 AMDGPUAsmParser::parseOperand(OperandVector &Operands, StringRef Mnemonic) {
2518   // Try to parse with a custom parser
2519   OperandMatchResultTy ResTy = MatchOperandParserImpl(Operands, Mnemonic);
2520 
2521   // If we successfully parsed the operand or if there as an error parsing,
2522   // we are done.
2523   //
2524   // If we are parsing after we reach EndOfStatement then this means we
2525   // are appending default values to the Operands list.  This is only done
2526   // by custom parser, so we shouldn't continue on to the generic parsing.
2527   if (ResTy == MatchOperand_Success || ResTy == MatchOperand_ParseFail ||
2528       getLexer().is(AsmToken::EndOfStatement))
2529     return ResTy;
2530 
2531   ResTy = parseRegOrImm(Operands);
2532 
2533   if (ResTy == MatchOperand_Success)
2534     return ResTy;
2535 
2536   if (getLexer().getKind() == AsmToken::Identifier) {
2537     // If this identifier is a symbol, we want to create an expression for it.
2538     // It is a little difficult to distinguish between a symbol name, and
2539     // an instruction flag like 'gds'.  In order to do this, we parse
2540     // all tokens as expressions and then treate the symbol name as the token
2541     // string when we want to interpret the operand as a token.
2542     const auto &Tok = Parser.getTok();
2543     SMLoc S = Tok.getLoc();
2544     const MCExpr *Expr = nullptr;
2545     if (!Parser.parseExpression(Expr)) {
2546       Operands.push_back(AMDGPUOperand::CreateExpr(this, Expr, S));
2547       return MatchOperand_Success;
2548     }
2549 
2550     Operands.push_back(AMDGPUOperand::CreateToken(this, Tok.getString(), Tok.getLoc()));
2551     Parser.Lex();
2552     return MatchOperand_Success;
2553   }
2554   return MatchOperand_NoMatch;
2555 }
2556 
2557 StringRef AMDGPUAsmParser::parseMnemonicSuffix(StringRef Name) {
2558   // Clear any forced encodings from the previous instruction.
2559   setForcedEncodingSize(0);
2560   setForcedDPP(false);
2561   setForcedSDWA(false);
2562 
2563   if (Name.endswith("_e64")) {
2564     setForcedEncodingSize(64);
2565     return Name.substr(0, Name.size() - 4);
2566   } else if (Name.endswith("_e32")) {
2567     setForcedEncodingSize(32);
2568     return Name.substr(0, Name.size() - 4);
2569   } else if (Name.endswith("_dpp")) {
2570     setForcedDPP(true);
2571     return Name.substr(0, Name.size() - 4);
2572   } else if (Name.endswith("_sdwa")) {
2573     setForcedSDWA(true);
2574     return Name.substr(0, Name.size() - 5);
2575   }
2576   return Name;
2577 }
2578 
2579 bool AMDGPUAsmParser::ParseInstruction(ParseInstructionInfo &Info,
2580                                        StringRef Name,
2581                                        SMLoc NameLoc, OperandVector &Operands) {
2582   // Add the instruction mnemonic
2583   Name = parseMnemonicSuffix(Name);
2584   Operands.push_back(AMDGPUOperand::CreateToken(this, Name, NameLoc));
2585 
2586   while (!getLexer().is(AsmToken::EndOfStatement)) {
2587     OperandMatchResultTy Res = parseOperand(Operands, Name);
2588 
2589     // Eat the comma or space if there is one.
2590     if (getLexer().is(AsmToken::Comma))
2591       Parser.Lex();
2592 
2593     switch (Res) {
2594       case MatchOperand_Success: break;
2595       case MatchOperand_ParseFail:
2596         Error(getLexer().getLoc(), "failed parsing operand.");
2597         while (!getLexer().is(AsmToken::EndOfStatement)) {
2598           Parser.Lex();
2599         }
2600         return true;
2601       case MatchOperand_NoMatch:
2602         Error(getLexer().getLoc(), "not a valid operand.");
2603         while (!getLexer().is(AsmToken::EndOfStatement)) {
2604           Parser.Lex();
2605         }
2606         return true;
2607     }
2608   }
2609 
2610   return false;
2611 }
2612 
2613 //===----------------------------------------------------------------------===//
2614 // Utility functions
2615 //===----------------------------------------------------------------------===//
2616 
2617 OperandMatchResultTy
2618 AMDGPUAsmParser::parseIntWithPrefix(const char *Prefix, int64_t &Int) {
2619   switch(getLexer().getKind()) {
2620     default: return MatchOperand_NoMatch;
2621     case AsmToken::Identifier: {
2622       StringRef Name = Parser.getTok().getString();
2623       if (!Name.equals(Prefix)) {
2624         return MatchOperand_NoMatch;
2625       }
2626 
2627       Parser.Lex();
2628       if (getLexer().isNot(AsmToken::Colon))
2629         return MatchOperand_ParseFail;
2630 
2631       Parser.Lex();
2632 
2633       bool IsMinus = false;
2634       if (getLexer().getKind() == AsmToken::Minus) {
2635         Parser.Lex();
2636         IsMinus = true;
2637       }
2638 
2639       if (getLexer().isNot(AsmToken::Integer))
2640         return MatchOperand_ParseFail;
2641 
2642       if (getParser().parseAbsoluteExpression(Int))
2643         return MatchOperand_ParseFail;
2644 
2645       if (IsMinus)
2646         Int = -Int;
2647       break;
2648     }
2649   }
2650   return MatchOperand_Success;
2651 }
2652 
2653 OperandMatchResultTy
2654 AMDGPUAsmParser::parseIntWithPrefix(const char *Prefix, OperandVector &Operands,
2655                                     AMDGPUOperand::ImmTy ImmTy,
2656                                     bool (*ConvertResult)(int64_t&)) {
2657   SMLoc S = Parser.getTok().getLoc();
2658   int64_t Value = 0;
2659 
2660   OperandMatchResultTy Res = parseIntWithPrefix(Prefix, Value);
2661   if (Res != MatchOperand_Success)
2662     return Res;
2663 
2664   if (ConvertResult && !ConvertResult(Value)) {
2665     return MatchOperand_ParseFail;
2666   }
2667 
2668   Operands.push_back(AMDGPUOperand::CreateImm(this, Value, S, ImmTy));
2669   return MatchOperand_Success;
2670 }
2671 
2672 OperandMatchResultTy AMDGPUAsmParser::parseOperandArrayWithPrefix(
2673   const char *Prefix,
2674   OperandVector &Operands,
2675   AMDGPUOperand::ImmTy ImmTy,
2676   bool (*ConvertResult)(int64_t&)) {
2677   StringRef Name = Parser.getTok().getString();
2678   if (!Name.equals(Prefix))
2679     return MatchOperand_NoMatch;
2680 
2681   Parser.Lex();
2682   if (getLexer().isNot(AsmToken::Colon))
2683     return MatchOperand_ParseFail;
2684 
2685   Parser.Lex();
2686   if (getLexer().isNot(AsmToken::LBrac))
2687     return MatchOperand_ParseFail;
2688   Parser.Lex();
2689 
2690   unsigned Val = 0;
2691   SMLoc S = Parser.getTok().getLoc();
2692 
2693   // FIXME: How to verify the number of elements matches the number of src
2694   // operands?
2695   for (int I = 0; I < 3; ++I) {
2696     if (I != 0) {
2697       if (getLexer().is(AsmToken::RBrac))
2698         break;
2699 
2700       if (getLexer().isNot(AsmToken::Comma))
2701         return MatchOperand_ParseFail;
2702       Parser.Lex();
2703     }
2704 
2705     if (getLexer().isNot(AsmToken::Integer))
2706       return MatchOperand_ParseFail;
2707 
2708     int64_t Op;
2709     if (getParser().parseAbsoluteExpression(Op))
2710       return MatchOperand_ParseFail;
2711 
2712     if (Op != 0 && Op != 1)
2713       return MatchOperand_ParseFail;
2714     Val |= (Op << I);
2715   }
2716 
2717   Parser.Lex();
2718   Operands.push_back(AMDGPUOperand::CreateImm(this, Val, S, ImmTy));
2719   return MatchOperand_Success;
2720 }
2721 
2722 OperandMatchResultTy
2723 AMDGPUAsmParser::parseNamedBit(const char *Name, OperandVector &Operands,
2724                                AMDGPUOperand::ImmTy ImmTy) {
2725   int64_t Bit = 0;
2726   SMLoc S = Parser.getTok().getLoc();
2727 
2728   // We are at the end of the statement, and this is a default argument, so
2729   // use a default value.
2730   if (getLexer().isNot(AsmToken::EndOfStatement)) {
2731     switch(getLexer().getKind()) {
2732       case AsmToken::Identifier: {
2733         StringRef Tok = Parser.getTok().getString();
2734         if (Tok == Name) {
2735           Bit = 1;
2736           Parser.Lex();
2737         } else if (Tok.startswith("no") && Tok.endswith(Name)) {
2738           Bit = 0;
2739           Parser.Lex();
2740         } else {
2741           return MatchOperand_NoMatch;
2742         }
2743         break;
2744       }
2745       default:
2746         return MatchOperand_NoMatch;
2747     }
2748   }
2749 
2750   Operands.push_back(AMDGPUOperand::CreateImm(this, Bit, S, ImmTy));
2751   return MatchOperand_Success;
2752 }
2753 
2754 static void addOptionalImmOperand(
2755   MCInst& Inst, const OperandVector& Operands,
2756   AMDGPUAsmParser::OptionalImmIndexMap& OptionalIdx,
2757   AMDGPUOperand::ImmTy ImmT,
2758   int64_t Default = 0) {
2759   auto i = OptionalIdx.find(ImmT);
2760   if (i != OptionalIdx.end()) {
2761     unsigned Idx = i->second;
2762     ((AMDGPUOperand &)*Operands[Idx]).addImmOperands(Inst, 1);
2763   } else {
2764     Inst.addOperand(MCOperand::createImm(Default));
2765   }
2766 }
2767 
2768 OperandMatchResultTy
2769 AMDGPUAsmParser::parseStringWithPrefix(StringRef Prefix, StringRef &Value) {
2770   if (getLexer().isNot(AsmToken::Identifier)) {
2771     return MatchOperand_NoMatch;
2772   }
2773   StringRef Tok = Parser.getTok().getString();
2774   if (Tok != Prefix) {
2775     return MatchOperand_NoMatch;
2776   }
2777 
2778   Parser.Lex();
2779   if (getLexer().isNot(AsmToken::Colon)) {
2780     return MatchOperand_ParseFail;
2781   }
2782 
2783   Parser.Lex();
2784   if (getLexer().isNot(AsmToken::Identifier)) {
2785     return MatchOperand_ParseFail;
2786   }
2787 
2788   Value = Parser.getTok().getString();
2789   return MatchOperand_Success;
2790 }
2791 
2792 //===----------------------------------------------------------------------===//
2793 // ds
2794 //===----------------------------------------------------------------------===//
2795 
2796 void AMDGPUAsmParser::cvtDSOffset01(MCInst &Inst,
2797                                     const OperandVector &Operands) {
2798   OptionalImmIndexMap OptionalIdx;
2799 
2800   for (unsigned i = 1, e = Operands.size(); i != e; ++i) {
2801     AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[i]);
2802 
2803     // Add the register arguments
2804     if (Op.isReg()) {
2805       Op.addRegOperands(Inst, 1);
2806       continue;
2807     }
2808 
2809     // Handle optional arguments
2810     OptionalIdx[Op.getImmTy()] = i;
2811   }
2812 
2813   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyOffset0);
2814   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyOffset1);
2815   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyGDS);
2816 
2817   Inst.addOperand(MCOperand::createReg(AMDGPU::M0)); // m0
2818 }
2819 
2820 void AMDGPUAsmParser::cvtDSImpl(MCInst &Inst, const OperandVector &Operands,
2821                                 bool IsGdsHardcoded) {
2822   OptionalImmIndexMap OptionalIdx;
2823 
2824   for (unsigned i = 1, e = Operands.size(); i != e; ++i) {
2825     AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[i]);
2826 
2827     // Add the register arguments
2828     if (Op.isReg()) {
2829       Op.addRegOperands(Inst, 1);
2830       continue;
2831     }
2832 
2833     if (Op.isToken() && Op.getToken() == "gds") {
2834       IsGdsHardcoded = true;
2835       continue;
2836     }
2837 
2838     // Handle optional arguments
2839     OptionalIdx[Op.getImmTy()] = i;
2840   }
2841 
2842   AMDGPUOperand::ImmTy OffsetType =
2843     (Inst.getOpcode() == AMDGPU::DS_SWIZZLE_B32_si ||
2844      Inst.getOpcode() == AMDGPU::DS_SWIZZLE_B32_vi) ? AMDGPUOperand::ImmTySwizzle :
2845                                                       AMDGPUOperand::ImmTyOffset;
2846 
2847   addOptionalImmOperand(Inst, Operands, OptionalIdx, OffsetType);
2848 
2849   if (!IsGdsHardcoded) {
2850     addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyGDS);
2851   }
2852   Inst.addOperand(MCOperand::createReg(AMDGPU::M0)); // m0
2853 }
2854 
2855 void AMDGPUAsmParser::cvtExp(MCInst &Inst, const OperandVector &Operands) {
2856   OptionalImmIndexMap OptionalIdx;
2857 
2858   unsigned OperandIdx[4];
2859   unsigned EnMask = 0;
2860   int SrcIdx = 0;
2861 
2862   for (unsigned i = 1, e = Operands.size(); i != e; ++i) {
2863     AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[i]);
2864 
2865     // Add the register arguments
2866     if (Op.isReg()) {
2867       assert(SrcIdx < 4);
2868       OperandIdx[SrcIdx] = Inst.size();
2869       Op.addRegOperands(Inst, 1);
2870       ++SrcIdx;
2871       continue;
2872     }
2873 
2874     if (Op.isOff()) {
2875       assert(SrcIdx < 4);
2876       OperandIdx[SrcIdx] = Inst.size();
2877       Inst.addOperand(MCOperand::createReg(AMDGPU::NoRegister));
2878       ++SrcIdx;
2879       continue;
2880     }
2881 
2882     if (Op.isImm() && Op.getImmTy() == AMDGPUOperand::ImmTyExpTgt) {
2883       Op.addImmOperands(Inst, 1);
2884       continue;
2885     }
2886 
2887     if (Op.isToken() && Op.getToken() == "done")
2888       continue;
2889 
2890     // Handle optional arguments
2891     OptionalIdx[Op.getImmTy()] = i;
2892   }
2893 
2894   assert(SrcIdx == 4);
2895 
2896   bool Compr = false;
2897   if (OptionalIdx.find(AMDGPUOperand::ImmTyExpCompr) != OptionalIdx.end()) {
2898     Compr = true;
2899     Inst.getOperand(OperandIdx[1]) = Inst.getOperand(OperandIdx[2]);
2900     Inst.getOperand(OperandIdx[2]).setReg(AMDGPU::NoRegister);
2901     Inst.getOperand(OperandIdx[3]).setReg(AMDGPU::NoRegister);
2902   }
2903 
2904   for (auto i = 0; i < SrcIdx; ++i) {
2905     if (Inst.getOperand(OperandIdx[i]).getReg() != AMDGPU::NoRegister) {
2906       EnMask |= Compr? (0x3 << i * 2) : (0x1 << i);
2907     }
2908   }
2909 
2910   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyExpVM);
2911   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyExpCompr);
2912 
2913   Inst.addOperand(MCOperand::createImm(EnMask));
2914 }
2915 
2916 //===----------------------------------------------------------------------===//
2917 // s_waitcnt
2918 //===----------------------------------------------------------------------===//
2919 
2920 static bool
2921 encodeCnt(
2922   const AMDGPU::IsaInfo::IsaVersion ISA,
2923   int64_t &IntVal,
2924   int64_t CntVal,
2925   bool Saturate,
2926   unsigned (*encode)(const IsaInfo::IsaVersion &Version, unsigned, unsigned),
2927   unsigned (*decode)(const IsaInfo::IsaVersion &Version, unsigned))
2928 {
2929   bool Failed = false;
2930 
2931   IntVal = encode(ISA, IntVal, CntVal);
2932   if (CntVal != decode(ISA, IntVal)) {
2933     if (Saturate) {
2934       IntVal = encode(ISA, IntVal, -1);
2935     } else {
2936       Failed = true;
2937     }
2938   }
2939   return Failed;
2940 }
2941 
2942 bool AMDGPUAsmParser::parseCnt(int64_t &IntVal) {
2943   StringRef CntName = Parser.getTok().getString();
2944   int64_t CntVal;
2945 
2946   Parser.Lex();
2947   if (getLexer().isNot(AsmToken::LParen))
2948     return true;
2949 
2950   Parser.Lex();
2951   if (getLexer().isNot(AsmToken::Integer))
2952     return true;
2953 
2954   SMLoc ValLoc = Parser.getTok().getLoc();
2955   if (getParser().parseAbsoluteExpression(CntVal))
2956     return true;
2957 
2958   AMDGPU::IsaInfo::IsaVersion ISA =
2959       AMDGPU::IsaInfo::getIsaVersion(getFeatureBits());
2960 
2961   bool Failed = true;
2962   bool Sat = CntName.endswith("_sat");
2963 
2964   if (CntName == "vmcnt" || CntName == "vmcnt_sat") {
2965     Failed = encodeCnt(ISA, IntVal, CntVal, Sat, encodeVmcnt, decodeVmcnt);
2966   } else if (CntName == "expcnt" || CntName == "expcnt_sat") {
2967     Failed = encodeCnt(ISA, IntVal, CntVal, Sat, encodeExpcnt, decodeExpcnt);
2968   } else if (CntName == "lgkmcnt" || CntName == "lgkmcnt_sat") {
2969     Failed = encodeCnt(ISA, IntVal, CntVal, Sat, encodeLgkmcnt, decodeLgkmcnt);
2970   }
2971 
2972   if (Failed) {
2973     Error(ValLoc, "too large value for " + CntName);
2974     return true;
2975   }
2976 
2977   if (getLexer().isNot(AsmToken::RParen)) {
2978     return true;
2979   }
2980 
2981   Parser.Lex();
2982   if (getLexer().is(AsmToken::Amp) || getLexer().is(AsmToken::Comma)) {
2983     const AsmToken NextToken = getLexer().peekTok();
2984     if (NextToken.is(AsmToken::Identifier)) {
2985       Parser.Lex();
2986     }
2987   }
2988 
2989   return false;
2990 }
2991 
2992 OperandMatchResultTy
2993 AMDGPUAsmParser::parseSWaitCntOps(OperandVector &Operands) {
2994   AMDGPU::IsaInfo::IsaVersion ISA =
2995       AMDGPU::IsaInfo::getIsaVersion(getFeatureBits());
2996   int64_t Waitcnt = getWaitcntBitMask(ISA);
2997   SMLoc S = Parser.getTok().getLoc();
2998 
2999   switch(getLexer().getKind()) {
3000     default: return MatchOperand_ParseFail;
3001     case AsmToken::Integer:
3002       // The operand can be an integer value.
3003       if (getParser().parseAbsoluteExpression(Waitcnt))
3004         return MatchOperand_ParseFail;
3005       break;
3006 
3007     case AsmToken::Identifier:
3008       do {
3009         if (parseCnt(Waitcnt))
3010           return MatchOperand_ParseFail;
3011       } while(getLexer().isNot(AsmToken::EndOfStatement));
3012       break;
3013   }
3014   Operands.push_back(AMDGPUOperand::CreateImm(this, Waitcnt, S));
3015   return MatchOperand_Success;
3016 }
3017 
3018 bool AMDGPUAsmParser::parseHwregConstruct(OperandInfoTy &HwReg, int64_t &Offset,
3019                                           int64_t &Width) {
3020   using namespace llvm::AMDGPU::Hwreg;
3021 
3022   if (Parser.getTok().getString() != "hwreg")
3023     return true;
3024   Parser.Lex();
3025 
3026   if (getLexer().isNot(AsmToken::LParen))
3027     return true;
3028   Parser.Lex();
3029 
3030   if (getLexer().is(AsmToken::Identifier)) {
3031     HwReg.IsSymbolic = true;
3032     HwReg.Id = ID_UNKNOWN_;
3033     const StringRef tok = Parser.getTok().getString();
3034     for (int i = ID_SYMBOLIC_FIRST_; i < ID_SYMBOLIC_LAST_; ++i) {
3035       if (tok == IdSymbolic[i]) {
3036         HwReg.Id = i;
3037         break;
3038       }
3039     }
3040     Parser.Lex();
3041   } else {
3042     HwReg.IsSymbolic = false;
3043     if (getLexer().isNot(AsmToken::Integer))
3044       return true;
3045     if (getParser().parseAbsoluteExpression(HwReg.Id))
3046       return true;
3047   }
3048 
3049   if (getLexer().is(AsmToken::RParen)) {
3050     Parser.Lex();
3051     return false;
3052   }
3053 
3054   // optional params
3055   if (getLexer().isNot(AsmToken::Comma))
3056     return true;
3057   Parser.Lex();
3058 
3059   if (getLexer().isNot(AsmToken::Integer))
3060     return true;
3061   if (getParser().parseAbsoluteExpression(Offset))
3062     return true;
3063 
3064   if (getLexer().isNot(AsmToken::Comma))
3065     return true;
3066   Parser.Lex();
3067 
3068   if (getLexer().isNot(AsmToken::Integer))
3069     return true;
3070   if (getParser().parseAbsoluteExpression(Width))
3071     return true;
3072 
3073   if (getLexer().isNot(AsmToken::RParen))
3074     return true;
3075   Parser.Lex();
3076 
3077   return false;
3078 }
3079 
3080 OperandMatchResultTy AMDGPUAsmParser::parseHwreg(OperandVector &Operands) {
3081   using namespace llvm::AMDGPU::Hwreg;
3082 
3083   int64_t Imm16Val = 0;
3084   SMLoc S = Parser.getTok().getLoc();
3085 
3086   switch(getLexer().getKind()) {
3087     default: return MatchOperand_NoMatch;
3088     case AsmToken::Integer:
3089       // The operand can be an integer value.
3090       if (getParser().parseAbsoluteExpression(Imm16Val))
3091         return MatchOperand_NoMatch;
3092       if (Imm16Val < 0 || !isUInt<16>(Imm16Val)) {
3093         Error(S, "invalid immediate: only 16-bit values are legal");
3094         // Do not return error code, but create an imm operand anyway and proceed
3095         // to the next operand, if any. That avoids unneccessary error messages.
3096       }
3097       break;
3098 
3099     case AsmToken::Identifier: {
3100         OperandInfoTy HwReg(ID_UNKNOWN_);
3101         int64_t Offset = OFFSET_DEFAULT_;
3102         int64_t Width = WIDTH_M1_DEFAULT_ + 1;
3103         if (parseHwregConstruct(HwReg, Offset, Width))
3104           return MatchOperand_ParseFail;
3105         if (HwReg.Id < 0 || !isUInt<ID_WIDTH_>(HwReg.Id)) {
3106           if (HwReg.IsSymbolic)
3107             Error(S, "invalid symbolic name of hardware register");
3108           else
3109             Error(S, "invalid code of hardware register: only 6-bit values are legal");
3110         }
3111         if (Offset < 0 || !isUInt<OFFSET_WIDTH_>(Offset))
3112           Error(S, "invalid bit offset: only 5-bit values are legal");
3113         if ((Width-1) < 0 || !isUInt<WIDTH_M1_WIDTH_>(Width-1))
3114           Error(S, "invalid bitfield width: only values from 1 to 32 are legal");
3115         Imm16Val = (HwReg.Id << ID_SHIFT_) | (Offset << OFFSET_SHIFT_) | ((Width-1) << WIDTH_M1_SHIFT_);
3116       }
3117       break;
3118   }
3119   Operands.push_back(AMDGPUOperand::CreateImm(this, Imm16Val, S, AMDGPUOperand::ImmTyHwreg));
3120   return MatchOperand_Success;
3121 }
3122 
3123 bool AMDGPUOperand::isSWaitCnt() const {
3124   return isImm();
3125 }
3126 
3127 bool AMDGPUOperand::isHwreg() const {
3128   return isImmTy(ImmTyHwreg);
3129 }
3130 
3131 bool AMDGPUAsmParser::parseSendMsgConstruct(OperandInfoTy &Msg, OperandInfoTy &Operation, int64_t &StreamId) {
3132   using namespace llvm::AMDGPU::SendMsg;
3133 
3134   if (Parser.getTok().getString() != "sendmsg")
3135     return true;
3136   Parser.Lex();
3137 
3138   if (getLexer().isNot(AsmToken::LParen))
3139     return true;
3140   Parser.Lex();
3141 
3142   if (getLexer().is(AsmToken::Identifier)) {
3143     Msg.IsSymbolic = true;
3144     Msg.Id = ID_UNKNOWN_;
3145     const std::string tok = Parser.getTok().getString();
3146     for (int i = ID_GAPS_FIRST_; i < ID_GAPS_LAST_; ++i) {
3147       switch(i) {
3148         default: continue; // Omit gaps.
3149         case ID_INTERRUPT: case ID_GS: case ID_GS_DONE:  case ID_SYSMSG: break;
3150       }
3151       if (tok == IdSymbolic[i]) {
3152         Msg.Id = i;
3153         break;
3154       }
3155     }
3156     Parser.Lex();
3157   } else {
3158     Msg.IsSymbolic = false;
3159     if (getLexer().isNot(AsmToken::Integer))
3160       return true;
3161     if (getParser().parseAbsoluteExpression(Msg.Id))
3162       return true;
3163     if (getLexer().is(AsmToken::Integer))
3164       if (getParser().parseAbsoluteExpression(Msg.Id))
3165         Msg.Id = ID_UNKNOWN_;
3166   }
3167   if (Msg.Id == ID_UNKNOWN_) // Don't know how to parse the rest.
3168     return false;
3169 
3170   if (!(Msg.Id == ID_GS || Msg.Id == ID_GS_DONE || Msg.Id == ID_SYSMSG)) {
3171     if (getLexer().isNot(AsmToken::RParen))
3172       return true;
3173     Parser.Lex();
3174     return false;
3175   }
3176 
3177   if (getLexer().isNot(AsmToken::Comma))
3178     return true;
3179   Parser.Lex();
3180 
3181   assert(Msg.Id == ID_GS || Msg.Id == ID_GS_DONE || Msg.Id == ID_SYSMSG);
3182   Operation.Id = ID_UNKNOWN_;
3183   if (getLexer().is(AsmToken::Identifier)) {
3184     Operation.IsSymbolic = true;
3185     const char* const *S = (Msg.Id == ID_SYSMSG) ? OpSysSymbolic : OpGsSymbolic;
3186     const int F = (Msg.Id == ID_SYSMSG) ? OP_SYS_FIRST_ : OP_GS_FIRST_;
3187     const int L = (Msg.Id == ID_SYSMSG) ? OP_SYS_LAST_ : OP_GS_LAST_;
3188     const StringRef Tok = Parser.getTok().getString();
3189     for (int i = F; i < L; ++i) {
3190       if (Tok == S[i]) {
3191         Operation.Id = i;
3192         break;
3193       }
3194     }
3195     Parser.Lex();
3196   } else {
3197     Operation.IsSymbolic = false;
3198     if (getLexer().isNot(AsmToken::Integer))
3199       return true;
3200     if (getParser().parseAbsoluteExpression(Operation.Id))
3201       return true;
3202   }
3203 
3204   if ((Msg.Id == ID_GS || Msg.Id == ID_GS_DONE) && Operation.Id != OP_GS_NOP) {
3205     // Stream id is optional.
3206     if (getLexer().is(AsmToken::RParen)) {
3207       Parser.Lex();
3208       return false;
3209     }
3210 
3211     if (getLexer().isNot(AsmToken::Comma))
3212       return true;
3213     Parser.Lex();
3214 
3215     if (getLexer().isNot(AsmToken::Integer))
3216       return true;
3217     if (getParser().parseAbsoluteExpression(StreamId))
3218       return true;
3219   }
3220 
3221   if (getLexer().isNot(AsmToken::RParen))
3222     return true;
3223   Parser.Lex();
3224   return false;
3225 }
3226 
3227 OperandMatchResultTy AMDGPUAsmParser::parseInterpSlot(OperandVector &Operands) {
3228   if (getLexer().getKind() != AsmToken::Identifier)
3229     return MatchOperand_NoMatch;
3230 
3231   StringRef Str = Parser.getTok().getString();
3232   int Slot = StringSwitch<int>(Str)
3233     .Case("p10", 0)
3234     .Case("p20", 1)
3235     .Case("p0", 2)
3236     .Default(-1);
3237 
3238   SMLoc S = Parser.getTok().getLoc();
3239   if (Slot == -1)
3240     return MatchOperand_ParseFail;
3241 
3242   Parser.Lex();
3243   Operands.push_back(AMDGPUOperand::CreateImm(this, Slot, S,
3244                                               AMDGPUOperand::ImmTyInterpSlot));
3245   return MatchOperand_Success;
3246 }
3247 
3248 OperandMatchResultTy AMDGPUAsmParser::parseInterpAttr(OperandVector &Operands) {
3249   if (getLexer().getKind() != AsmToken::Identifier)
3250     return MatchOperand_NoMatch;
3251 
3252   StringRef Str = Parser.getTok().getString();
3253   if (!Str.startswith("attr"))
3254     return MatchOperand_NoMatch;
3255 
3256   StringRef Chan = Str.take_back(2);
3257   int AttrChan = StringSwitch<int>(Chan)
3258     .Case(".x", 0)
3259     .Case(".y", 1)
3260     .Case(".z", 2)
3261     .Case(".w", 3)
3262     .Default(-1);
3263   if (AttrChan == -1)
3264     return MatchOperand_ParseFail;
3265 
3266   Str = Str.drop_back(2).drop_front(4);
3267 
3268   uint8_t Attr;
3269   if (Str.getAsInteger(10, Attr))
3270     return MatchOperand_ParseFail;
3271 
3272   SMLoc S = Parser.getTok().getLoc();
3273   Parser.Lex();
3274   if (Attr > 63) {
3275     Error(S, "out of bounds attr");
3276     return MatchOperand_Success;
3277   }
3278 
3279   SMLoc SChan = SMLoc::getFromPointer(Chan.data());
3280 
3281   Operands.push_back(AMDGPUOperand::CreateImm(this, Attr, S,
3282                                               AMDGPUOperand::ImmTyInterpAttr));
3283   Operands.push_back(AMDGPUOperand::CreateImm(this, AttrChan, SChan,
3284                                               AMDGPUOperand::ImmTyAttrChan));
3285   return MatchOperand_Success;
3286 }
3287 
3288 void AMDGPUAsmParser::errorExpTgt() {
3289   Error(Parser.getTok().getLoc(), "invalid exp target");
3290 }
3291 
3292 OperandMatchResultTy AMDGPUAsmParser::parseExpTgtImpl(StringRef Str,
3293                                                       uint8_t &Val) {
3294   if (Str == "null") {
3295     Val = 9;
3296     return MatchOperand_Success;
3297   }
3298 
3299   if (Str.startswith("mrt")) {
3300     Str = Str.drop_front(3);
3301     if (Str == "z") { // == mrtz
3302       Val = 8;
3303       return MatchOperand_Success;
3304     }
3305 
3306     if (Str.getAsInteger(10, Val))
3307       return MatchOperand_ParseFail;
3308 
3309     if (Val > 7)
3310       errorExpTgt();
3311 
3312     return MatchOperand_Success;
3313   }
3314 
3315   if (Str.startswith("pos")) {
3316     Str = Str.drop_front(3);
3317     if (Str.getAsInteger(10, Val))
3318       return MatchOperand_ParseFail;
3319 
3320     if (Val > 3)
3321       errorExpTgt();
3322 
3323     Val += 12;
3324     return MatchOperand_Success;
3325   }
3326 
3327   if (Str.startswith("param")) {
3328     Str = Str.drop_front(5);
3329     if (Str.getAsInteger(10, Val))
3330       return MatchOperand_ParseFail;
3331 
3332     if (Val >= 32)
3333       errorExpTgt();
3334 
3335     Val += 32;
3336     return MatchOperand_Success;
3337   }
3338 
3339   if (Str.startswith("invalid_target_")) {
3340     Str = Str.drop_front(15);
3341     if (Str.getAsInteger(10, Val))
3342       return MatchOperand_ParseFail;
3343 
3344     errorExpTgt();
3345     return MatchOperand_Success;
3346   }
3347 
3348   return MatchOperand_NoMatch;
3349 }
3350 
3351 OperandMatchResultTy AMDGPUAsmParser::parseExpTgt(OperandVector &Operands) {
3352   uint8_t Val;
3353   StringRef Str = Parser.getTok().getString();
3354 
3355   auto Res = parseExpTgtImpl(Str, Val);
3356   if (Res != MatchOperand_Success)
3357     return Res;
3358 
3359   SMLoc S = Parser.getTok().getLoc();
3360   Parser.Lex();
3361 
3362   Operands.push_back(AMDGPUOperand::CreateImm(this, Val, S,
3363                                               AMDGPUOperand::ImmTyExpTgt));
3364   return MatchOperand_Success;
3365 }
3366 
3367 OperandMatchResultTy
3368 AMDGPUAsmParser::parseSendMsgOp(OperandVector &Operands) {
3369   using namespace llvm::AMDGPU::SendMsg;
3370 
3371   int64_t Imm16Val = 0;
3372   SMLoc S = Parser.getTok().getLoc();
3373 
3374   switch(getLexer().getKind()) {
3375   default:
3376     return MatchOperand_NoMatch;
3377   case AsmToken::Integer:
3378     // The operand can be an integer value.
3379     if (getParser().parseAbsoluteExpression(Imm16Val))
3380       return MatchOperand_NoMatch;
3381     if (Imm16Val < 0 || !isUInt<16>(Imm16Val)) {
3382       Error(S, "invalid immediate: only 16-bit values are legal");
3383       // Do not return error code, but create an imm operand anyway and proceed
3384       // to the next operand, if any. That avoids unneccessary error messages.
3385     }
3386     break;
3387   case AsmToken::Identifier: {
3388       OperandInfoTy Msg(ID_UNKNOWN_);
3389       OperandInfoTy Operation(OP_UNKNOWN_);
3390       int64_t StreamId = STREAM_ID_DEFAULT_;
3391       if (parseSendMsgConstruct(Msg, Operation, StreamId))
3392         return MatchOperand_ParseFail;
3393       do {
3394         // Validate and encode message ID.
3395         if (! ((ID_INTERRUPT <= Msg.Id && Msg.Id <= ID_GS_DONE)
3396                 || Msg.Id == ID_SYSMSG)) {
3397           if (Msg.IsSymbolic)
3398             Error(S, "invalid/unsupported symbolic name of message");
3399           else
3400             Error(S, "invalid/unsupported code of message");
3401           break;
3402         }
3403         Imm16Val = (Msg.Id << ID_SHIFT_);
3404         // Validate and encode operation ID.
3405         if (Msg.Id == ID_GS || Msg.Id == ID_GS_DONE) {
3406           if (! (OP_GS_FIRST_ <= Operation.Id && Operation.Id < OP_GS_LAST_)) {
3407             if (Operation.IsSymbolic)
3408               Error(S, "invalid symbolic name of GS_OP");
3409             else
3410               Error(S, "invalid code of GS_OP: only 2-bit values are legal");
3411             break;
3412           }
3413           if (Operation.Id == OP_GS_NOP
3414               && Msg.Id != ID_GS_DONE) {
3415             Error(S, "invalid GS_OP: NOP is for GS_DONE only");
3416             break;
3417           }
3418           Imm16Val |= (Operation.Id << OP_SHIFT_);
3419         }
3420         if (Msg.Id == ID_SYSMSG) {
3421           if (! (OP_SYS_FIRST_ <= Operation.Id && Operation.Id < OP_SYS_LAST_)) {
3422             if (Operation.IsSymbolic)
3423               Error(S, "invalid/unsupported symbolic name of SYSMSG_OP");
3424             else
3425               Error(S, "invalid/unsupported code of SYSMSG_OP");
3426             break;
3427           }
3428           Imm16Val |= (Operation.Id << OP_SHIFT_);
3429         }
3430         // Validate and encode stream ID.
3431         if ((Msg.Id == ID_GS || Msg.Id == ID_GS_DONE) && Operation.Id != OP_GS_NOP) {
3432           if (! (STREAM_ID_FIRST_ <= StreamId && StreamId < STREAM_ID_LAST_)) {
3433             Error(S, "invalid stream id: only 2-bit values are legal");
3434             break;
3435           }
3436           Imm16Val |= (StreamId << STREAM_ID_SHIFT_);
3437         }
3438       } while (false);
3439     }
3440     break;
3441   }
3442   Operands.push_back(AMDGPUOperand::CreateImm(this, Imm16Val, S, AMDGPUOperand::ImmTySendMsg));
3443   return MatchOperand_Success;
3444 }
3445 
3446 bool AMDGPUOperand::isSendMsg() const {
3447   return isImmTy(ImmTySendMsg);
3448 }
3449 
3450 //===----------------------------------------------------------------------===//
3451 // parser helpers
3452 //===----------------------------------------------------------------------===//
3453 
3454 bool
3455 AMDGPUAsmParser::trySkipId(const StringRef Id) {
3456   if (getLexer().getKind() == AsmToken::Identifier &&
3457       Parser.getTok().getString() == Id) {
3458     Parser.Lex();
3459     return true;
3460   }
3461   return false;
3462 }
3463 
3464 bool
3465 AMDGPUAsmParser::trySkipToken(const AsmToken::TokenKind Kind) {
3466   if (getLexer().getKind() == Kind) {
3467     Parser.Lex();
3468     return true;
3469   }
3470   return false;
3471 }
3472 
3473 bool
3474 AMDGPUAsmParser::skipToken(const AsmToken::TokenKind Kind,
3475                            const StringRef ErrMsg) {
3476   if (!trySkipToken(Kind)) {
3477     Error(Parser.getTok().getLoc(), ErrMsg);
3478     return false;
3479   }
3480   return true;
3481 }
3482 
3483 bool
3484 AMDGPUAsmParser::parseExpr(int64_t &Imm) {
3485   return !getParser().parseAbsoluteExpression(Imm);
3486 }
3487 
3488 bool
3489 AMDGPUAsmParser::parseString(StringRef &Val, const StringRef ErrMsg) {
3490   SMLoc S = Parser.getTok().getLoc();
3491   if (getLexer().getKind() == AsmToken::String) {
3492     Val = Parser.getTok().getStringContents();
3493     Parser.Lex();
3494     return true;
3495   } else {
3496     Error(S, ErrMsg);
3497     return false;
3498   }
3499 }
3500 
3501 //===----------------------------------------------------------------------===//
3502 // swizzle
3503 //===----------------------------------------------------------------------===//
3504 
3505 LLVM_READNONE
3506 static unsigned
3507 encodeBitmaskPerm(const unsigned AndMask,
3508                   const unsigned OrMask,
3509                   const unsigned XorMask) {
3510   using namespace llvm::AMDGPU::Swizzle;
3511 
3512   return BITMASK_PERM_ENC |
3513          (AndMask << BITMASK_AND_SHIFT) |
3514          (OrMask  << BITMASK_OR_SHIFT)  |
3515          (XorMask << BITMASK_XOR_SHIFT);
3516 }
3517 
3518 bool
3519 AMDGPUAsmParser::parseSwizzleOperands(const unsigned OpNum, int64_t* Op,
3520                                       const unsigned MinVal,
3521                                       const unsigned MaxVal,
3522                                       const StringRef ErrMsg) {
3523   for (unsigned i = 0; i < OpNum; ++i) {
3524     if (!skipToken(AsmToken::Comma, "expected a comma")){
3525       return false;
3526     }
3527     SMLoc ExprLoc = Parser.getTok().getLoc();
3528     if (!parseExpr(Op[i])) {
3529       return false;
3530     }
3531     if (Op[i] < MinVal || Op[i] > MaxVal) {
3532       Error(ExprLoc, ErrMsg);
3533       return false;
3534     }
3535   }
3536 
3537   return true;
3538 }
3539 
3540 bool
3541 AMDGPUAsmParser::parseSwizzleQuadPerm(int64_t &Imm) {
3542   using namespace llvm::AMDGPU::Swizzle;
3543 
3544   int64_t Lane[LANE_NUM];
3545   if (parseSwizzleOperands(LANE_NUM, Lane, 0, LANE_MAX,
3546                            "expected a 2-bit lane id")) {
3547     Imm = QUAD_PERM_ENC;
3548     for (auto i = 0; i < LANE_NUM; ++i) {
3549       Imm |= Lane[i] << (LANE_SHIFT * i);
3550     }
3551     return true;
3552   }
3553   return false;
3554 }
3555 
3556 bool
3557 AMDGPUAsmParser::parseSwizzleBroadcast(int64_t &Imm) {
3558   using namespace llvm::AMDGPU::Swizzle;
3559 
3560   SMLoc S = Parser.getTok().getLoc();
3561   int64_t GroupSize;
3562   int64_t LaneIdx;
3563 
3564   if (!parseSwizzleOperands(1, &GroupSize,
3565                             2, 32,
3566                             "group size must be in the interval [2,32]")) {
3567     return false;
3568   }
3569   if (!isPowerOf2_64(GroupSize)) {
3570     Error(S, "group size must be a power of two");
3571     return false;
3572   }
3573   if (parseSwizzleOperands(1, &LaneIdx,
3574                            0, GroupSize - 1,
3575                            "lane id must be in the interval [0,group size - 1]")) {
3576     Imm = encodeBitmaskPerm(BITMASK_MAX - GroupSize + 1, LaneIdx, 0);
3577     return true;
3578   }
3579   return false;
3580 }
3581 
3582 bool
3583 AMDGPUAsmParser::parseSwizzleReverse(int64_t &Imm) {
3584   using namespace llvm::AMDGPU::Swizzle;
3585 
3586   SMLoc S = Parser.getTok().getLoc();
3587   int64_t GroupSize;
3588 
3589   if (!parseSwizzleOperands(1, &GroupSize,
3590       2, 32, "group size must be in the interval [2,32]")) {
3591     return false;
3592   }
3593   if (!isPowerOf2_64(GroupSize)) {
3594     Error(S, "group size must be a power of two");
3595     return false;
3596   }
3597 
3598   Imm = encodeBitmaskPerm(BITMASK_MAX, 0, GroupSize - 1);
3599   return true;
3600 }
3601 
3602 bool
3603 AMDGPUAsmParser::parseSwizzleSwap(int64_t &Imm) {
3604   using namespace llvm::AMDGPU::Swizzle;
3605 
3606   SMLoc S = Parser.getTok().getLoc();
3607   int64_t GroupSize;
3608 
3609   if (!parseSwizzleOperands(1, &GroupSize,
3610       1, 16, "group size must be in the interval [1,16]")) {
3611     return false;
3612   }
3613   if (!isPowerOf2_64(GroupSize)) {
3614     Error(S, "group size must be a power of two");
3615     return false;
3616   }
3617 
3618   Imm = encodeBitmaskPerm(BITMASK_MAX, 0, GroupSize);
3619   return true;
3620 }
3621 
3622 bool
3623 AMDGPUAsmParser::parseSwizzleBitmaskPerm(int64_t &Imm) {
3624   using namespace llvm::AMDGPU::Swizzle;
3625 
3626   if (!skipToken(AsmToken::Comma, "expected a comma")) {
3627     return false;
3628   }
3629 
3630   StringRef Ctl;
3631   SMLoc StrLoc = Parser.getTok().getLoc();
3632   if (!parseString(Ctl)) {
3633     return false;
3634   }
3635   if (Ctl.size() != BITMASK_WIDTH) {
3636     Error(StrLoc, "expected a 5-character mask");
3637     return false;
3638   }
3639 
3640   unsigned AndMask = 0;
3641   unsigned OrMask = 0;
3642   unsigned XorMask = 0;
3643 
3644   for (size_t i = 0; i < Ctl.size(); ++i) {
3645     unsigned Mask = 1 << (BITMASK_WIDTH - 1 - i);
3646     switch(Ctl[i]) {
3647     default:
3648       Error(StrLoc, "invalid mask");
3649       return false;
3650     case '0':
3651       break;
3652     case '1':
3653       OrMask |= Mask;
3654       break;
3655     case 'p':
3656       AndMask |= Mask;
3657       break;
3658     case 'i':
3659       AndMask |= Mask;
3660       XorMask |= Mask;
3661       break;
3662     }
3663   }
3664 
3665   Imm = encodeBitmaskPerm(AndMask, OrMask, XorMask);
3666   return true;
3667 }
3668 
3669 bool
3670 AMDGPUAsmParser::parseSwizzleOffset(int64_t &Imm) {
3671 
3672   SMLoc OffsetLoc = Parser.getTok().getLoc();
3673 
3674   if (!parseExpr(Imm)) {
3675     return false;
3676   }
3677   if (!isUInt<16>(Imm)) {
3678     Error(OffsetLoc, "expected a 16-bit offset");
3679     return false;
3680   }
3681   return true;
3682 }
3683 
3684 bool
3685 AMDGPUAsmParser::parseSwizzleMacro(int64_t &Imm) {
3686   using namespace llvm::AMDGPU::Swizzle;
3687 
3688   if (skipToken(AsmToken::LParen, "expected a left parentheses")) {
3689 
3690     SMLoc ModeLoc = Parser.getTok().getLoc();
3691     bool Ok = false;
3692 
3693     if (trySkipId(IdSymbolic[ID_QUAD_PERM])) {
3694       Ok = parseSwizzleQuadPerm(Imm);
3695     } else if (trySkipId(IdSymbolic[ID_BITMASK_PERM])) {
3696       Ok = parseSwizzleBitmaskPerm(Imm);
3697     } else if (trySkipId(IdSymbolic[ID_BROADCAST])) {
3698       Ok = parseSwizzleBroadcast(Imm);
3699     } else if (trySkipId(IdSymbolic[ID_SWAP])) {
3700       Ok = parseSwizzleSwap(Imm);
3701     } else if (trySkipId(IdSymbolic[ID_REVERSE])) {
3702       Ok = parseSwizzleReverse(Imm);
3703     } else {
3704       Error(ModeLoc, "expected a swizzle mode");
3705     }
3706 
3707     return Ok && skipToken(AsmToken::RParen, "expected a closing parentheses");
3708   }
3709 
3710   return false;
3711 }
3712 
3713 OperandMatchResultTy
3714 AMDGPUAsmParser::parseSwizzleOp(OperandVector &Operands) {
3715   SMLoc S = Parser.getTok().getLoc();
3716   int64_t Imm = 0;
3717 
3718   if (trySkipId("offset")) {
3719 
3720     bool Ok = false;
3721     if (skipToken(AsmToken::Colon, "expected a colon")) {
3722       if (trySkipId("swizzle")) {
3723         Ok = parseSwizzleMacro(Imm);
3724       } else {
3725         Ok = parseSwizzleOffset(Imm);
3726       }
3727     }
3728 
3729     Operands.push_back(AMDGPUOperand::CreateImm(this, Imm, S, AMDGPUOperand::ImmTySwizzle));
3730 
3731     return Ok? MatchOperand_Success : MatchOperand_ParseFail;
3732   } else {
3733     return MatchOperand_NoMatch;
3734   }
3735 }
3736 
3737 bool
3738 AMDGPUOperand::isSwizzle() const {
3739   return isImmTy(ImmTySwizzle);
3740 }
3741 
3742 //===----------------------------------------------------------------------===//
3743 // sopp branch targets
3744 //===----------------------------------------------------------------------===//
3745 
3746 OperandMatchResultTy
3747 AMDGPUAsmParser::parseSOppBrTarget(OperandVector &Operands) {
3748   SMLoc S = Parser.getTok().getLoc();
3749 
3750   switch (getLexer().getKind()) {
3751     default: return MatchOperand_ParseFail;
3752     case AsmToken::Integer: {
3753       int64_t Imm;
3754       if (getParser().parseAbsoluteExpression(Imm))
3755         return MatchOperand_ParseFail;
3756       Operands.push_back(AMDGPUOperand::CreateImm(this, Imm, S));
3757       return MatchOperand_Success;
3758     }
3759 
3760     case AsmToken::Identifier:
3761       Operands.push_back(AMDGPUOperand::CreateExpr(this,
3762           MCSymbolRefExpr::create(getContext().getOrCreateSymbol(
3763                                   Parser.getTok().getString()), getContext()), S));
3764       Parser.Lex();
3765       return MatchOperand_Success;
3766   }
3767 }
3768 
3769 //===----------------------------------------------------------------------===//
3770 // mubuf
3771 //===----------------------------------------------------------------------===//
3772 
3773 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultGLC() const {
3774   return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTyGLC);
3775 }
3776 
3777 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultSLC() const {
3778   return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTySLC);
3779 }
3780 
3781 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultTFE() const {
3782   return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTyTFE);
3783 }
3784 
3785 void AMDGPUAsmParser::cvtMubufImpl(MCInst &Inst,
3786                                const OperandVector &Operands,
3787                                bool IsAtomic, bool IsAtomicReturn) {
3788   OptionalImmIndexMap OptionalIdx;
3789   assert(IsAtomicReturn ? IsAtomic : true);
3790 
3791   for (unsigned i = 1, e = Operands.size(); i != e; ++i) {
3792     AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[i]);
3793 
3794     // Add the register arguments
3795     if (Op.isReg()) {
3796       Op.addRegOperands(Inst, 1);
3797       continue;
3798     }
3799 
3800     // Handle the case where soffset is an immediate
3801     if (Op.isImm() && Op.getImmTy() == AMDGPUOperand::ImmTyNone) {
3802       Op.addImmOperands(Inst, 1);
3803       continue;
3804     }
3805 
3806     // Handle tokens like 'offen' which are sometimes hard-coded into the
3807     // asm string.  There are no MCInst operands for these.
3808     if (Op.isToken()) {
3809       continue;
3810     }
3811     assert(Op.isImm());
3812 
3813     // Handle optional arguments
3814     OptionalIdx[Op.getImmTy()] = i;
3815   }
3816 
3817   // Copy $vdata_in operand and insert as $vdata for MUBUF_Atomic RTN insns.
3818   if (IsAtomicReturn) {
3819     MCInst::iterator I = Inst.begin(); // $vdata_in is always at the beginning.
3820     Inst.insert(I, *I);
3821   }
3822 
3823   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyOffset);
3824   if (!IsAtomic) { // glc is hard-coded.
3825     addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyGLC);
3826   }
3827   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySLC);
3828   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyTFE);
3829 }
3830 
3831 void AMDGPUAsmParser::cvtMtbuf(MCInst &Inst, const OperandVector &Operands) {
3832   OptionalImmIndexMap OptionalIdx;
3833 
3834   for (unsigned i = 1, e = Operands.size(); i != e; ++i) {
3835     AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[i]);
3836 
3837     // Add the register arguments
3838     if (Op.isReg()) {
3839       Op.addRegOperands(Inst, 1);
3840       continue;
3841     }
3842 
3843     // Handle the case where soffset is an immediate
3844     if (Op.isImm() && Op.getImmTy() == AMDGPUOperand::ImmTyNone) {
3845       Op.addImmOperands(Inst, 1);
3846       continue;
3847     }
3848 
3849     // Handle tokens like 'offen' which are sometimes hard-coded into the
3850     // asm string.  There are no MCInst operands for these.
3851     if (Op.isToken()) {
3852       continue;
3853     }
3854     assert(Op.isImm());
3855 
3856     // Handle optional arguments
3857     OptionalIdx[Op.getImmTy()] = i;
3858   }
3859 
3860   addOptionalImmOperand(Inst, Operands, OptionalIdx,
3861                         AMDGPUOperand::ImmTyOffset);
3862   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyDFMT);
3863   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyNFMT);
3864   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyGLC);
3865   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySLC);
3866   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyTFE);
3867 }
3868 
3869 //===----------------------------------------------------------------------===//
3870 // mimg
3871 //===----------------------------------------------------------------------===//
3872 
3873 void AMDGPUAsmParser::cvtMIMG(MCInst &Inst, const OperandVector &Operands) {
3874   unsigned I = 1;
3875   const MCInstrDesc &Desc = MII.get(Inst.getOpcode());
3876   for (unsigned J = 0; J < Desc.getNumDefs(); ++J) {
3877     ((AMDGPUOperand &)*Operands[I++]).addRegOperands(Inst, 1);
3878   }
3879 
3880   OptionalImmIndexMap OptionalIdx;
3881 
3882   for (unsigned E = Operands.size(); I != E; ++I) {
3883     AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[I]);
3884 
3885     // Add the register arguments
3886     if (Op.isRegOrImm()) {
3887       Op.addRegOrImmOperands(Inst, 1);
3888       continue;
3889     } else if (Op.isImmModifier()) {
3890       OptionalIdx[Op.getImmTy()] = I;
3891     } else {
3892       llvm_unreachable("unexpected operand type");
3893     }
3894   }
3895 
3896   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyDMask);
3897   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyUNorm);
3898   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyGLC);
3899   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyDA);
3900   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyR128);
3901   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyTFE);
3902   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyLWE);
3903   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySLC);
3904 }
3905 
3906 void AMDGPUAsmParser::cvtMIMGAtomic(MCInst &Inst, const OperandVector &Operands) {
3907   unsigned I = 1;
3908   const MCInstrDesc &Desc = MII.get(Inst.getOpcode());
3909   for (unsigned J = 0; J < Desc.getNumDefs(); ++J) {
3910     ((AMDGPUOperand &)*Operands[I++]).addRegOperands(Inst, 1);
3911   }
3912 
3913   // Add src, same as dst
3914   ((AMDGPUOperand &)*Operands[I]).addRegOperands(Inst, 1);
3915 
3916   OptionalImmIndexMap OptionalIdx;
3917 
3918   for (unsigned E = Operands.size(); I != E; ++I) {
3919     AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[I]);
3920 
3921     // Add the register arguments
3922     if (Op.isRegOrImm()) {
3923       Op.addRegOrImmOperands(Inst, 1);
3924       continue;
3925     } else if (Op.isImmModifier()) {
3926       OptionalIdx[Op.getImmTy()] = I;
3927     } else {
3928       llvm_unreachable("unexpected operand type");
3929     }
3930   }
3931 
3932   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyDMask);
3933   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyUNorm);
3934   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyGLC);
3935   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyDA);
3936   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyR128);
3937   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyTFE);
3938   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyLWE);
3939   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySLC);
3940 }
3941 
3942 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultDMask() const {
3943   return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTyDMask);
3944 }
3945 
3946 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultUNorm() const {
3947   return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTyUNorm);
3948 }
3949 
3950 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultDA() const {
3951   return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTyDA);
3952 }
3953 
3954 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultR128() const {
3955   return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTyR128);
3956 }
3957 
3958 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultLWE() const {
3959   return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTyLWE);
3960 }
3961 
3962 //===----------------------------------------------------------------------===//
3963 // smrd
3964 //===----------------------------------------------------------------------===//
3965 
3966 bool AMDGPUOperand::isSMRDOffset8() const {
3967   return isImm() && isUInt<8>(getImm());
3968 }
3969 
3970 bool AMDGPUOperand::isSMRDOffset20() const {
3971   return isImm() && isUInt<20>(getImm());
3972 }
3973 
3974 bool AMDGPUOperand::isSMRDLiteralOffset() const {
3975   // 32-bit literals are only supported on CI and we only want to use them
3976   // when the offset is > 8-bits.
3977   return isImm() && !isUInt<8>(getImm()) && isUInt<32>(getImm());
3978 }
3979 
3980 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultSMRDOffset8() const {
3981   return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTyOffset);
3982 }
3983 
3984 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultSMRDOffset20() const {
3985   return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTyOffset);
3986 }
3987 
3988 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultSMRDLiteralOffset() const {
3989   return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTyOffset);
3990 }
3991 
3992 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultOffsetU12() const {
3993   return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTyOffset);
3994 }
3995 
3996 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultOffsetS13() const {
3997   return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTyOffset);
3998 }
3999 
4000 //===----------------------------------------------------------------------===//
4001 // vop3
4002 //===----------------------------------------------------------------------===//
4003 
4004 static bool ConvertOmodMul(int64_t &Mul) {
4005   if (Mul != 1 && Mul != 2 && Mul != 4)
4006     return false;
4007 
4008   Mul >>= 1;
4009   return true;
4010 }
4011 
4012 static bool ConvertOmodDiv(int64_t &Div) {
4013   if (Div == 1) {
4014     Div = 0;
4015     return true;
4016   }
4017 
4018   if (Div == 2) {
4019     Div = 3;
4020     return true;
4021   }
4022 
4023   return false;
4024 }
4025 
4026 static bool ConvertBoundCtrl(int64_t &BoundCtrl) {
4027   if (BoundCtrl == 0) {
4028     BoundCtrl = 1;
4029     return true;
4030   }
4031 
4032   if (BoundCtrl == -1) {
4033     BoundCtrl = 0;
4034     return true;
4035   }
4036 
4037   return false;
4038 }
4039 
4040 // Note: the order in this table matches the order of operands in AsmString.
4041 static const OptionalOperand AMDGPUOptionalOperandTable[] = {
4042   {"offen",   AMDGPUOperand::ImmTyOffen, true, nullptr},
4043   {"idxen",   AMDGPUOperand::ImmTyIdxen, true, nullptr},
4044   {"addr64",  AMDGPUOperand::ImmTyAddr64, true, nullptr},
4045   {"offset0", AMDGPUOperand::ImmTyOffset0, false, nullptr},
4046   {"offset1", AMDGPUOperand::ImmTyOffset1, false, nullptr},
4047   {"gds",     AMDGPUOperand::ImmTyGDS, true, nullptr},
4048   {"offset",  AMDGPUOperand::ImmTyOffset, false, nullptr},
4049   {"dfmt",    AMDGPUOperand::ImmTyDFMT, false, nullptr},
4050   {"nfmt",    AMDGPUOperand::ImmTyNFMT, false, nullptr},
4051   {"glc",     AMDGPUOperand::ImmTyGLC, true, nullptr},
4052   {"slc",     AMDGPUOperand::ImmTySLC, true, nullptr},
4053   {"tfe",     AMDGPUOperand::ImmTyTFE, true, nullptr},
4054   {"clamp",   AMDGPUOperand::ImmTyClampSI, true, nullptr},
4055   {"omod",    AMDGPUOperand::ImmTyOModSI, false, ConvertOmodMul},
4056   {"unorm",   AMDGPUOperand::ImmTyUNorm, true, nullptr},
4057   {"da",      AMDGPUOperand::ImmTyDA,    true, nullptr},
4058   {"r128",    AMDGPUOperand::ImmTyR128,  true, nullptr},
4059   {"lwe",     AMDGPUOperand::ImmTyLWE,   true, nullptr},
4060   {"dmask",   AMDGPUOperand::ImmTyDMask, false, nullptr},
4061   {"row_mask",   AMDGPUOperand::ImmTyDppRowMask, false, nullptr},
4062   {"bank_mask",  AMDGPUOperand::ImmTyDppBankMask, false, nullptr},
4063   {"bound_ctrl", AMDGPUOperand::ImmTyDppBoundCtrl, false, ConvertBoundCtrl},
4064   {"dst_sel",    AMDGPUOperand::ImmTySdwaDstSel, false, nullptr},
4065   {"src0_sel",   AMDGPUOperand::ImmTySdwaSrc0Sel, false, nullptr},
4066   {"src1_sel",   AMDGPUOperand::ImmTySdwaSrc1Sel, false, nullptr},
4067   {"dst_unused", AMDGPUOperand::ImmTySdwaDstUnused, false, nullptr},
4068   {"compr", AMDGPUOperand::ImmTyExpCompr, true, nullptr },
4069   {"vm", AMDGPUOperand::ImmTyExpVM, true, nullptr},
4070   {"op_sel", AMDGPUOperand::ImmTyOpSel, false, nullptr},
4071   {"op_sel_hi", AMDGPUOperand::ImmTyOpSelHi, false, nullptr},
4072   {"neg_lo", AMDGPUOperand::ImmTyNegLo, false, nullptr},
4073   {"neg_hi", AMDGPUOperand::ImmTyNegHi, false, nullptr}
4074 };
4075 
4076 OperandMatchResultTy AMDGPUAsmParser::parseOptionalOperand(OperandVector &Operands) {
4077   OperandMatchResultTy res;
4078   for (const OptionalOperand &Op : AMDGPUOptionalOperandTable) {
4079     // try to parse any optional operand here
4080     if (Op.IsBit) {
4081       res = parseNamedBit(Op.Name, Operands, Op.Type);
4082     } else if (Op.Type == AMDGPUOperand::ImmTyOModSI) {
4083       res = parseOModOperand(Operands);
4084     } else if (Op.Type == AMDGPUOperand::ImmTySdwaDstSel ||
4085                Op.Type == AMDGPUOperand::ImmTySdwaSrc0Sel ||
4086                Op.Type == AMDGPUOperand::ImmTySdwaSrc1Sel) {
4087       res = parseSDWASel(Operands, Op.Name, Op.Type);
4088     } else if (Op.Type == AMDGPUOperand::ImmTySdwaDstUnused) {
4089       res = parseSDWADstUnused(Operands);
4090     } else if (Op.Type == AMDGPUOperand::ImmTyOpSel ||
4091                Op.Type == AMDGPUOperand::ImmTyOpSelHi ||
4092                Op.Type == AMDGPUOperand::ImmTyNegLo ||
4093                Op.Type == AMDGPUOperand::ImmTyNegHi) {
4094       res = parseOperandArrayWithPrefix(Op.Name, Operands, Op.Type,
4095                                         Op.ConvertResult);
4096     } else {
4097       res = parseIntWithPrefix(Op.Name, Operands, Op.Type, Op.ConvertResult);
4098     }
4099     if (res != MatchOperand_NoMatch) {
4100       return res;
4101     }
4102   }
4103   return MatchOperand_NoMatch;
4104 }
4105 
4106 OperandMatchResultTy AMDGPUAsmParser::parseOModOperand(OperandVector &Operands) {
4107   StringRef Name = Parser.getTok().getString();
4108   if (Name == "mul") {
4109     return parseIntWithPrefix("mul", Operands,
4110                               AMDGPUOperand::ImmTyOModSI, ConvertOmodMul);
4111   }
4112 
4113   if (Name == "div") {
4114     return parseIntWithPrefix("div", Operands,
4115                               AMDGPUOperand::ImmTyOModSI, ConvertOmodDiv);
4116   }
4117 
4118   return MatchOperand_NoMatch;
4119 }
4120 
4121 void AMDGPUAsmParser::cvtId(MCInst &Inst, const OperandVector &Operands) {
4122   unsigned I = 1;
4123   const MCInstrDesc &Desc = MII.get(Inst.getOpcode());
4124   for (unsigned J = 0; J < Desc.getNumDefs(); ++J) {
4125     ((AMDGPUOperand &)*Operands[I++]).addRegOperands(Inst, 1);
4126   }
4127   for (unsigned E = Operands.size(); I != E; ++I)
4128     ((AMDGPUOperand &)*Operands[I]).addRegOrImmOperands(Inst, 1);
4129 }
4130 
4131 void AMDGPUAsmParser::cvtVOP3_2_mod(MCInst &Inst, const OperandVector &Operands) {
4132   uint64_t TSFlags = MII.get(Inst.getOpcode()).TSFlags;
4133   if (TSFlags & SIInstrFlags::VOP3) {
4134     cvtVOP3(Inst, Operands);
4135   } else {
4136     cvtId(Inst, Operands);
4137   }
4138 }
4139 
4140 static bool isRegOrImmWithInputMods(const MCInstrDesc &Desc, unsigned OpNum) {
4141       // 1. This operand is input modifiers
4142   return Desc.OpInfo[OpNum].OperandType == AMDGPU::OPERAND_INPUT_MODS
4143       // 2. This is not last operand
4144       && Desc.NumOperands > (OpNum + 1)
4145       // 3. Next operand is register class
4146       && Desc.OpInfo[OpNum + 1].RegClass != -1
4147       // 4. Next register is not tied to any other operand
4148       && Desc.getOperandConstraint(OpNum + 1, MCOI::OperandConstraint::TIED_TO) == -1;
4149 }
4150 
4151 void AMDGPUAsmParser::cvtVOP3Impl(MCInst &Inst, const OperandVector &Operands,
4152                                   OptionalImmIndexMap &OptionalIdx) {
4153   unsigned I = 1;
4154   const MCInstrDesc &Desc = MII.get(Inst.getOpcode());
4155   for (unsigned J = 0; J < Desc.getNumDefs(); ++J) {
4156     ((AMDGPUOperand &)*Operands[I++]).addRegOperands(Inst, 1);
4157   }
4158 
4159   for (unsigned E = Operands.size(); I != E; ++I) {
4160     AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[I]);
4161     if (isRegOrImmWithInputMods(Desc, Inst.getNumOperands())) {
4162       Op.addRegOrImmWithFPInputModsOperands(Inst, 2);
4163     } else if (Op.isImmModifier()) {
4164       OptionalIdx[Op.getImmTy()] = I;
4165     } else if (Op.isRegOrImm()) {
4166       Op.addRegOrImmOperands(Inst, 1);
4167     } else {
4168       llvm_unreachable("unhandled operand type");
4169     }
4170   }
4171 }
4172 
4173 void AMDGPUAsmParser::cvtVOP3(MCInst &Inst, const OperandVector &Operands) {
4174   OptionalImmIndexMap OptionalIdx;
4175 
4176   cvtVOP3Impl(Inst, Operands, OptionalIdx);
4177 
4178   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyClampSI);
4179   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyOModSI);
4180 
4181   // special case v_mac_{f16, f32}:
4182   // it has src2 register operand that is tied to dst operand
4183   // we don't allow modifiers for this operand in assembler so src2_modifiers
4184   // should be 0
4185   if (Inst.getOpcode() == AMDGPU::V_MAC_F32_e64_si ||
4186       Inst.getOpcode() == AMDGPU::V_MAC_F32_e64_vi ||
4187       Inst.getOpcode() == AMDGPU::V_MAC_F16_e64_vi) {
4188     auto it = Inst.begin();
4189     std::advance(
4190       it,
4191       AMDGPU::getNamedOperandIdx(Inst.getOpcode() == AMDGPU::V_MAC_F16_e64_vi ?
4192                                      AMDGPU::V_MAC_F16_e64 :
4193                                      AMDGPU::V_MAC_F32_e64,
4194                                  AMDGPU::OpName::src2_modifiers));
4195     it = Inst.insert(it, MCOperand::createImm(0)); // no modifiers for src2
4196     ++it;
4197     Inst.insert(it, Inst.getOperand(0)); // src2 = dst
4198   }
4199 }
4200 
4201 void AMDGPUAsmParser::cvtVOP3OMod(MCInst &Inst, const OperandVector &Operands) {
4202   OptionalImmIndexMap OptionalIdx;
4203 
4204   unsigned I = 1;
4205   const MCInstrDesc &Desc = MII.get(Inst.getOpcode());
4206   for (unsigned J = 0; J < Desc.getNumDefs(); ++J) {
4207     ((AMDGPUOperand &)*Operands[I++]).addRegOperands(Inst, 1);
4208   }
4209 
4210   for (unsigned E = Operands.size(); I != E; ++I) {
4211     AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[I]);
4212     if (Op.isMod()) {
4213       OptionalIdx[Op.getImmTy()] = I;
4214     } else {
4215       Op.addRegOrImmOperands(Inst, 1);
4216     }
4217   }
4218 
4219   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyClampSI);
4220   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyOModSI);
4221 }
4222 
4223 void AMDGPUAsmParser::cvtVOP3P(MCInst &Inst, const OperandVector &Operands) {
4224   OptionalImmIndexMap OptIdx;
4225 
4226   cvtVOP3Impl(Inst, Operands, OptIdx);
4227 
4228   // FIXME: This is messy. Parse the modifiers as if it was a normal VOP3
4229   // instruction, and then figure out where to actually put the modifiers
4230   int Opc = Inst.getOpcode();
4231 
4232   if (AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::clamp) != -1) {
4233     addOptionalImmOperand(Inst, Operands, OptIdx, AMDGPUOperand::ImmTyClampSI);
4234   }
4235 
4236   addOptionalImmOperand(Inst, Operands, OptIdx, AMDGPUOperand::ImmTyOpSel);
4237   addOptionalImmOperand(Inst, Operands, OptIdx, AMDGPUOperand::ImmTyOpSelHi, -1);
4238 
4239   int NegLoIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::neg_lo);
4240   if (NegLoIdx != -1) {
4241     addOptionalImmOperand(Inst, Operands, OptIdx, AMDGPUOperand::ImmTyNegLo);
4242     addOptionalImmOperand(Inst, Operands, OptIdx, AMDGPUOperand::ImmTyNegHi);
4243   }
4244 
4245   const int Ops[] = { AMDGPU::OpName::src0,
4246                       AMDGPU::OpName::src1,
4247                       AMDGPU::OpName::src2 };
4248   const int ModOps[] = { AMDGPU::OpName::src0_modifiers,
4249                          AMDGPU::OpName::src1_modifiers,
4250                          AMDGPU::OpName::src2_modifiers };
4251 
4252   int OpSelIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::op_sel);
4253   int OpSelHiIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::op_sel_hi);
4254 
4255   unsigned OpSel = Inst.getOperand(OpSelIdx).getImm();
4256   unsigned OpSelHi = Inst.getOperand(OpSelHiIdx).getImm();
4257   unsigned NegLo = 0;
4258   unsigned NegHi = 0;
4259 
4260   if (NegLoIdx != -1) {
4261     int NegHiIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::neg_hi);
4262     NegLo = Inst.getOperand(NegLoIdx).getImm();
4263     NegHi = Inst.getOperand(NegHiIdx).getImm();
4264   }
4265 
4266   for (int J = 0; J < 3; ++J) {
4267     int OpIdx = AMDGPU::getNamedOperandIdx(Opc, Ops[J]);
4268     if (OpIdx == -1)
4269       break;
4270 
4271     uint32_t ModVal = 0;
4272 
4273     if ((OpSel & (1 << J)) != 0)
4274       ModVal |= SISrcMods::OP_SEL_0;
4275 
4276     if ((OpSelHi & (1 << J)) != 0)
4277       ModVal |= SISrcMods::OP_SEL_1;
4278 
4279     if ((NegLo & (1 << J)) != 0)
4280       ModVal |= SISrcMods::NEG;
4281 
4282     if ((NegHi & (1 << J)) != 0)
4283       ModVal |= SISrcMods::NEG_HI;
4284 
4285     int ModIdx = AMDGPU::getNamedOperandIdx(Opc, ModOps[J]);
4286 
4287     Inst.getOperand(ModIdx).setImm(ModVal);
4288   }
4289 }
4290 
4291 //===----------------------------------------------------------------------===//
4292 // dpp
4293 //===----------------------------------------------------------------------===//
4294 
4295 bool AMDGPUOperand::isDPPCtrl() const {
4296   bool result = isImm() && getImmTy() == ImmTyDppCtrl && isUInt<9>(getImm());
4297   if (result) {
4298     int64_t Imm = getImm();
4299     return ((Imm >= 0x000) && (Imm <= 0x0ff)) ||
4300            ((Imm >= 0x101) && (Imm <= 0x10f)) ||
4301            ((Imm >= 0x111) && (Imm <= 0x11f)) ||
4302            ((Imm >= 0x121) && (Imm <= 0x12f)) ||
4303            (Imm == 0x130) ||
4304            (Imm == 0x134) ||
4305            (Imm == 0x138) ||
4306            (Imm == 0x13c) ||
4307            (Imm == 0x140) ||
4308            (Imm == 0x141) ||
4309            (Imm == 0x142) ||
4310            (Imm == 0x143);
4311   }
4312   return false;
4313 }
4314 
4315 bool AMDGPUOperand::isGPRIdxMode() const {
4316   return isImm() && isUInt<4>(getImm());
4317 }
4318 
4319 bool AMDGPUOperand::isS16Imm() const {
4320   return isImm() && (isInt<16>(getImm()) || isUInt<16>(getImm()));
4321 }
4322 
4323 bool AMDGPUOperand::isU16Imm() const {
4324   return isImm() && isUInt<16>(getImm());
4325 }
4326 
4327 OperandMatchResultTy
4328 AMDGPUAsmParser::parseDPPCtrl(OperandVector &Operands) {
4329   SMLoc S = Parser.getTok().getLoc();
4330   StringRef Prefix;
4331   int64_t Int;
4332 
4333   if (getLexer().getKind() == AsmToken::Identifier) {
4334     Prefix = Parser.getTok().getString();
4335   } else {
4336     return MatchOperand_NoMatch;
4337   }
4338 
4339   if (Prefix == "row_mirror") {
4340     Int = 0x140;
4341     Parser.Lex();
4342   } else if (Prefix == "row_half_mirror") {
4343     Int = 0x141;
4344     Parser.Lex();
4345   } else {
4346     // Check to prevent parseDPPCtrlOps from eating invalid tokens
4347     if (Prefix != "quad_perm"
4348         && Prefix != "row_shl"
4349         && Prefix != "row_shr"
4350         && Prefix != "row_ror"
4351         && Prefix != "wave_shl"
4352         && Prefix != "wave_rol"
4353         && Prefix != "wave_shr"
4354         && Prefix != "wave_ror"
4355         && Prefix != "row_bcast") {
4356       return MatchOperand_NoMatch;
4357     }
4358 
4359     Parser.Lex();
4360     if (getLexer().isNot(AsmToken::Colon))
4361       return MatchOperand_ParseFail;
4362 
4363     if (Prefix == "quad_perm") {
4364       // quad_perm:[%d,%d,%d,%d]
4365       Parser.Lex();
4366       if (getLexer().isNot(AsmToken::LBrac))
4367         return MatchOperand_ParseFail;
4368       Parser.Lex();
4369 
4370       if (getParser().parseAbsoluteExpression(Int) || !(0 <= Int && Int <=3))
4371         return MatchOperand_ParseFail;
4372 
4373       for (int i = 0; i < 3; ++i) {
4374         if (getLexer().isNot(AsmToken::Comma))
4375           return MatchOperand_ParseFail;
4376         Parser.Lex();
4377 
4378         int64_t Temp;
4379         if (getParser().parseAbsoluteExpression(Temp) || !(0 <= Temp && Temp <=3))
4380           return MatchOperand_ParseFail;
4381         const int shift = i*2 + 2;
4382         Int += (Temp << shift);
4383       }
4384 
4385       if (getLexer().isNot(AsmToken::RBrac))
4386         return MatchOperand_ParseFail;
4387       Parser.Lex();
4388 
4389     } else {
4390       // sel:%d
4391       Parser.Lex();
4392       if (getParser().parseAbsoluteExpression(Int))
4393         return MatchOperand_ParseFail;
4394 
4395       if (Prefix == "row_shl" && 1 <= Int && Int <= 15) {
4396         Int |= 0x100;
4397       } else if (Prefix == "row_shr" && 1 <= Int && Int <= 15) {
4398         Int |= 0x110;
4399       } else if (Prefix == "row_ror" && 1 <= Int && Int <= 15) {
4400         Int |= 0x120;
4401       } else if (Prefix == "wave_shl" && 1 == Int) {
4402         Int = 0x130;
4403       } else if (Prefix == "wave_rol" && 1 == Int) {
4404         Int = 0x134;
4405       } else if (Prefix == "wave_shr" && 1 == Int) {
4406         Int = 0x138;
4407       } else if (Prefix == "wave_ror" && 1 == Int) {
4408         Int = 0x13C;
4409       } else if (Prefix == "row_bcast") {
4410         if (Int == 15) {
4411           Int = 0x142;
4412         } else if (Int == 31) {
4413           Int = 0x143;
4414         } else {
4415           return MatchOperand_ParseFail;
4416         }
4417       } else {
4418         return MatchOperand_ParseFail;
4419       }
4420     }
4421   }
4422 
4423   Operands.push_back(AMDGPUOperand::CreateImm(this, Int, S, AMDGPUOperand::ImmTyDppCtrl));
4424   return MatchOperand_Success;
4425 }
4426 
4427 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultRowMask() const {
4428   return AMDGPUOperand::CreateImm(this, 0xf, SMLoc(), AMDGPUOperand::ImmTyDppRowMask);
4429 }
4430 
4431 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultBankMask() const {
4432   return AMDGPUOperand::CreateImm(this, 0xf, SMLoc(), AMDGPUOperand::ImmTyDppBankMask);
4433 }
4434 
4435 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultBoundCtrl() const {
4436   return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTyDppBoundCtrl);
4437 }
4438 
4439 void AMDGPUAsmParser::cvtDPP(MCInst &Inst, const OperandVector &Operands) {
4440   OptionalImmIndexMap OptionalIdx;
4441 
4442   unsigned I = 1;
4443   const MCInstrDesc &Desc = MII.get(Inst.getOpcode());
4444   for (unsigned J = 0; J < Desc.getNumDefs(); ++J) {
4445     ((AMDGPUOperand &)*Operands[I++]).addRegOperands(Inst, 1);
4446   }
4447 
4448   for (unsigned E = Operands.size(); I != E; ++I) {
4449     AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[I]);
4450     // Add the register arguments
4451     if (Op.isReg() && Op.Reg.RegNo == AMDGPU::VCC) {
4452       // VOP2b (v_add_u32, v_sub_u32 ...) dpp use "vcc" token.
4453       // Skip it.
4454       continue;
4455     } if (isRegOrImmWithInputMods(Desc, Inst.getNumOperands())) {
4456       Op.addRegWithFPInputModsOperands(Inst, 2);
4457     } else if (Op.isDPPCtrl()) {
4458       Op.addImmOperands(Inst, 1);
4459     } else if (Op.isImm()) {
4460       // Handle optional arguments
4461       OptionalIdx[Op.getImmTy()] = I;
4462     } else {
4463       llvm_unreachable("Invalid operand type");
4464     }
4465   }
4466 
4467   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyDppRowMask, 0xf);
4468   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyDppBankMask, 0xf);
4469   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyDppBoundCtrl);
4470 
4471   // special case v_mac_{f16, f32}:
4472   // it has src2 register operand that is tied to dst operand
4473   if (Inst.getOpcode() == AMDGPU::V_MAC_F32_dpp ||
4474       Inst.getOpcode() == AMDGPU::V_MAC_F16_dpp) {
4475     auto it = Inst.begin();
4476     std::advance(
4477         it, AMDGPU::getNamedOperandIdx(Inst.getOpcode(), AMDGPU::OpName::src2));
4478     Inst.insert(it, Inst.getOperand(0)); // src2 = dst
4479   }
4480 }
4481 
4482 //===----------------------------------------------------------------------===//
4483 // sdwa
4484 //===----------------------------------------------------------------------===//
4485 
4486 OperandMatchResultTy
4487 AMDGPUAsmParser::parseSDWASel(OperandVector &Operands, StringRef Prefix,
4488                               AMDGPUOperand::ImmTy Type) {
4489   using namespace llvm::AMDGPU::SDWA;
4490 
4491   SMLoc S = Parser.getTok().getLoc();
4492   StringRef Value;
4493   OperandMatchResultTy res;
4494 
4495   res = parseStringWithPrefix(Prefix, Value);
4496   if (res != MatchOperand_Success) {
4497     return res;
4498   }
4499 
4500   int64_t Int;
4501   Int = StringSwitch<int64_t>(Value)
4502         .Case("BYTE_0", SdwaSel::BYTE_0)
4503         .Case("BYTE_1", SdwaSel::BYTE_1)
4504         .Case("BYTE_2", SdwaSel::BYTE_2)
4505         .Case("BYTE_3", SdwaSel::BYTE_3)
4506         .Case("WORD_0", SdwaSel::WORD_0)
4507         .Case("WORD_1", SdwaSel::WORD_1)
4508         .Case("DWORD", SdwaSel::DWORD)
4509         .Default(0xffffffff);
4510   Parser.Lex(); // eat last token
4511 
4512   if (Int == 0xffffffff) {
4513     return MatchOperand_ParseFail;
4514   }
4515 
4516   Operands.push_back(AMDGPUOperand::CreateImm(this, Int, S, Type));
4517   return MatchOperand_Success;
4518 }
4519 
4520 OperandMatchResultTy
4521 AMDGPUAsmParser::parseSDWADstUnused(OperandVector &Operands) {
4522   using namespace llvm::AMDGPU::SDWA;
4523 
4524   SMLoc S = Parser.getTok().getLoc();
4525   StringRef Value;
4526   OperandMatchResultTy res;
4527 
4528   res = parseStringWithPrefix("dst_unused", Value);
4529   if (res != MatchOperand_Success) {
4530     return res;
4531   }
4532 
4533   int64_t Int;
4534   Int = StringSwitch<int64_t>(Value)
4535         .Case("UNUSED_PAD", DstUnused::UNUSED_PAD)
4536         .Case("UNUSED_SEXT", DstUnused::UNUSED_SEXT)
4537         .Case("UNUSED_PRESERVE", DstUnused::UNUSED_PRESERVE)
4538         .Default(0xffffffff);
4539   Parser.Lex(); // eat last token
4540 
4541   if (Int == 0xffffffff) {
4542     return MatchOperand_ParseFail;
4543   }
4544 
4545   Operands.push_back(AMDGPUOperand::CreateImm(this, Int, S, AMDGPUOperand::ImmTySdwaDstUnused));
4546   return MatchOperand_Success;
4547 }
4548 
4549 void AMDGPUAsmParser::cvtSdwaVOP1(MCInst &Inst, const OperandVector &Operands) {
4550   cvtSDWA(Inst, Operands, SIInstrFlags::VOP1);
4551 }
4552 
4553 void AMDGPUAsmParser::cvtSdwaVOP2(MCInst &Inst, const OperandVector &Operands) {
4554   cvtSDWA(Inst, Operands, SIInstrFlags::VOP2);
4555 }
4556 
4557 void AMDGPUAsmParser::cvtSdwaVOP2b(MCInst &Inst, const OperandVector &Operands) {
4558   cvtSDWA(Inst, Operands, SIInstrFlags::VOP2, true);
4559 }
4560 
4561 void AMDGPUAsmParser::cvtSdwaVOPC(MCInst &Inst, const OperandVector &Operands) {
4562   cvtSDWA(Inst, Operands, SIInstrFlags::VOPC, isVI());
4563 }
4564 
4565 void AMDGPUAsmParser::cvtSDWA(MCInst &Inst, const OperandVector &Operands,
4566                               uint64_t BasicInstType, bool skipVcc) {
4567   using namespace llvm::AMDGPU::SDWA;
4568   OptionalImmIndexMap OptionalIdx;
4569   bool skippedVcc = false;
4570 
4571   unsigned I = 1;
4572   const MCInstrDesc &Desc = MII.get(Inst.getOpcode());
4573   for (unsigned J = 0; J < Desc.getNumDefs(); ++J) {
4574     ((AMDGPUOperand &)*Operands[I++]).addRegOperands(Inst, 1);
4575   }
4576 
4577   for (unsigned E = Operands.size(); I != E; ++I) {
4578     AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[I]);
4579     if (skipVcc && !skippedVcc && Op.isReg() && Op.Reg.RegNo == AMDGPU::VCC) {
4580       // VOP2b (v_add_u32, v_sub_u32 ...) sdwa use "vcc" token as dst.
4581       // Skip it if it's 2nd (e.g. v_add_i32_sdwa v1, vcc, v2, v3)
4582       // or 4th (v_addc_u32_sdwa v1, vcc, v2, v3, vcc) operand.
4583       // Skip VCC only if we didn't skip it on previous iteration.
4584       if (BasicInstType == SIInstrFlags::VOP2 &&
4585           (Inst.getNumOperands() == 1 || Inst.getNumOperands() == 5)) {
4586         skippedVcc = true;
4587         continue;
4588       } else if (BasicInstType == SIInstrFlags::VOPC &&
4589                  Inst.getNumOperands() == 0) {
4590         skippedVcc = true;
4591         continue;
4592       }
4593     }
4594     if (isRegOrImmWithInputMods(Desc, Inst.getNumOperands())) {
4595       Op.addRegWithInputModsOperands(Inst, 2);
4596     } else if (Op.isImm()) {
4597       // Handle optional arguments
4598       OptionalIdx[Op.getImmTy()] = I;
4599     } else {
4600       llvm_unreachable("Invalid operand type");
4601     }
4602     skippedVcc = false;
4603   }
4604 
4605   if (Inst.getOpcode() != AMDGPU::V_NOP_sdwa_gfx9 &&
4606       Inst.getOpcode() != AMDGPU::V_NOP_sdwa_vi) {
4607     // v_nop_sdwa_sdwa_vi/gfx9 has no optional sdwa arguments
4608     switch (BasicInstType) {
4609     case SIInstrFlags::VOP1:
4610       addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyClampSI, 0);
4611       if (AMDGPU::getNamedOperandIdx(Inst.getOpcode(), AMDGPU::OpName::omod) != -1) {
4612         addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyOModSI, 0);
4613       }
4614       addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySdwaDstSel, SdwaSel::DWORD);
4615       addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySdwaDstUnused, DstUnused::UNUSED_PRESERVE);
4616       addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySdwaSrc0Sel, SdwaSel::DWORD);
4617       break;
4618 
4619     case SIInstrFlags::VOP2:
4620       addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyClampSI, 0);
4621       if (AMDGPU::getNamedOperandIdx(Inst.getOpcode(), AMDGPU::OpName::omod) != -1) {
4622         addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyOModSI, 0);
4623       }
4624       addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySdwaDstSel, SdwaSel::DWORD);
4625       addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySdwaDstUnused, DstUnused::UNUSED_PRESERVE);
4626       addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySdwaSrc0Sel, SdwaSel::DWORD);
4627       addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySdwaSrc1Sel, SdwaSel::DWORD);
4628       break;
4629 
4630     case SIInstrFlags::VOPC:
4631       addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyClampSI, 0);
4632       addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySdwaSrc0Sel, SdwaSel::DWORD);
4633       addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySdwaSrc1Sel, SdwaSel::DWORD);
4634       break;
4635 
4636     default:
4637       llvm_unreachable("Invalid instruction type. Only VOP1, VOP2 and VOPC allowed");
4638     }
4639   }
4640 
4641   // special case v_mac_{f16, f32}:
4642   // it has src2 register operand that is tied to dst operand
4643   if (Inst.getOpcode() == AMDGPU::V_MAC_F32_sdwa_vi ||
4644       Inst.getOpcode() == AMDGPU::V_MAC_F16_sdwa_vi)  {
4645     auto it = Inst.begin();
4646     std::advance(
4647       it, AMDGPU::getNamedOperandIdx(Inst.getOpcode(), AMDGPU::OpName::src2));
4648     Inst.insert(it, Inst.getOperand(0)); // src2 = dst
4649   }
4650 }
4651 
4652 /// Force static initialization.
4653 extern "C" void LLVMInitializeAMDGPUAsmParser() {
4654   RegisterMCAsmParser<AMDGPUAsmParser> A(getTheAMDGPUTarget());
4655   RegisterMCAsmParser<AMDGPUAsmParser> B(getTheGCNTarget());
4656 }
4657 
4658 #define GET_REGISTER_MATCHER
4659 #define GET_MATCHER_IMPLEMENTATION
4660 #include "AMDGPUGenAsmMatcher.inc"
4661 
4662 // This fuction should be defined after auto-generated include so that we have
4663 // MatchClassKind enum defined
4664 unsigned AMDGPUAsmParser::validateTargetOperandClass(MCParsedAsmOperand &Op,
4665                                                      unsigned Kind) {
4666   // Tokens like "glc" would be parsed as immediate operands in ParseOperand().
4667   // But MatchInstructionImpl() expects to meet token and fails to validate
4668   // operand. This method checks if we are given immediate operand but expect to
4669   // get corresponding token.
4670   AMDGPUOperand &Operand = (AMDGPUOperand&)Op;
4671   switch (Kind) {
4672   case MCK_addr64:
4673     return Operand.isAddr64() ? Match_Success : Match_InvalidOperand;
4674   case MCK_gds:
4675     return Operand.isGDS() ? Match_Success : Match_InvalidOperand;
4676   case MCK_glc:
4677     return Operand.isGLC() ? Match_Success : Match_InvalidOperand;
4678   case MCK_idxen:
4679     return Operand.isIdxen() ? Match_Success : Match_InvalidOperand;
4680   case MCK_offen:
4681     return Operand.isOffen() ? Match_Success : Match_InvalidOperand;
4682   case MCK_SSrcB32:
4683     // When operands have expression values, they will return true for isToken,
4684     // because it is not possible to distinguish between a token and an
4685     // expression at parse time. MatchInstructionImpl() will always try to
4686     // match an operand as a token, when isToken returns true, and when the
4687     // name of the expression is not a valid token, the match will fail,
4688     // so we need to handle it here.
4689     return Operand.isSSrcB32() ? Match_Success : Match_InvalidOperand;
4690   case MCK_SSrcF32:
4691     return Operand.isSSrcF32() ? Match_Success : Match_InvalidOperand;
4692   case MCK_SoppBrTarget:
4693     return Operand.isSoppBrTarget() ? Match_Success : Match_InvalidOperand;
4694   case MCK_VReg32OrOff:
4695     return Operand.isVReg32OrOff() ? Match_Success : Match_InvalidOperand;
4696   case MCK_InterpSlot:
4697     return Operand.isInterpSlot() ? Match_Success : Match_InvalidOperand;
4698   case MCK_Attr:
4699     return Operand.isInterpAttr() ? Match_Success : Match_InvalidOperand;
4700   case MCK_AttrChan:
4701     return Operand.isAttrChan() ? Match_Success : Match_InvalidOperand;
4702   default:
4703     return Match_InvalidOperand;
4704   }
4705 }
4706