1 //===-- AMDGPUAsmParser.cpp - Parse SI asm to MCInst instructions ---------===//
2 //
3 //                     The LLVM Compiler Infrastructure
4 //
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
7 //
8 //===----------------------------------------------------------------------===//
9 
10 #include "AMDKernelCodeT.h"
11 #include "MCTargetDesc/AMDGPUMCTargetDesc.h"
12 #include "MCTargetDesc/AMDGPUTargetStreamer.h"
13 #include "SIDefines.h"
14 #include "Utils/AMDGPUAsmUtils.h"
15 #include "Utils/AMDGPUBaseInfo.h"
16 #include "Utils/AMDKernelCodeTUtils.h"
17 #include "llvm/ADT/APFloat.h"
18 #include "llvm/ADT/APInt.h"
19 #include "llvm/ADT/ArrayRef.h"
20 #include "llvm/ADT/STLExtras.h"
21 #include "llvm/ADT/SmallBitVector.h"
22 #include "llvm/ADT/SmallString.h"
23 #include "llvm/ADT/StringRef.h"
24 #include "llvm/ADT/StringSwitch.h"
25 #include "llvm/ADT/Twine.h"
26 #include "llvm/BinaryFormat/ELF.h"
27 #include "llvm/CodeGen/MachineValueType.h"
28 #include "llvm/MC/MCAsmInfo.h"
29 #include "llvm/MC/MCContext.h"
30 #include "llvm/MC/MCExpr.h"
31 #include "llvm/MC/MCInst.h"
32 #include "llvm/MC/MCInstrDesc.h"
33 #include "llvm/MC/MCInstrInfo.h"
34 #include "llvm/MC/MCParser/MCAsmLexer.h"
35 #include "llvm/MC/MCParser/MCAsmParser.h"
36 #include "llvm/MC/MCParser/MCAsmParserExtension.h"
37 #include "llvm/MC/MCParser/MCParsedAsmOperand.h"
38 #include "llvm/MC/MCParser/MCTargetAsmParser.h"
39 #include "llvm/MC/MCRegisterInfo.h"
40 #include "llvm/MC/MCStreamer.h"
41 #include "llvm/MC/MCSubtargetInfo.h"
42 #include "llvm/MC/MCSymbol.h"
43 #include "llvm/Support/Casting.h"
44 #include "llvm/Support/ErrorHandling.h"
45 #include "llvm/Support/MathExtras.h"
46 #include "llvm/Support/SMLoc.h"
47 #include "llvm/Support/TargetRegistry.h"
48 #include "llvm/Support/raw_ostream.h"
49 #include <algorithm>
50 #include <cassert>
51 #include <cstdint>
52 #include <cstring>
53 #include <iterator>
54 #include <map>
55 #include <memory>
56 #include <string>
57 
58 using namespace llvm;
59 using namespace llvm::AMDGPU;
60 
61 namespace {
62 
63 class AMDGPUAsmParser;
64 
65 enum RegisterKind { IS_UNKNOWN, IS_VGPR, IS_SGPR, IS_TTMP, IS_SPECIAL };
66 
67 //===----------------------------------------------------------------------===//
68 // Operand
69 //===----------------------------------------------------------------------===//
70 
71 class AMDGPUOperand : public MCParsedAsmOperand {
72   enum KindTy {
73     Token,
74     Immediate,
75     Register,
76     Expression
77   } Kind;
78 
79   SMLoc StartLoc, EndLoc;
80   const AMDGPUAsmParser *AsmParser;
81 
82 public:
83   AMDGPUOperand(KindTy Kind_, const AMDGPUAsmParser *AsmParser_)
84     : MCParsedAsmOperand(), Kind(Kind_), AsmParser(AsmParser_) {}
85 
86   typedef std::unique_ptr<AMDGPUOperand> Ptr;
87 
88   struct Modifiers {
89     bool Abs = false;
90     bool Neg = false;
91     bool Sext = false;
92 
93     bool hasFPModifiers() const { return Abs || Neg; }
94     bool hasIntModifiers() const { return Sext; }
95     bool hasModifiers() const { return hasFPModifiers() || hasIntModifiers(); }
96 
97     int64_t getFPModifiersOperand() const {
98       int64_t Operand = 0;
99       Operand |= Abs ? SISrcMods::ABS : 0;
100       Operand |= Neg ? SISrcMods::NEG : 0;
101       return Operand;
102     }
103 
104     int64_t getIntModifiersOperand() const {
105       int64_t Operand = 0;
106       Operand |= Sext ? SISrcMods::SEXT : 0;
107       return Operand;
108     }
109 
110     int64_t getModifiersOperand() const {
111       assert(!(hasFPModifiers() && hasIntModifiers())
112            && "fp and int modifiers should not be used simultaneously");
113       if (hasFPModifiers()) {
114         return getFPModifiersOperand();
115       } else if (hasIntModifiers()) {
116         return getIntModifiersOperand();
117       } else {
118         return 0;
119       }
120     }
121 
122     friend raw_ostream &operator <<(raw_ostream &OS, AMDGPUOperand::Modifiers Mods);
123   };
124 
125   enum ImmTy {
126     ImmTyNone,
127     ImmTyGDS,
128     ImmTyOffen,
129     ImmTyIdxen,
130     ImmTyAddr64,
131     ImmTyOffset,
132     ImmTyOffset0,
133     ImmTyOffset1,
134     ImmTyGLC,
135     ImmTySLC,
136     ImmTyTFE,
137     ImmTyClampSI,
138     ImmTyOModSI,
139     ImmTyDppCtrl,
140     ImmTyDppRowMask,
141     ImmTyDppBankMask,
142     ImmTyDppBoundCtrl,
143     ImmTySdwaDstSel,
144     ImmTySdwaSrc0Sel,
145     ImmTySdwaSrc1Sel,
146     ImmTySdwaDstUnused,
147     ImmTyDMask,
148     ImmTyUNorm,
149     ImmTyDA,
150     ImmTyR128,
151     ImmTyLWE,
152     ImmTyExpTgt,
153     ImmTyExpCompr,
154     ImmTyExpVM,
155     ImmTyDFMT,
156     ImmTyNFMT,
157     ImmTyHwreg,
158     ImmTyOff,
159     ImmTySendMsg,
160     ImmTyInterpSlot,
161     ImmTyInterpAttr,
162     ImmTyAttrChan,
163     ImmTyOpSel,
164     ImmTyOpSelHi,
165     ImmTyNegLo,
166     ImmTyNegHi,
167     ImmTySwizzle
168   };
169 
170   struct TokOp {
171     const char *Data;
172     unsigned Length;
173   };
174 
175   struct ImmOp {
176     int64_t Val;
177     ImmTy Type;
178     bool IsFPImm;
179     Modifiers Mods;
180   };
181 
182   struct RegOp {
183     unsigned RegNo;
184     bool IsForcedVOP3;
185     Modifiers Mods;
186   };
187 
188   union {
189     TokOp Tok;
190     ImmOp Imm;
191     RegOp Reg;
192     const MCExpr *Expr;
193   };
194 
195   bool isToken() const override {
196     if (Kind == Token)
197       return true;
198 
199     if (Kind != Expression || !Expr)
200       return false;
201 
202     // When parsing operands, we can't always tell if something was meant to be
203     // a token, like 'gds', or an expression that references a global variable.
204     // In this case, we assume the string is an expression, and if we need to
205     // interpret is a token, then we treat the symbol name as the token.
206     return isa<MCSymbolRefExpr>(Expr);
207   }
208 
209   bool isImm() const override {
210     return Kind == Immediate;
211   }
212 
213   bool isInlinableImm(MVT type) const;
214   bool isLiteralImm(MVT type) const;
215 
216   bool isRegKind() const {
217     return Kind == Register;
218   }
219 
220   bool isReg() const override {
221     return isRegKind() && !hasModifiers();
222   }
223 
224   bool isRegOrImmWithInputMods(MVT type) const {
225     return isRegKind() || isInlinableImm(type);
226   }
227 
228   bool isRegOrImmWithInt16InputMods() const {
229     return isRegOrImmWithInputMods(MVT::i16);
230   }
231 
232   bool isRegOrImmWithInt32InputMods() const {
233     return isRegOrImmWithInputMods(MVT::i32);
234   }
235 
236   bool isRegOrImmWithInt64InputMods() const {
237     return isRegOrImmWithInputMods(MVT::i64);
238   }
239 
240   bool isRegOrImmWithFP16InputMods() const {
241     return isRegOrImmWithInputMods(MVT::f16);
242   }
243 
244   bool isRegOrImmWithFP32InputMods() const {
245     return isRegOrImmWithInputMods(MVT::f32);
246   }
247 
248   bool isRegOrImmWithFP64InputMods() const {
249     return isRegOrImmWithInputMods(MVT::f64);
250   }
251 
252   bool isVReg() const {
253     return isRegClass(AMDGPU::VGPR_32RegClassID) ||
254            isRegClass(AMDGPU::VReg_64RegClassID) ||
255            isRegClass(AMDGPU::VReg_96RegClassID) ||
256            isRegClass(AMDGPU::VReg_128RegClassID) ||
257            isRegClass(AMDGPU::VReg_256RegClassID) ||
258            isRegClass(AMDGPU::VReg_512RegClassID);
259   }
260 
261   bool isVReg32OrOff() const {
262     return isOff() || isRegClass(AMDGPU::VGPR_32RegClassID);
263   }
264 
265   bool isSDWARegKind() const;
266 
267   bool isImmTy(ImmTy ImmT) const {
268     return isImm() && Imm.Type == ImmT;
269   }
270 
271   bool isImmModifier() const {
272     return isImm() && Imm.Type != ImmTyNone;
273   }
274 
275   bool isClampSI() const { return isImmTy(ImmTyClampSI); }
276   bool isOModSI() const { return isImmTy(ImmTyOModSI); }
277   bool isDMask() const { return isImmTy(ImmTyDMask); }
278   bool isUNorm() const { return isImmTy(ImmTyUNorm); }
279   bool isDA() const { return isImmTy(ImmTyDA); }
280   bool isR128() const { return isImmTy(ImmTyUNorm); }
281   bool isLWE() const { return isImmTy(ImmTyLWE); }
282   bool isOff() const { return isImmTy(ImmTyOff); }
283   bool isExpTgt() const { return isImmTy(ImmTyExpTgt); }
284   bool isExpVM() const { return isImmTy(ImmTyExpVM); }
285   bool isExpCompr() const { return isImmTy(ImmTyExpCompr); }
286   bool isOffen() const { return isImmTy(ImmTyOffen); }
287   bool isIdxen() const { return isImmTy(ImmTyIdxen); }
288   bool isAddr64() const { return isImmTy(ImmTyAddr64); }
289   bool isOffset() const { return isImmTy(ImmTyOffset) && isUInt<16>(getImm()); }
290   bool isOffset0() const { return isImmTy(ImmTyOffset0) && isUInt<16>(getImm()); }
291   bool isOffset1() const { return isImmTy(ImmTyOffset1) && isUInt<8>(getImm()); }
292 
293   bool isOffsetU12() const { return isImmTy(ImmTyOffset) && isUInt<12>(getImm()); }
294   bool isOffsetS13() const { return isImmTy(ImmTyOffset) && isInt<13>(getImm()); }
295   bool isGDS() const { return isImmTy(ImmTyGDS); }
296   bool isGLC() const { return isImmTy(ImmTyGLC); }
297   bool isSLC() const { return isImmTy(ImmTySLC); }
298   bool isTFE() const { return isImmTy(ImmTyTFE); }
299   bool isDFMT() const { return isImmTy(ImmTyDFMT) && isUInt<8>(getImm()); }
300   bool isNFMT() const { return isImmTy(ImmTyNFMT) && isUInt<8>(getImm()); }
301   bool isBankMask() const { return isImmTy(ImmTyDppBankMask); }
302   bool isRowMask() const { return isImmTy(ImmTyDppRowMask); }
303   bool isBoundCtrl() const { return isImmTy(ImmTyDppBoundCtrl); }
304   bool isSDWADstSel() const { return isImmTy(ImmTySdwaDstSel); }
305   bool isSDWASrc0Sel() const { return isImmTy(ImmTySdwaSrc0Sel); }
306   bool isSDWASrc1Sel() const { return isImmTy(ImmTySdwaSrc1Sel); }
307   bool isSDWADstUnused() const { return isImmTy(ImmTySdwaDstUnused); }
308   bool isInterpSlot() const { return isImmTy(ImmTyInterpSlot); }
309   bool isInterpAttr() const { return isImmTy(ImmTyInterpAttr); }
310   bool isAttrChan() const { return isImmTy(ImmTyAttrChan); }
311   bool isOpSel() const { return isImmTy(ImmTyOpSel); }
312   bool isOpSelHi() const { return isImmTy(ImmTyOpSelHi); }
313   bool isNegLo() const { return isImmTy(ImmTyNegLo); }
314   bool isNegHi() const { return isImmTy(ImmTyNegHi); }
315 
316   bool isMod() const {
317     return isClampSI() || isOModSI();
318   }
319 
320   bool isRegOrImm() const {
321     return isReg() || isImm();
322   }
323 
324   bool isRegClass(unsigned RCID) const;
325 
326   bool isRegOrInlineNoMods(unsigned RCID, MVT type) const {
327     return (isRegClass(RCID) || isInlinableImm(type)) && !hasModifiers();
328   }
329 
330   bool isSCSrcB16() const {
331     return isRegOrInlineNoMods(AMDGPU::SReg_32RegClassID, MVT::i16);
332   }
333 
334   bool isSCSrcV2B16() const {
335     return isSCSrcB16();
336   }
337 
338   bool isSCSrcB32() const {
339     return isRegOrInlineNoMods(AMDGPU::SReg_32RegClassID, MVT::i32);
340   }
341 
342   bool isSCSrcB64() const {
343     return isRegOrInlineNoMods(AMDGPU::SReg_64RegClassID, MVT::i64);
344   }
345 
346   bool isSCSrcF16() const {
347     return isRegOrInlineNoMods(AMDGPU::SReg_32RegClassID, MVT::f16);
348   }
349 
350   bool isSCSrcV2F16() const {
351     return isSCSrcF16();
352   }
353 
354   bool isSCSrcF32() const {
355     return isRegOrInlineNoMods(AMDGPU::SReg_32RegClassID, MVT::f32);
356   }
357 
358   bool isSCSrcF64() const {
359     return isRegOrInlineNoMods(AMDGPU::SReg_64RegClassID, MVT::f64);
360   }
361 
362   bool isSSrcB32() const {
363     return isSCSrcB32() || isLiteralImm(MVT::i32) || isExpr();
364   }
365 
366   bool isSSrcB16() const {
367     return isSCSrcB16() || isLiteralImm(MVT::i16);
368   }
369 
370   bool isSSrcV2B16() const {
371     llvm_unreachable("cannot happen");
372     return isSSrcB16();
373   }
374 
375   bool isSSrcB64() const {
376     // TODO: Find out how SALU supports extension of 32-bit literals to 64 bits.
377     // See isVSrc64().
378     return isSCSrcB64() || isLiteralImm(MVT::i64);
379   }
380 
381   bool isSSrcF32() const {
382     return isSCSrcB32() || isLiteralImm(MVT::f32) || isExpr();
383   }
384 
385   bool isSSrcF64() const {
386     return isSCSrcB64() || isLiteralImm(MVT::f64);
387   }
388 
389   bool isSSrcF16() const {
390     return isSCSrcB16() || isLiteralImm(MVT::f16);
391   }
392 
393   bool isSSrcV2F16() const {
394     llvm_unreachable("cannot happen");
395     return isSSrcF16();
396   }
397 
398   bool isVCSrcB32() const {
399     return isRegOrInlineNoMods(AMDGPU::VS_32RegClassID, MVT::i32);
400   }
401 
402   bool isVCSrcB64() const {
403     return isRegOrInlineNoMods(AMDGPU::VS_64RegClassID, MVT::i64);
404   }
405 
406   bool isVCSrcB16() const {
407     return isRegOrInlineNoMods(AMDGPU::VS_32RegClassID, MVT::i16);
408   }
409 
410   bool isVCSrcV2B16() const {
411     return isVCSrcB16();
412   }
413 
414   bool isVCSrcF32() const {
415     return isRegOrInlineNoMods(AMDGPU::VS_32RegClassID, MVT::f32);
416   }
417 
418   bool isVCSrcF64() const {
419     return isRegOrInlineNoMods(AMDGPU::VS_64RegClassID, MVT::f64);
420   }
421 
422   bool isVCSrcF16() const {
423     return isRegOrInlineNoMods(AMDGPU::VS_32RegClassID, MVT::f16);
424   }
425 
426   bool isVCSrcV2F16() const {
427     return isVCSrcF16();
428   }
429 
430   bool isVSrcB32() const {
431     return isVCSrcF32() || isLiteralImm(MVT::i32);
432   }
433 
434   bool isVSrcB64() const {
435     return isVCSrcF64() || isLiteralImm(MVT::i64);
436   }
437 
438   bool isVSrcB16() const {
439     return isVCSrcF16() || isLiteralImm(MVT::i16);
440   }
441 
442   bool isVSrcV2B16() const {
443     llvm_unreachable("cannot happen");
444     return isVSrcB16();
445   }
446 
447   bool isVSrcF32() const {
448     return isVCSrcF32() || isLiteralImm(MVT::f32);
449   }
450 
451   bool isVSrcF64() const {
452     return isVCSrcF64() || isLiteralImm(MVT::f64);
453   }
454 
455   bool isVSrcF16() const {
456     return isVCSrcF16() || isLiteralImm(MVT::f16);
457   }
458 
459   bool isVSrcV2F16() const {
460     llvm_unreachable("cannot happen");
461     return isVSrcF16();
462   }
463 
464   bool isKImmFP32() const {
465     return isLiteralImm(MVT::f32);
466   }
467 
468   bool isKImmFP16() const {
469     return isLiteralImm(MVT::f16);
470   }
471 
472   bool isMem() const override {
473     return false;
474   }
475 
476   bool isExpr() const {
477     return Kind == Expression;
478   }
479 
480   bool isSoppBrTarget() const {
481     return isExpr() || isImm();
482   }
483 
484   bool isSWaitCnt() const;
485   bool isHwreg() const;
486   bool isSendMsg() const;
487   bool isSwizzle() const;
488   bool isSMRDOffset8() const;
489   bool isSMRDOffset20() const;
490   bool isSMRDLiteralOffset() const;
491   bool isDPPCtrl() const;
492   bool isGPRIdxMode() const;
493   bool isS16Imm() const;
494   bool isU16Imm() const;
495 
496   StringRef getExpressionAsToken() const {
497     assert(isExpr());
498     const MCSymbolRefExpr *S = cast<MCSymbolRefExpr>(Expr);
499     return S->getSymbol().getName();
500   }
501 
502   StringRef getToken() const {
503     assert(isToken());
504 
505     if (Kind == Expression)
506       return getExpressionAsToken();
507 
508     return StringRef(Tok.Data, Tok.Length);
509   }
510 
511   int64_t getImm() const {
512     assert(isImm());
513     return Imm.Val;
514   }
515 
516   ImmTy getImmTy() const {
517     assert(isImm());
518     return Imm.Type;
519   }
520 
521   unsigned getReg() const override {
522     return Reg.RegNo;
523   }
524 
525   SMLoc getStartLoc() const override {
526     return StartLoc;
527   }
528 
529   SMLoc getEndLoc() const override {
530     return EndLoc;
531   }
532 
533   Modifiers getModifiers() const {
534     assert(isRegKind() || isImmTy(ImmTyNone));
535     return isRegKind() ? Reg.Mods : Imm.Mods;
536   }
537 
538   void setModifiers(Modifiers Mods) {
539     assert(isRegKind() || isImmTy(ImmTyNone));
540     if (isRegKind())
541       Reg.Mods = Mods;
542     else
543       Imm.Mods = Mods;
544   }
545 
546   bool hasModifiers() const {
547     return getModifiers().hasModifiers();
548   }
549 
550   bool hasFPModifiers() const {
551     return getModifiers().hasFPModifiers();
552   }
553 
554   bool hasIntModifiers() const {
555     return getModifiers().hasIntModifiers();
556   }
557 
558   uint64_t applyInputFPModifiers(uint64_t Val, unsigned Size) const;
559 
560   void addImmOperands(MCInst &Inst, unsigned N, bool ApplyModifiers = true) const;
561 
562   void addLiteralImmOperand(MCInst &Inst, int64_t Val, bool ApplyModifiers) const;
563 
564   template <unsigned Bitwidth>
565   void addKImmFPOperands(MCInst &Inst, unsigned N) const;
566 
567   void addKImmFP16Operands(MCInst &Inst, unsigned N) const {
568     addKImmFPOperands<16>(Inst, N);
569   }
570 
571   void addKImmFP32Operands(MCInst &Inst, unsigned N) const {
572     addKImmFPOperands<32>(Inst, N);
573   }
574 
575   void addRegOperands(MCInst &Inst, unsigned N) const;
576 
577   void addRegOrImmOperands(MCInst &Inst, unsigned N) const {
578     if (isRegKind())
579       addRegOperands(Inst, N);
580     else if (isExpr())
581       Inst.addOperand(MCOperand::createExpr(Expr));
582     else
583       addImmOperands(Inst, N);
584   }
585 
586   void addRegOrImmWithInputModsOperands(MCInst &Inst, unsigned N) const {
587     Modifiers Mods = getModifiers();
588     Inst.addOperand(MCOperand::createImm(Mods.getModifiersOperand()));
589     if (isRegKind()) {
590       addRegOperands(Inst, N);
591     } else {
592       addImmOperands(Inst, N, false);
593     }
594   }
595 
596   void addRegOrImmWithFPInputModsOperands(MCInst &Inst, unsigned N) const {
597     assert(!hasIntModifiers());
598     addRegOrImmWithInputModsOperands(Inst, N);
599   }
600 
601   void addRegOrImmWithIntInputModsOperands(MCInst &Inst, unsigned N) const {
602     assert(!hasFPModifiers());
603     addRegOrImmWithInputModsOperands(Inst, N);
604   }
605 
606   void addRegWithInputModsOperands(MCInst &Inst, unsigned N) const {
607     Modifiers Mods = getModifiers();
608     Inst.addOperand(MCOperand::createImm(Mods.getModifiersOperand()));
609     assert(isRegKind());
610     addRegOperands(Inst, N);
611   }
612 
613   void addRegWithFPInputModsOperands(MCInst &Inst, unsigned N) const {
614     assert(!hasIntModifiers());
615     addRegWithInputModsOperands(Inst, N);
616   }
617 
618   void addRegWithIntInputModsOperands(MCInst &Inst, unsigned N) const {
619     assert(!hasFPModifiers());
620     addRegWithInputModsOperands(Inst, N);
621   }
622 
623   void addSoppBrTargetOperands(MCInst &Inst, unsigned N) const {
624     if (isImm())
625       addImmOperands(Inst, N);
626     else {
627       assert(isExpr());
628       Inst.addOperand(MCOperand::createExpr(Expr));
629     }
630   }
631 
632   static void printImmTy(raw_ostream& OS, ImmTy Type) {
633     switch (Type) {
634     case ImmTyNone: OS << "None"; break;
635     case ImmTyGDS: OS << "GDS"; break;
636     case ImmTyOffen: OS << "Offen"; break;
637     case ImmTyIdxen: OS << "Idxen"; break;
638     case ImmTyAddr64: OS << "Addr64"; break;
639     case ImmTyOffset: OS << "Offset"; break;
640     case ImmTyOffset0: OS << "Offset0"; break;
641     case ImmTyOffset1: OS << "Offset1"; break;
642     case ImmTyGLC: OS << "GLC"; break;
643     case ImmTySLC: OS << "SLC"; break;
644     case ImmTyTFE: OS << "TFE"; break;
645     case ImmTyDFMT: OS << "DFMT"; break;
646     case ImmTyNFMT: OS << "NFMT"; break;
647     case ImmTyClampSI: OS << "ClampSI"; break;
648     case ImmTyOModSI: OS << "OModSI"; break;
649     case ImmTyDppCtrl: OS << "DppCtrl"; break;
650     case ImmTyDppRowMask: OS << "DppRowMask"; break;
651     case ImmTyDppBankMask: OS << "DppBankMask"; break;
652     case ImmTyDppBoundCtrl: OS << "DppBoundCtrl"; break;
653     case ImmTySdwaDstSel: OS << "SdwaDstSel"; break;
654     case ImmTySdwaSrc0Sel: OS << "SdwaSrc0Sel"; break;
655     case ImmTySdwaSrc1Sel: OS << "SdwaSrc1Sel"; break;
656     case ImmTySdwaDstUnused: OS << "SdwaDstUnused"; break;
657     case ImmTyDMask: OS << "DMask"; break;
658     case ImmTyUNorm: OS << "UNorm"; break;
659     case ImmTyDA: OS << "DA"; break;
660     case ImmTyR128: OS << "R128"; break;
661     case ImmTyLWE: OS << "LWE"; break;
662     case ImmTyOff: OS << "Off"; break;
663     case ImmTyExpTgt: OS << "ExpTgt"; break;
664     case ImmTyExpCompr: OS << "ExpCompr"; break;
665     case ImmTyExpVM: OS << "ExpVM"; break;
666     case ImmTyHwreg: OS << "Hwreg"; break;
667     case ImmTySendMsg: OS << "SendMsg"; break;
668     case ImmTyInterpSlot: OS << "InterpSlot"; break;
669     case ImmTyInterpAttr: OS << "InterpAttr"; break;
670     case ImmTyAttrChan: OS << "AttrChan"; break;
671     case ImmTyOpSel: OS << "OpSel"; break;
672     case ImmTyOpSelHi: OS << "OpSelHi"; break;
673     case ImmTyNegLo: OS << "NegLo"; break;
674     case ImmTyNegHi: OS << "NegHi"; break;
675     case ImmTySwizzle: OS << "Swizzle"; break;
676     }
677   }
678 
679   void print(raw_ostream &OS) const override {
680     switch (Kind) {
681     case Register:
682       OS << "<register " << getReg() << " mods: " << Reg.Mods << '>';
683       break;
684     case Immediate:
685       OS << '<' << getImm();
686       if (getImmTy() != ImmTyNone) {
687         OS << " type: "; printImmTy(OS, getImmTy());
688       }
689       OS << " mods: " << Imm.Mods << '>';
690       break;
691     case Token:
692       OS << '\'' << getToken() << '\'';
693       break;
694     case Expression:
695       OS << "<expr " << *Expr << '>';
696       break;
697     }
698   }
699 
700   static AMDGPUOperand::Ptr CreateImm(const AMDGPUAsmParser *AsmParser,
701                                       int64_t Val, SMLoc Loc,
702                                       ImmTy Type = ImmTyNone,
703                                       bool IsFPImm = false) {
704     auto Op = llvm::make_unique<AMDGPUOperand>(Immediate, AsmParser);
705     Op->Imm.Val = Val;
706     Op->Imm.IsFPImm = IsFPImm;
707     Op->Imm.Type = Type;
708     Op->Imm.Mods = Modifiers();
709     Op->StartLoc = Loc;
710     Op->EndLoc = Loc;
711     return Op;
712   }
713 
714   static AMDGPUOperand::Ptr CreateToken(const AMDGPUAsmParser *AsmParser,
715                                         StringRef Str, SMLoc Loc,
716                                         bool HasExplicitEncodingSize = true) {
717     auto Res = llvm::make_unique<AMDGPUOperand>(Token, AsmParser);
718     Res->Tok.Data = Str.data();
719     Res->Tok.Length = Str.size();
720     Res->StartLoc = Loc;
721     Res->EndLoc = Loc;
722     return Res;
723   }
724 
725   static AMDGPUOperand::Ptr CreateReg(const AMDGPUAsmParser *AsmParser,
726                                       unsigned RegNo, SMLoc S,
727                                       SMLoc E,
728                                       bool ForceVOP3) {
729     auto Op = llvm::make_unique<AMDGPUOperand>(Register, AsmParser);
730     Op->Reg.RegNo = RegNo;
731     Op->Reg.Mods = Modifiers();
732     Op->Reg.IsForcedVOP3 = ForceVOP3;
733     Op->StartLoc = S;
734     Op->EndLoc = E;
735     return Op;
736   }
737 
738   static AMDGPUOperand::Ptr CreateExpr(const AMDGPUAsmParser *AsmParser,
739                                        const class MCExpr *Expr, SMLoc S) {
740     auto Op = llvm::make_unique<AMDGPUOperand>(Expression, AsmParser);
741     Op->Expr = Expr;
742     Op->StartLoc = S;
743     Op->EndLoc = S;
744     return Op;
745   }
746 };
747 
748 raw_ostream &operator <<(raw_ostream &OS, AMDGPUOperand::Modifiers Mods) {
749   OS << "abs:" << Mods.Abs << " neg: " << Mods.Neg << " sext:" << Mods.Sext;
750   return OS;
751 }
752 
753 //===----------------------------------------------------------------------===//
754 // AsmParser
755 //===----------------------------------------------------------------------===//
756 
757 // Holds info related to the current kernel, e.g. count of SGPRs used.
758 // Kernel scope begins at .amdgpu_hsa_kernel directive, ends at next
759 // .amdgpu_hsa_kernel or at EOF.
760 class KernelScopeInfo {
761   int SgprIndexUnusedMin = -1;
762   int VgprIndexUnusedMin = -1;
763   MCContext *Ctx = nullptr;
764 
765   void usesSgprAt(int i) {
766     if (i >= SgprIndexUnusedMin) {
767       SgprIndexUnusedMin = ++i;
768       if (Ctx) {
769         MCSymbol * const Sym = Ctx->getOrCreateSymbol(Twine(".kernel.sgpr_count"));
770         Sym->setVariableValue(MCConstantExpr::create(SgprIndexUnusedMin, *Ctx));
771       }
772     }
773   }
774 
775   void usesVgprAt(int i) {
776     if (i >= VgprIndexUnusedMin) {
777       VgprIndexUnusedMin = ++i;
778       if (Ctx) {
779         MCSymbol * const Sym = Ctx->getOrCreateSymbol(Twine(".kernel.vgpr_count"));
780         Sym->setVariableValue(MCConstantExpr::create(VgprIndexUnusedMin, *Ctx));
781       }
782     }
783   }
784 
785 public:
786   KernelScopeInfo() = default;
787 
788   void initialize(MCContext &Context) {
789     Ctx = &Context;
790     usesSgprAt(SgprIndexUnusedMin = -1);
791     usesVgprAt(VgprIndexUnusedMin = -1);
792   }
793 
794   void usesRegister(RegisterKind RegKind, unsigned DwordRegIndex, unsigned RegWidth) {
795     switch (RegKind) {
796       case IS_SGPR: usesSgprAt(DwordRegIndex + RegWidth - 1); break;
797       case IS_VGPR: usesVgprAt(DwordRegIndex + RegWidth - 1); break;
798       default: break;
799     }
800   }
801 };
802 
803 class AMDGPUAsmParser : public MCTargetAsmParser {
804   const MCInstrInfo &MII;
805   MCAsmParser &Parser;
806 
807   unsigned ForcedEncodingSize = 0;
808   bool ForcedDPP = false;
809   bool ForcedSDWA = false;
810   KernelScopeInfo KernelScope;
811 
812   /// @name Auto-generated Match Functions
813   /// {
814 
815 #define GET_ASSEMBLER_HEADER
816 #include "AMDGPUGenAsmMatcher.inc"
817 
818   /// }
819 
820 private:
821   bool ParseAsAbsoluteExpression(uint32_t &Ret);
822   bool ParseDirectiveMajorMinor(uint32_t &Major, uint32_t &Minor);
823   bool ParseDirectiveHSACodeObjectVersion();
824   bool ParseDirectiveHSACodeObjectISA();
825   bool ParseDirectiveCodeObjectMetadata();
826   bool ParseAMDKernelCodeTValue(StringRef ID, amd_kernel_code_t &Header);
827   bool ParseDirectiveAMDKernelCodeT();
828   bool subtargetHasRegister(const MCRegisterInfo &MRI, unsigned RegNo) const;
829   bool ParseDirectiveAMDGPUHsaKernel();
830   bool AddNextRegisterToList(unsigned& Reg, unsigned& RegWidth,
831                              RegisterKind RegKind, unsigned Reg1,
832                              unsigned RegNum);
833   bool ParseAMDGPURegister(RegisterKind& RegKind, unsigned& Reg,
834                            unsigned& RegNum, unsigned& RegWidth,
835                            unsigned *DwordRegIndex);
836   void cvtMubufImpl(MCInst &Inst, const OperandVector &Operands,
837                     bool IsAtomic, bool IsAtomicReturn);
838   void cvtDSImpl(MCInst &Inst, const OperandVector &Operands,
839                  bool IsGdsHardcoded);
840 
841 public:
842   enum AMDGPUMatchResultTy {
843     Match_PreferE32 = FIRST_TARGET_MATCH_RESULT_TY
844   };
845 
846   typedef std::map<AMDGPUOperand::ImmTy, unsigned> OptionalImmIndexMap;
847 
848   AMDGPUAsmParser(const MCSubtargetInfo &STI, MCAsmParser &_Parser,
849                const MCInstrInfo &MII,
850                const MCTargetOptions &Options)
851       : MCTargetAsmParser(Options, STI), MII(MII), Parser(_Parser) {
852     MCAsmParserExtension::Initialize(Parser);
853 
854     if (getFeatureBits().none()) {
855       // Set default features.
856       copySTI().ToggleFeature("SOUTHERN_ISLANDS");
857     }
858 
859     setAvailableFeatures(ComputeAvailableFeatures(getFeatureBits()));
860 
861     {
862       // TODO: make those pre-defined variables read-only.
863       // Currently there is none suitable machinery in the core llvm-mc for this.
864       // MCSymbol::isRedefinable is intended for another purpose, and
865       // AsmParser::parseDirectiveSet() cannot be specialized for specific target.
866       AMDGPU::IsaInfo::IsaVersion ISA =
867           AMDGPU::IsaInfo::getIsaVersion(getFeatureBits());
868       MCContext &Ctx = getContext();
869       MCSymbol *Sym =
870           Ctx.getOrCreateSymbol(Twine(".option.machine_version_major"));
871       Sym->setVariableValue(MCConstantExpr::create(ISA.Major, Ctx));
872       Sym = Ctx.getOrCreateSymbol(Twine(".option.machine_version_minor"));
873       Sym->setVariableValue(MCConstantExpr::create(ISA.Minor, Ctx));
874       Sym = Ctx.getOrCreateSymbol(Twine(".option.machine_version_stepping"));
875       Sym->setVariableValue(MCConstantExpr::create(ISA.Stepping, Ctx));
876     }
877     KernelScope.initialize(getContext());
878   }
879 
880   bool isSI() const {
881     return AMDGPU::isSI(getSTI());
882   }
883 
884   bool isCI() const {
885     return AMDGPU::isCI(getSTI());
886   }
887 
888   bool isVI() const {
889     return AMDGPU::isVI(getSTI());
890   }
891 
892   bool isGFX9() const {
893     return AMDGPU::isGFX9(getSTI());
894   }
895 
896   bool hasInv2PiInlineImm() const {
897     return getFeatureBits()[AMDGPU::FeatureInv2PiInlineImm];
898   }
899 
900   bool hasFlatOffsets() const {
901     return getFeatureBits()[AMDGPU::FeatureFlatInstOffsets];
902   }
903 
904   bool hasSGPR102_SGPR103() const {
905     return !isVI();
906   }
907 
908   AMDGPUTargetStreamer &getTargetStreamer() {
909     MCTargetStreamer &TS = *getParser().getStreamer().getTargetStreamer();
910     return static_cast<AMDGPUTargetStreamer &>(TS);
911   }
912 
913   const MCRegisterInfo *getMRI() const {
914     // We need this const_cast because for some reason getContext() is not const
915     // in MCAsmParser.
916     return const_cast<AMDGPUAsmParser*>(this)->getContext().getRegisterInfo();
917   }
918 
919   const MCInstrInfo *getMII() const {
920     return &MII;
921   }
922 
923   const FeatureBitset &getFeatureBits() const {
924     return getSTI().getFeatureBits();
925   }
926 
927   void setForcedEncodingSize(unsigned Size) { ForcedEncodingSize = Size; }
928   void setForcedDPP(bool ForceDPP_) { ForcedDPP = ForceDPP_; }
929   void setForcedSDWA(bool ForceSDWA_) { ForcedSDWA = ForceSDWA_; }
930 
931   unsigned getForcedEncodingSize() const { return ForcedEncodingSize; }
932   bool isForcedVOP3() const { return ForcedEncodingSize == 64; }
933   bool isForcedDPP() const { return ForcedDPP; }
934   bool isForcedSDWA() const { return ForcedSDWA; }
935   ArrayRef<unsigned> getMatchedVariants() const;
936 
937   std::unique_ptr<AMDGPUOperand> parseRegister();
938   bool ParseRegister(unsigned &RegNo, SMLoc &StartLoc, SMLoc &EndLoc) override;
939   unsigned checkTargetMatchPredicate(MCInst &Inst) override;
940   unsigned validateTargetOperandClass(MCParsedAsmOperand &Op,
941                                       unsigned Kind) override;
942   bool MatchAndEmitInstruction(SMLoc IDLoc, unsigned &Opcode,
943                                OperandVector &Operands, MCStreamer &Out,
944                                uint64_t &ErrorInfo,
945                                bool MatchingInlineAsm) override;
946   bool ParseDirective(AsmToken DirectiveID) override;
947   OperandMatchResultTy parseOperand(OperandVector &Operands, StringRef Mnemonic);
948   StringRef parseMnemonicSuffix(StringRef Name);
949   bool ParseInstruction(ParseInstructionInfo &Info, StringRef Name,
950                         SMLoc NameLoc, OperandVector &Operands) override;
951   //bool ProcessInstruction(MCInst &Inst);
952 
953   OperandMatchResultTy parseIntWithPrefix(const char *Prefix, int64_t &Int);
954 
955   OperandMatchResultTy
956   parseIntWithPrefix(const char *Prefix, OperandVector &Operands,
957                      AMDGPUOperand::ImmTy ImmTy = AMDGPUOperand::ImmTyNone,
958                      bool (*ConvertResult)(int64_t &) = nullptr);
959 
960   OperandMatchResultTy parseOperandArrayWithPrefix(
961     const char *Prefix,
962     OperandVector &Operands,
963     AMDGPUOperand::ImmTy ImmTy = AMDGPUOperand::ImmTyNone,
964     bool (*ConvertResult)(int64_t&) = nullptr);
965 
966   OperandMatchResultTy
967   parseNamedBit(const char *Name, OperandVector &Operands,
968                 AMDGPUOperand::ImmTy ImmTy = AMDGPUOperand::ImmTyNone);
969   OperandMatchResultTy parseStringWithPrefix(StringRef Prefix,
970                                              StringRef &Value);
971 
972   bool parseAbsoluteExpr(int64_t &Val, bool AbsMod = false);
973   OperandMatchResultTy parseImm(OperandVector &Operands, bool AbsMod = false);
974   OperandMatchResultTy parseReg(OperandVector &Operands);
975   OperandMatchResultTy parseRegOrImm(OperandVector &Operands, bool AbsMod = false);
976   OperandMatchResultTy parseRegOrImmWithFPInputMods(OperandVector &Operands, bool AllowImm = true);
977   OperandMatchResultTy parseRegOrImmWithIntInputMods(OperandVector &Operands, bool AllowImm = true);
978   OperandMatchResultTy parseRegWithFPInputMods(OperandVector &Operands);
979   OperandMatchResultTy parseRegWithIntInputMods(OperandVector &Operands);
980   OperandMatchResultTy parseVReg32OrOff(OperandVector &Operands);
981 
982   void cvtDSOffset01(MCInst &Inst, const OperandVector &Operands);
983   void cvtDS(MCInst &Inst, const OperandVector &Operands) { cvtDSImpl(Inst, Operands, false); }
984   void cvtDSGds(MCInst &Inst, const OperandVector &Operands) { cvtDSImpl(Inst, Operands, true); }
985   void cvtExp(MCInst &Inst, const OperandVector &Operands);
986 
987   bool parseCnt(int64_t &IntVal);
988   OperandMatchResultTy parseSWaitCntOps(OperandVector &Operands);
989   OperandMatchResultTy parseHwreg(OperandVector &Operands);
990 
991 private:
992   struct OperandInfoTy {
993     int64_t Id;
994     bool IsSymbolic;
995     OperandInfoTy(int64_t Id_) : Id(Id_), IsSymbolic(false) { }
996   };
997 
998   bool parseSendMsgConstruct(OperandInfoTy &Msg, OperandInfoTy &Operation, int64_t &StreamId);
999   bool parseHwregConstruct(OperandInfoTy &HwReg, int64_t &Offset, int64_t &Width);
1000 
1001   void errorExpTgt();
1002   OperandMatchResultTy parseExpTgtImpl(StringRef Str, uint8_t &Val);
1003 
1004   bool validateInstruction(const MCInst &Inst, const SMLoc &IDLoc);
1005   bool validateConstantBusLimitations(const MCInst &Inst);
1006   bool validateEarlyClobberLimitations(const MCInst &Inst);
1007   bool usesConstantBus(const MCInst &Inst, unsigned OpIdx);
1008   bool isInlineConstant(const MCInst &Inst, unsigned OpIdx) const;
1009   unsigned findImplicitSGPRReadInVOP(const MCInst &Inst) const;
1010 
1011   bool trySkipId(const StringRef Id);
1012   bool trySkipToken(const AsmToken::TokenKind Kind);
1013   bool skipToken(const AsmToken::TokenKind Kind, const StringRef ErrMsg);
1014   bool parseString(StringRef &Val, const StringRef ErrMsg = "expected a string");
1015   bool parseExpr(int64_t &Imm);
1016 
1017 public:
1018   OperandMatchResultTy parseOptionalOperand(OperandVector &Operands);
1019 
1020   OperandMatchResultTy parseExpTgt(OperandVector &Operands);
1021   OperandMatchResultTy parseSendMsgOp(OperandVector &Operands);
1022   OperandMatchResultTy parseInterpSlot(OperandVector &Operands);
1023   OperandMatchResultTy parseInterpAttr(OperandVector &Operands);
1024   OperandMatchResultTy parseSOppBrTarget(OperandVector &Operands);
1025 
1026   bool parseSwizzleOperands(const unsigned OpNum, int64_t* Op,
1027                             const unsigned MinVal,
1028                             const unsigned MaxVal,
1029                             const StringRef ErrMsg);
1030   OperandMatchResultTy parseSwizzleOp(OperandVector &Operands);
1031   bool parseSwizzleOffset(int64_t &Imm);
1032   bool parseSwizzleMacro(int64_t &Imm);
1033   bool parseSwizzleQuadPerm(int64_t &Imm);
1034   bool parseSwizzleBitmaskPerm(int64_t &Imm);
1035   bool parseSwizzleBroadcast(int64_t &Imm);
1036   bool parseSwizzleSwap(int64_t &Imm);
1037   bool parseSwizzleReverse(int64_t &Imm);
1038 
1039   void cvtMubuf(MCInst &Inst, const OperandVector &Operands) { cvtMubufImpl(Inst, Operands, false, false); }
1040   void cvtMubufAtomic(MCInst &Inst, const OperandVector &Operands) { cvtMubufImpl(Inst, Operands, true, false); }
1041   void cvtMubufAtomicReturn(MCInst &Inst, const OperandVector &Operands) { cvtMubufImpl(Inst, Operands, true, true); }
1042   void cvtMtbuf(MCInst &Inst, const OperandVector &Operands);
1043 
1044   AMDGPUOperand::Ptr defaultGLC() const;
1045   AMDGPUOperand::Ptr defaultSLC() const;
1046   AMDGPUOperand::Ptr defaultTFE() const;
1047 
1048   AMDGPUOperand::Ptr defaultDMask() const;
1049   AMDGPUOperand::Ptr defaultUNorm() const;
1050   AMDGPUOperand::Ptr defaultDA() const;
1051   AMDGPUOperand::Ptr defaultR128() const;
1052   AMDGPUOperand::Ptr defaultLWE() const;
1053   AMDGPUOperand::Ptr defaultSMRDOffset8() const;
1054   AMDGPUOperand::Ptr defaultSMRDOffset20() const;
1055   AMDGPUOperand::Ptr defaultSMRDLiteralOffset() const;
1056   AMDGPUOperand::Ptr defaultOffsetU12() const;
1057   AMDGPUOperand::Ptr defaultOffsetS13() const;
1058 
1059   OperandMatchResultTy parseOModOperand(OperandVector &Operands);
1060 
1061   void cvtVOP3(MCInst &Inst, const OperandVector &Operands,
1062                OptionalImmIndexMap &OptionalIdx);
1063   void cvtVOP3(MCInst &Inst, const OperandVector &Operands);
1064   void cvtVOP3P(MCInst &Inst, const OperandVector &Operands);
1065 
1066   void cvtMIMG(MCInst &Inst, const OperandVector &Operands,
1067                bool IsAtomic = false);
1068   void cvtMIMGAtomic(MCInst &Inst, const OperandVector &Operands);
1069 
1070   OperandMatchResultTy parseDPPCtrl(OperandVector &Operands);
1071   AMDGPUOperand::Ptr defaultRowMask() const;
1072   AMDGPUOperand::Ptr defaultBankMask() const;
1073   AMDGPUOperand::Ptr defaultBoundCtrl() const;
1074   void cvtDPP(MCInst &Inst, const OperandVector &Operands);
1075 
1076   OperandMatchResultTy parseSDWASel(OperandVector &Operands, StringRef Prefix,
1077                                     AMDGPUOperand::ImmTy Type);
1078   OperandMatchResultTy parseSDWADstUnused(OperandVector &Operands);
1079   void cvtSdwaVOP1(MCInst &Inst, const OperandVector &Operands);
1080   void cvtSdwaVOP2(MCInst &Inst, const OperandVector &Operands);
1081   void cvtSdwaVOP2b(MCInst &Inst, const OperandVector &Operands);
1082   void cvtSdwaVOPC(MCInst &Inst, const OperandVector &Operands);
1083   void cvtSDWA(MCInst &Inst, const OperandVector &Operands,
1084                 uint64_t BasicInstType, bool skipVcc = false);
1085 };
1086 
1087 struct OptionalOperand {
1088   const char *Name;
1089   AMDGPUOperand::ImmTy Type;
1090   bool IsBit;
1091   bool (*ConvertResult)(int64_t&);
1092 };
1093 
1094 } // end anonymous namespace
1095 
1096 // May be called with integer type with equivalent bitwidth.
1097 static const fltSemantics *getFltSemantics(unsigned Size) {
1098   switch (Size) {
1099   case 4:
1100     return &APFloat::IEEEsingle();
1101   case 8:
1102     return &APFloat::IEEEdouble();
1103   case 2:
1104     return &APFloat::IEEEhalf();
1105   default:
1106     llvm_unreachable("unsupported fp type");
1107   }
1108 }
1109 
1110 static const fltSemantics *getFltSemantics(MVT VT) {
1111   return getFltSemantics(VT.getSizeInBits() / 8);
1112 }
1113 
1114 static const fltSemantics *getOpFltSemantics(uint8_t OperandType) {
1115   switch (OperandType) {
1116   case AMDGPU::OPERAND_REG_IMM_INT32:
1117   case AMDGPU::OPERAND_REG_IMM_FP32:
1118   case AMDGPU::OPERAND_REG_INLINE_C_INT32:
1119   case AMDGPU::OPERAND_REG_INLINE_C_FP32:
1120     return &APFloat::IEEEsingle();
1121   case AMDGPU::OPERAND_REG_IMM_INT64:
1122   case AMDGPU::OPERAND_REG_IMM_FP64:
1123   case AMDGPU::OPERAND_REG_INLINE_C_INT64:
1124   case AMDGPU::OPERAND_REG_INLINE_C_FP64:
1125     return &APFloat::IEEEdouble();
1126   case AMDGPU::OPERAND_REG_IMM_INT16:
1127   case AMDGPU::OPERAND_REG_IMM_FP16:
1128   case AMDGPU::OPERAND_REG_INLINE_C_INT16:
1129   case AMDGPU::OPERAND_REG_INLINE_C_FP16:
1130   case AMDGPU::OPERAND_REG_INLINE_C_V2INT16:
1131   case AMDGPU::OPERAND_REG_INLINE_C_V2FP16:
1132     return &APFloat::IEEEhalf();
1133   default:
1134     llvm_unreachable("unsupported fp type");
1135   }
1136 }
1137 
1138 //===----------------------------------------------------------------------===//
1139 // Operand
1140 //===----------------------------------------------------------------------===//
1141 
1142 static bool canLosslesslyConvertToFPType(APFloat &FPLiteral, MVT VT) {
1143   bool Lost;
1144 
1145   // Convert literal to single precision
1146   APFloat::opStatus Status = FPLiteral.convert(*getFltSemantics(VT),
1147                                                APFloat::rmNearestTiesToEven,
1148                                                &Lost);
1149   // We allow precision lost but not overflow or underflow
1150   if (Status != APFloat::opOK &&
1151       Lost &&
1152       ((Status & APFloat::opOverflow)  != 0 ||
1153        (Status & APFloat::opUnderflow) != 0)) {
1154     return false;
1155   }
1156 
1157   return true;
1158 }
1159 
1160 bool AMDGPUOperand::isInlinableImm(MVT type) const {
1161   if (!isImmTy(ImmTyNone)) {
1162     // Only plain immediates are inlinable (e.g. "clamp" attribute is not)
1163     return false;
1164   }
1165   // TODO: We should avoid using host float here. It would be better to
1166   // check the float bit values which is what a few other places do.
1167   // We've had bot failures before due to weird NaN support on mips hosts.
1168 
1169   APInt Literal(64, Imm.Val);
1170 
1171   if (Imm.IsFPImm) { // We got fp literal token
1172     if (type == MVT::f64 || type == MVT::i64) { // Expected 64-bit operand
1173       return AMDGPU::isInlinableLiteral64(Imm.Val,
1174                                           AsmParser->hasInv2PiInlineImm());
1175     }
1176 
1177     APFloat FPLiteral(APFloat::IEEEdouble(), APInt(64, Imm.Val));
1178     if (!canLosslesslyConvertToFPType(FPLiteral, type))
1179       return false;
1180 
1181     if (type.getScalarSizeInBits() == 16) {
1182       return AMDGPU::isInlinableLiteral16(
1183         static_cast<int16_t>(FPLiteral.bitcastToAPInt().getZExtValue()),
1184         AsmParser->hasInv2PiInlineImm());
1185     }
1186 
1187     // Check if single precision literal is inlinable
1188     return AMDGPU::isInlinableLiteral32(
1189       static_cast<int32_t>(FPLiteral.bitcastToAPInt().getZExtValue()),
1190       AsmParser->hasInv2PiInlineImm());
1191   }
1192 
1193   // We got int literal token.
1194   if (type == MVT::f64 || type == MVT::i64) { // Expected 64-bit operand
1195     return AMDGPU::isInlinableLiteral64(Imm.Val,
1196                                         AsmParser->hasInv2PiInlineImm());
1197   }
1198 
1199   if (type.getScalarSizeInBits() == 16) {
1200     return AMDGPU::isInlinableLiteral16(
1201       static_cast<int16_t>(Literal.getLoBits(16).getSExtValue()),
1202       AsmParser->hasInv2PiInlineImm());
1203   }
1204 
1205   return AMDGPU::isInlinableLiteral32(
1206     static_cast<int32_t>(Literal.getLoBits(32).getZExtValue()),
1207     AsmParser->hasInv2PiInlineImm());
1208 }
1209 
1210 bool AMDGPUOperand::isLiteralImm(MVT type) const {
1211   // Check that this imediate can be added as literal
1212   if (!isImmTy(ImmTyNone)) {
1213     return false;
1214   }
1215 
1216   if (!Imm.IsFPImm) {
1217     // We got int literal token.
1218 
1219     if (type == MVT::f64 && hasFPModifiers()) {
1220       // Cannot apply fp modifiers to int literals preserving the same semantics
1221       // for VOP1/2/C and VOP3 because of integer truncation. To avoid ambiguity,
1222       // disable these cases.
1223       return false;
1224     }
1225 
1226     unsigned Size = type.getSizeInBits();
1227     if (Size == 64)
1228       Size = 32;
1229 
1230     // FIXME: 64-bit operands can zero extend, sign extend, or pad zeroes for FP
1231     // types.
1232     return isUIntN(Size, Imm.Val) || isIntN(Size, Imm.Val);
1233   }
1234 
1235   // We got fp literal token
1236   if (type == MVT::f64) { // Expected 64-bit fp operand
1237     // We would set low 64-bits of literal to zeroes but we accept this literals
1238     return true;
1239   }
1240 
1241   if (type == MVT::i64) { // Expected 64-bit int operand
1242     // We don't allow fp literals in 64-bit integer instructions. It is
1243     // unclear how we should encode them.
1244     return false;
1245   }
1246 
1247   APFloat FPLiteral(APFloat::IEEEdouble(), APInt(64, Imm.Val));
1248   return canLosslesslyConvertToFPType(FPLiteral, type);
1249 }
1250 
1251 bool AMDGPUOperand::isRegClass(unsigned RCID) const {
1252   return isRegKind() && AsmParser->getMRI()->getRegClass(RCID).contains(getReg());
1253 }
1254 
1255 bool AMDGPUOperand::isSDWARegKind() const {
1256   if (AsmParser->isVI())
1257     return isVReg();
1258   else if (AsmParser->isGFX9())
1259     return isRegKind();
1260   else
1261     return false;
1262 }
1263 
1264 uint64_t AMDGPUOperand::applyInputFPModifiers(uint64_t Val, unsigned Size) const
1265 {
1266   assert(isImmTy(ImmTyNone) && Imm.Mods.hasFPModifiers());
1267   assert(Size == 2 || Size == 4 || Size == 8);
1268 
1269   const uint64_t FpSignMask = (1ULL << (Size * 8 - 1));
1270 
1271   if (Imm.Mods.Abs) {
1272     Val &= ~FpSignMask;
1273   }
1274   if (Imm.Mods.Neg) {
1275     Val ^= FpSignMask;
1276   }
1277 
1278   return Val;
1279 }
1280 
1281 void AMDGPUOperand::addImmOperands(MCInst &Inst, unsigned N, bool ApplyModifiers) const {
1282 
1283   if (AMDGPU::isSISrcOperand(AsmParser->getMII()->get(Inst.getOpcode()),
1284                              Inst.getNumOperands())) {
1285     addLiteralImmOperand(Inst, Imm.Val,
1286                          ApplyModifiers &
1287                          isImmTy(ImmTyNone) && Imm.Mods.hasFPModifiers());
1288   } else {
1289     assert(!isImmTy(ImmTyNone) || !hasModifiers());
1290     Inst.addOperand(MCOperand::createImm(Imm.Val));
1291   }
1292 }
1293 
1294 void AMDGPUOperand::addLiteralImmOperand(MCInst &Inst, int64_t Val, bool ApplyModifiers) const {
1295   const auto& InstDesc = AsmParser->getMII()->get(Inst.getOpcode());
1296   auto OpNum = Inst.getNumOperands();
1297   // Check that this operand accepts literals
1298   assert(AMDGPU::isSISrcOperand(InstDesc, OpNum));
1299 
1300   if (ApplyModifiers) {
1301     assert(AMDGPU::isSISrcFPOperand(InstDesc, OpNum));
1302     const unsigned Size = Imm.IsFPImm ? sizeof(double) : getOperandSize(InstDesc, OpNum);
1303     Val = applyInputFPModifiers(Val, Size);
1304   }
1305 
1306   APInt Literal(64, Val);
1307   uint8_t OpTy = InstDesc.OpInfo[OpNum].OperandType;
1308 
1309   if (Imm.IsFPImm) { // We got fp literal token
1310     switch (OpTy) {
1311     case AMDGPU::OPERAND_REG_IMM_INT64:
1312     case AMDGPU::OPERAND_REG_IMM_FP64:
1313     case AMDGPU::OPERAND_REG_INLINE_C_INT64:
1314     case AMDGPU::OPERAND_REG_INLINE_C_FP64: {
1315       if (AMDGPU::isInlinableLiteral64(Literal.getZExtValue(),
1316                                        AsmParser->hasInv2PiInlineImm())) {
1317         Inst.addOperand(MCOperand::createImm(Literal.getZExtValue()));
1318         return;
1319       }
1320 
1321       // Non-inlineable
1322       if (AMDGPU::isSISrcFPOperand(InstDesc, OpNum)) { // Expected 64-bit fp operand
1323         // For fp operands we check if low 32 bits are zeros
1324         if (Literal.getLoBits(32) != 0) {
1325           const_cast<AMDGPUAsmParser *>(AsmParser)->Warning(Inst.getLoc(),
1326           "Can't encode literal as exact 64-bit floating-point operand. "
1327           "Low 32-bits will be set to zero");
1328         }
1329 
1330         Inst.addOperand(MCOperand::createImm(Literal.lshr(32).getZExtValue()));
1331         return;
1332       }
1333 
1334       // We don't allow fp literals in 64-bit integer instructions. It is
1335       // unclear how we should encode them. This case should be checked earlier
1336       // in predicate methods (isLiteralImm())
1337       llvm_unreachable("fp literal in 64-bit integer instruction.");
1338     }
1339     case AMDGPU::OPERAND_REG_IMM_INT32:
1340     case AMDGPU::OPERAND_REG_IMM_FP32:
1341     case AMDGPU::OPERAND_REG_INLINE_C_INT32:
1342     case AMDGPU::OPERAND_REG_INLINE_C_FP32:
1343     case AMDGPU::OPERAND_REG_IMM_INT16:
1344     case AMDGPU::OPERAND_REG_IMM_FP16:
1345     case AMDGPU::OPERAND_REG_INLINE_C_INT16:
1346     case AMDGPU::OPERAND_REG_INLINE_C_FP16:
1347     case AMDGPU::OPERAND_REG_INLINE_C_V2INT16:
1348     case AMDGPU::OPERAND_REG_INLINE_C_V2FP16: {
1349       bool lost;
1350       APFloat FPLiteral(APFloat::IEEEdouble(), Literal);
1351       // Convert literal to single precision
1352       FPLiteral.convert(*getOpFltSemantics(OpTy),
1353                         APFloat::rmNearestTiesToEven, &lost);
1354       // We allow precision lost but not overflow or underflow. This should be
1355       // checked earlier in isLiteralImm()
1356 
1357       uint64_t ImmVal = FPLiteral.bitcastToAPInt().getZExtValue();
1358       if (OpTy == AMDGPU::OPERAND_REG_INLINE_C_V2INT16 ||
1359           OpTy == AMDGPU::OPERAND_REG_INLINE_C_V2FP16) {
1360         ImmVal |= (ImmVal << 16);
1361       }
1362 
1363       Inst.addOperand(MCOperand::createImm(ImmVal));
1364       return;
1365     }
1366     default:
1367       llvm_unreachable("invalid operand size");
1368     }
1369 
1370     return;
1371   }
1372 
1373    // We got int literal token.
1374   // Only sign extend inline immediates.
1375   // FIXME: No errors on truncation
1376   switch (OpTy) {
1377   case AMDGPU::OPERAND_REG_IMM_INT32:
1378   case AMDGPU::OPERAND_REG_IMM_FP32:
1379   case AMDGPU::OPERAND_REG_INLINE_C_INT32:
1380   case AMDGPU::OPERAND_REG_INLINE_C_FP32: {
1381     if (isInt<32>(Val) &&
1382         AMDGPU::isInlinableLiteral32(static_cast<int32_t>(Val),
1383                                      AsmParser->hasInv2PiInlineImm())) {
1384       Inst.addOperand(MCOperand::createImm(Val));
1385       return;
1386     }
1387 
1388     Inst.addOperand(MCOperand::createImm(Val & 0xffffffff));
1389     return;
1390   }
1391   case AMDGPU::OPERAND_REG_IMM_INT64:
1392   case AMDGPU::OPERAND_REG_IMM_FP64:
1393   case AMDGPU::OPERAND_REG_INLINE_C_INT64:
1394   case AMDGPU::OPERAND_REG_INLINE_C_FP64: {
1395     if (AMDGPU::isInlinableLiteral64(Val, AsmParser->hasInv2PiInlineImm())) {
1396       Inst.addOperand(MCOperand::createImm(Val));
1397       return;
1398     }
1399 
1400     Inst.addOperand(MCOperand::createImm(Lo_32(Val)));
1401     return;
1402   }
1403   case AMDGPU::OPERAND_REG_IMM_INT16:
1404   case AMDGPU::OPERAND_REG_IMM_FP16:
1405   case AMDGPU::OPERAND_REG_INLINE_C_INT16:
1406   case AMDGPU::OPERAND_REG_INLINE_C_FP16: {
1407     if (isInt<16>(Val) &&
1408         AMDGPU::isInlinableLiteral16(static_cast<int16_t>(Val),
1409                                      AsmParser->hasInv2PiInlineImm())) {
1410       Inst.addOperand(MCOperand::createImm(Val));
1411       return;
1412     }
1413 
1414     Inst.addOperand(MCOperand::createImm(Val & 0xffff));
1415     return;
1416   }
1417   case AMDGPU::OPERAND_REG_INLINE_C_V2INT16:
1418   case AMDGPU::OPERAND_REG_INLINE_C_V2FP16: {
1419     auto LiteralVal = static_cast<uint16_t>(Literal.getLoBits(16).getZExtValue());
1420     assert(AMDGPU::isInlinableLiteral16(LiteralVal,
1421                                         AsmParser->hasInv2PiInlineImm()));
1422 
1423     uint32_t ImmVal = static_cast<uint32_t>(LiteralVal) << 16 |
1424                       static_cast<uint32_t>(LiteralVal);
1425     Inst.addOperand(MCOperand::createImm(ImmVal));
1426     return;
1427   }
1428   default:
1429     llvm_unreachable("invalid operand size");
1430   }
1431 }
1432 
1433 template <unsigned Bitwidth>
1434 void AMDGPUOperand::addKImmFPOperands(MCInst &Inst, unsigned N) const {
1435   APInt Literal(64, Imm.Val);
1436 
1437   if (!Imm.IsFPImm) {
1438     // We got int literal token.
1439     Inst.addOperand(MCOperand::createImm(Literal.getLoBits(Bitwidth).getZExtValue()));
1440     return;
1441   }
1442 
1443   bool Lost;
1444   APFloat FPLiteral(APFloat::IEEEdouble(), Literal);
1445   FPLiteral.convert(*getFltSemantics(Bitwidth / 8),
1446                     APFloat::rmNearestTiesToEven, &Lost);
1447   Inst.addOperand(MCOperand::createImm(FPLiteral.bitcastToAPInt().getZExtValue()));
1448 }
1449 
1450 void AMDGPUOperand::addRegOperands(MCInst &Inst, unsigned N) const {
1451   Inst.addOperand(MCOperand::createReg(AMDGPU::getMCReg(getReg(), AsmParser->getSTI())));
1452 }
1453 
1454 //===----------------------------------------------------------------------===//
1455 // AsmParser
1456 //===----------------------------------------------------------------------===//
1457 
1458 static int getRegClass(RegisterKind Is, unsigned RegWidth) {
1459   if (Is == IS_VGPR) {
1460     switch (RegWidth) {
1461       default: return -1;
1462       case 1: return AMDGPU::VGPR_32RegClassID;
1463       case 2: return AMDGPU::VReg_64RegClassID;
1464       case 3: return AMDGPU::VReg_96RegClassID;
1465       case 4: return AMDGPU::VReg_128RegClassID;
1466       case 8: return AMDGPU::VReg_256RegClassID;
1467       case 16: return AMDGPU::VReg_512RegClassID;
1468     }
1469   } else if (Is == IS_TTMP) {
1470     switch (RegWidth) {
1471       default: return -1;
1472       case 1: return AMDGPU::TTMP_32RegClassID;
1473       case 2: return AMDGPU::TTMP_64RegClassID;
1474       case 4: return AMDGPU::TTMP_128RegClassID;
1475     }
1476   } else if (Is == IS_SGPR) {
1477     switch (RegWidth) {
1478       default: return -1;
1479       case 1: return AMDGPU::SGPR_32RegClassID;
1480       case 2: return AMDGPU::SGPR_64RegClassID;
1481       case 4: return AMDGPU::SGPR_128RegClassID;
1482       case 8: return AMDGPU::SReg_256RegClassID;
1483       case 16: return AMDGPU::SReg_512RegClassID;
1484     }
1485   }
1486   return -1;
1487 }
1488 
1489 static unsigned getSpecialRegForName(StringRef RegName) {
1490   return StringSwitch<unsigned>(RegName)
1491     .Case("exec", AMDGPU::EXEC)
1492     .Case("vcc", AMDGPU::VCC)
1493     .Case("flat_scratch", AMDGPU::FLAT_SCR)
1494     .Case("m0", AMDGPU::M0)
1495     .Case("scc", AMDGPU::SCC)
1496     .Case("tba", AMDGPU::TBA)
1497     .Case("tma", AMDGPU::TMA)
1498     .Case("flat_scratch_lo", AMDGPU::FLAT_SCR_LO)
1499     .Case("flat_scratch_hi", AMDGPU::FLAT_SCR_HI)
1500     .Case("vcc_lo", AMDGPU::VCC_LO)
1501     .Case("vcc_hi", AMDGPU::VCC_HI)
1502     .Case("exec_lo", AMDGPU::EXEC_LO)
1503     .Case("exec_hi", AMDGPU::EXEC_HI)
1504     .Case("tma_lo", AMDGPU::TMA_LO)
1505     .Case("tma_hi", AMDGPU::TMA_HI)
1506     .Case("tba_lo", AMDGPU::TBA_LO)
1507     .Case("tba_hi", AMDGPU::TBA_HI)
1508     .Default(0);
1509 }
1510 
1511 bool AMDGPUAsmParser::ParseRegister(unsigned &RegNo, SMLoc &StartLoc,
1512                                     SMLoc &EndLoc) {
1513   auto R = parseRegister();
1514   if (!R) return true;
1515   assert(R->isReg());
1516   RegNo = R->getReg();
1517   StartLoc = R->getStartLoc();
1518   EndLoc = R->getEndLoc();
1519   return false;
1520 }
1521 
1522 bool AMDGPUAsmParser::AddNextRegisterToList(unsigned &Reg, unsigned &RegWidth,
1523                                             RegisterKind RegKind, unsigned Reg1,
1524                                             unsigned RegNum) {
1525   switch (RegKind) {
1526   case IS_SPECIAL:
1527     if (Reg == AMDGPU::EXEC_LO && Reg1 == AMDGPU::EXEC_HI) {
1528       Reg = AMDGPU::EXEC;
1529       RegWidth = 2;
1530       return true;
1531     }
1532     if (Reg == AMDGPU::FLAT_SCR_LO && Reg1 == AMDGPU::FLAT_SCR_HI) {
1533       Reg = AMDGPU::FLAT_SCR;
1534       RegWidth = 2;
1535       return true;
1536     }
1537     if (Reg == AMDGPU::VCC_LO && Reg1 == AMDGPU::VCC_HI) {
1538       Reg = AMDGPU::VCC;
1539       RegWidth = 2;
1540       return true;
1541     }
1542     if (Reg == AMDGPU::TBA_LO && Reg1 == AMDGPU::TBA_HI) {
1543       Reg = AMDGPU::TBA;
1544       RegWidth = 2;
1545       return true;
1546     }
1547     if (Reg == AMDGPU::TMA_LO && Reg1 == AMDGPU::TMA_HI) {
1548       Reg = AMDGPU::TMA;
1549       RegWidth = 2;
1550       return true;
1551     }
1552     return false;
1553   case IS_VGPR:
1554   case IS_SGPR:
1555   case IS_TTMP:
1556     if (Reg1 != Reg + RegWidth) {
1557       return false;
1558     }
1559     RegWidth++;
1560     return true;
1561   default:
1562     llvm_unreachable("unexpected register kind");
1563   }
1564 }
1565 
1566 bool AMDGPUAsmParser::ParseAMDGPURegister(RegisterKind &RegKind, unsigned &Reg,
1567                                           unsigned &RegNum, unsigned &RegWidth,
1568                                           unsigned *DwordRegIndex) {
1569   if (DwordRegIndex) { *DwordRegIndex = 0; }
1570   const MCRegisterInfo *TRI = getContext().getRegisterInfo();
1571   if (getLexer().is(AsmToken::Identifier)) {
1572     StringRef RegName = Parser.getTok().getString();
1573     if ((Reg = getSpecialRegForName(RegName))) {
1574       Parser.Lex();
1575       RegKind = IS_SPECIAL;
1576     } else {
1577       unsigned RegNumIndex = 0;
1578       if (RegName[0] == 'v') {
1579         RegNumIndex = 1;
1580         RegKind = IS_VGPR;
1581       } else if (RegName[0] == 's') {
1582         RegNumIndex = 1;
1583         RegKind = IS_SGPR;
1584       } else if (RegName.startswith("ttmp")) {
1585         RegNumIndex = strlen("ttmp");
1586         RegKind = IS_TTMP;
1587       } else {
1588         return false;
1589       }
1590       if (RegName.size() > RegNumIndex) {
1591         // Single 32-bit register: vXX.
1592         if (RegName.substr(RegNumIndex).getAsInteger(10, RegNum))
1593           return false;
1594         Parser.Lex();
1595         RegWidth = 1;
1596       } else {
1597         // Range of registers: v[XX:YY]. ":YY" is optional.
1598         Parser.Lex();
1599         int64_t RegLo, RegHi;
1600         if (getLexer().isNot(AsmToken::LBrac))
1601           return false;
1602         Parser.Lex();
1603 
1604         if (getParser().parseAbsoluteExpression(RegLo))
1605           return false;
1606 
1607         const bool isRBrace = getLexer().is(AsmToken::RBrac);
1608         if (!isRBrace && getLexer().isNot(AsmToken::Colon))
1609           return false;
1610         Parser.Lex();
1611 
1612         if (isRBrace) {
1613           RegHi = RegLo;
1614         } else {
1615           if (getParser().parseAbsoluteExpression(RegHi))
1616             return false;
1617 
1618           if (getLexer().isNot(AsmToken::RBrac))
1619             return false;
1620           Parser.Lex();
1621         }
1622         RegNum = (unsigned) RegLo;
1623         RegWidth = (RegHi - RegLo) + 1;
1624       }
1625     }
1626   } else if (getLexer().is(AsmToken::LBrac)) {
1627     // List of consecutive registers: [s0,s1,s2,s3]
1628     Parser.Lex();
1629     if (!ParseAMDGPURegister(RegKind, Reg, RegNum, RegWidth, nullptr))
1630       return false;
1631     if (RegWidth != 1)
1632       return false;
1633     RegisterKind RegKind1;
1634     unsigned Reg1, RegNum1, RegWidth1;
1635     do {
1636       if (getLexer().is(AsmToken::Comma)) {
1637         Parser.Lex();
1638       } else if (getLexer().is(AsmToken::RBrac)) {
1639         Parser.Lex();
1640         break;
1641       } else if (ParseAMDGPURegister(RegKind1, Reg1, RegNum1, RegWidth1, nullptr)) {
1642         if (RegWidth1 != 1) {
1643           return false;
1644         }
1645         if (RegKind1 != RegKind) {
1646           return false;
1647         }
1648         if (!AddNextRegisterToList(Reg, RegWidth, RegKind1, Reg1, RegNum1)) {
1649           return false;
1650         }
1651       } else {
1652         return false;
1653       }
1654     } while (true);
1655   } else {
1656     return false;
1657   }
1658   switch (RegKind) {
1659   case IS_SPECIAL:
1660     RegNum = 0;
1661     RegWidth = 1;
1662     break;
1663   case IS_VGPR:
1664   case IS_SGPR:
1665   case IS_TTMP:
1666   {
1667     unsigned Size = 1;
1668     if (RegKind == IS_SGPR || RegKind == IS_TTMP) {
1669       // SGPR and TTMP registers must be aligned. Max required alignment is 4 dwords.
1670       Size = std::min(RegWidth, 4u);
1671     }
1672     if (RegNum % Size != 0)
1673       return false;
1674     if (DwordRegIndex) { *DwordRegIndex = RegNum; }
1675     RegNum = RegNum / Size;
1676     int RCID = getRegClass(RegKind, RegWidth);
1677     if (RCID == -1)
1678       return false;
1679     const MCRegisterClass RC = TRI->getRegClass(RCID);
1680     if (RegNum >= RC.getNumRegs())
1681       return false;
1682     Reg = RC.getRegister(RegNum);
1683     break;
1684   }
1685 
1686   default:
1687     llvm_unreachable("unexpected register kind");
1688   }
1689 
1690   if (!subtargetHasRegister(*TRI, Reg))
1691     return false;
1692   return true;
1693 }
1694 
1695 std::unique_ptr<AMDGPUOperand> AMDGPUAsmParser::parseRegister() {
1696   const auto &Tok = Parser.getTok();
1697   SMLoc StartLoc = Tok.getLoc();
1698   SMLoc EndLoc = Tok.getEndLoc();
1699   RegisterKind RegKind;
1700   unsigned Reg, RegNum, RegWidth, DwordRegIndex;
1701 
1702   if (!ParseAMDGPURegister(RegKind, Reg, RegNum, RegWidth, &DwordRegIndex)) {
1703     return nullptr;
1704   }
1705   KernelScope.usesRegister(RegKind, DwordRegIndex, RegWidth);
1706   return AMDGPUOperand::CreateReg(this, Reg, StartLoc, EndLoc, false);
1707 }
1708 
1709 bool
1710 AMDGPUAsmParser::parseAbsoluteExpr(int64_t &Val, bool AbsMod) {
1711   if (AbsMod && getLexer().peekTok().is(AsmToken::Pipe) &&
1712       (getLexer().getKind() == AsmToken::Integer ||
1713        getLexer().getKind() == AsmToken::Real)) {
1714 
1715     // This is a workaround for handling operands like these:
1716     //     |1.0|
1717     //     |-1|
1718     // This syntax is not compatible with syntax of standard
1719     // MC expressions (due to the trailing '|').
1720 
1721     SMLoc EndLoc;
1722     const MCExpr *Expr;
1723 
1724     if (getParser().parsePrimaryExpr(Expr, EndLoc)) {
1725       return true;
1726     }
1727 
1728     return !Expr->evaluateAsAbsolute(Val);
1729   }
1730 
1731   return getParser().parseAbsoluteExpression(Val);
1732 }
1733 
1734 OperandMatchResultTy
1735 AMDGPUAsmParser::parseImm(OperandVector &Operands, bool AbsMod) {
1736   // TODO: add syntactic sugar for 1/(2*PI)
1737   bool Minus = false;
1738   if (getLexer().getKind() == AsmToken::Minus) {
1739     Minus = true;
1740     Parser.Lex();
1741   }
1742 
1743   SMLoc S = Parser.getTok().getLoc();
1744   switch(getLexer().getKind()) {
1745   case AsmToken::Integer: {
1746     int64_t IntVal;
1747     if (parseAbsoluteExpr(IntVal, AbsMod))
1748       return MatchOperand_ParseFail;
1749     if (Minus)
1750       IntVal *= -1;
1751     Operands.push_back(AMDGPUOperand::CreateImm(this, IntVal, S));
1752     return MatchOperand_Success;
1753   }
1754   case AsmToken::Real: {
1755     int64_t IntVal;
1756     if (parseAbsoluteExpr(IntVal, AbsMod))
1757       return MatchOperand_ParseFail;
1758 
1759     APFloat F(BitsToDouble(IntVal));
1760     if (Minus)
1761       F.changeSign();
1762     Operands.push_back(
1763         AMDGPUOperand::CreateImm(this, F.bitcastToAPInt().getZExtValue(), S,
1764                                  AMDGPUOperand::ImmTyNone, true));
1765     return MatchOperand_Success;
1766   }
1767   default:
1768     return Minus ? MatchOperand_ParseFail : MatchOperand_NoMatch;
1769   }
1770 }
1771 
1772 OperandMatchResultTy
1773 AMDGPUAsmParser::parseReg(OperandVector &Operands) {
1774   if (auto R = parseRegister()) {
1775     assert(R->isReg());
1776     R->Reg.IsForcedVOP3 = isForcedVOP3();
1777     Operands.push_back(std::move(R));
1778     return MatchOperand_Success;
1779   }
1780   return MatchOperand_NoMatch;
1781 }
1782 
1783 OperandMatchResultTy
1784 AMDGPUAsmParser::parseRegOrImm(OperandVector &Operands, bool AbsMod) {
1785   auto res = parseImm(Operands, AbsMod);
1786   if (res != MatchOperand_NoMatch) {
1787     return res;
1788   }
1789 
1790   return parseReg(Operands);
1791 }
1792 
1793 OperandMatchResultTy
1794 AMDGPUAsmParser::parseRegOrImmWithFPInputMods(OperandVector &Operands,
1795                                               bool AllowImm) {
1796   bool Negate = false, Negate2 = false, Abs = false, Abs2 = false;
1797 
1798   if (getLexer().getKind()== AsmToken::Minus) {
1799     const AsmToken NextToken = getLexer().peekTok();
1800 
1801     // Disable ambiguous constructs like '--1' etc. Should use neg(-1) instead.
1802     if (NextToken.is(AsmToken::Minus)) {
1803       Error(Parser.getTok().getLoc(), "invalid syntax, expected 'neg' modifier");
1804       return MatchOperand_ParseFail;
1805     }
1806 
1807     // '-' followed by an integer literal N should be interpreted as integer
1808     // negation rather than a floating-point NEG modifier applied to N.
1809     // Beside being contr-intuitive, such use of floating-point NEG modifier
1810     // results in different meaning of integer literals used with VOP1/2/C
1811     // and VOP3, for example:
1812     //    v_exp_f32_e32 v5, -1 // VOP1: src0 = 0xFFFFFFFF
1813     //    v_exp_f32_e64 v5, -1 // VOP3: src0 = 0x80000001
1814     // Negative fp literals should be handled likewise for unifomtity
1815     if (!NextToken.is(AsmToken::Integer) && !NextToken.is(AsmToken::Real)) {
1816       Parser.Lex();
1817       Negate = true;
1818     }
1819   }
1820 
1821   if (getLexer().getKind() == AsmToken::Identifier &&
1822       Parser.getTok().getString() == "neg") {
1823     if (Negate) {
1824       Error(Parser.getTok().getLoc(), "expected register or immediate");
1825       return MatchOperand_ParseFail;
1826     }
1827     Parser.Lex();
1828     Negate2 = true;
1829     if (getLexer().isNot(AsmToken::LParen)) {
1830       Error(Parser.getTok().getLoc(), "expected left paren after neg");
1831       return MatchOperand_ParseFail;
1832     }
1833     Parser.Lex();
1834   }
1835 
1836   if (getLexer().getKind() == AsmToken::Identifier &&
1837       Parser.getTok().getString() == "abs") {
1838     Parser.Lex();
1839     Abs2 = true;
1840     if (getLexer().isNot(AsmToken::LParen)) {
1841       Error(Parser.getTok().getLoc(), "expected left paren after abs");
1842       return MatchOperand_ParseFail;
1843     }
1844     Parser.Lex();
1845   }
1846 
1847   if (getLexer().getKind() == AsmToken::Pipe) {
1848     if (Abs2) {
1849       Error(Parser.getTok().getLoc(), "expected register or immediate");
1850       return MatchOperand_ParseFail;
1851     }
1852     Parser.Lex();
1853     Abs = true;
1854   }
1855 
1856   OperandMatchResultTy Res;
1857   if (AllowImm) {
1858     Res = parseRegOrImm(Operands, Abs);
1859   } else {
1860     Res = parseReg(Operands);
1861   }
1862   if (Res != MatchOperand_Success) {
1863     return Res;
1864   }
1865 
1866   AMDGPUOperand::Modifiers Mods;
1867   if (Abs) {
1868     if (getLexer().getKind() != AsmToken::Pipe) {
1869       Error(Parser.getTok().getLoc(), "expected vertical bar");
1870       return MatchOperand_ParseFail;
1871     }
1872     Parser.Lex();
1873     Mods.Abs = true;
1874   }
1875   if (Abs2) {
1876     if (getLexer().isNot(AsmToken::RParen)) {
1877       Error(Parser.getTok().getLoc(), "expected closing parentheses");
1878       return MatchOperand_ParseFail;
1879     }
1880     Parser.Lex();
1881     Mods.Abs = true;
1882   }
1883 
1884   if (Negate) {
1885     Mods.Neg = true;
1886   } else if (Negate2) {
1887     if (getLexer().isNot(AsmToken::RParen)) {
1888       Error(Parser.getTok().getLoc(), "expected closing parentheses");
1889       return MatchOperand_ParseFail;
1890     }
1891     Parser.Lex();
1892     Mods.Neg = true;
1893   }
1894 
1895   if (Mods.hasFPModifiers()) {
1896     AMDGPUOperand &Op = static_cast<AMDGPUOperand &>(*Operands.back());
1897     Op.setModifiers(Mods);
1898   }
1899   return MatchOperand_Success;
1900 }
1901 
1902 OperandMatchResultTy
1903 AMDGPUAsmParser::parseRegOrImmWithIntInputMods(OperandVector &Operands,
1904                                                bool AllowImm) {
1905   bool Sext = false;
1906 
1907   if (getLexer().getKind() == AsmToken::Identifier &&
1908       Parser.getTok().getString() == "sext") {
1909     Parser.Lex();
1910     Sext = true;
1911     if (getLexer().isNot(AsmToken::LParen)) {
1912       Error(Parser.getTok().getLoc(), "expected left paren after sext");
1913       return MatchOperand_ParseFail;
1914     }
1915     Parser.Lex();
1916   }
1917 
1918   OperandMatchResultTy Res;
1919   if (AllowImm) {
1920     Res = parseRegOrImm(Operands);
1921   } else {
1922     Res = parseReg(Operands);
1923   }
1924   if (Res != MatchOperand_Success) {
1925     return Res;
1926   }
1927 
1928   AMDGPUOperand::Modifiers Mods;
1929   if (Sext) {
1930     if (getLexer().isNot(AsmToken::RParen)) {
1931       Error(Parser.getTok().getLoc(), "expected closing parentheses");
1932       return MatchOperand_ParseFail;
1933     }
1934     Parser.Lex();
1935     Mods.Sext = true;
1936   }
1937 
1938   if (Mods.hasIntModifiers()) {
1939     AMDGPUOperand &Op = static_cast<AMDGPUOperand &>(*Operands.back());
1940     Op.setModifiers(Mods);
1941   }
1942 
1943   return MatchOperand_Success;
1944 }
1945 
1946 OperandMatchResultTy
1947 AMDGPUAsmParser::parseRegWithFPInputMods(OperandVector &Operands) {
1948   return parseRegOrImmWithFPInputMods(Operands, false);
1949 }
1950 
1951 OperandMatchResultTy
1952 AMDGPUAsmParser::parseRegWithIntInputMods(OperandVector &Operands) {
1953   return parseRegOrImmWithIntInputMods(Operands, false);
1954 }
1955 
1956 OperandMatchResultTy AMDGPUAsmParser::parseVReg32OrOff(OperandVector &Operands) {
1957   std::unique_ptr<AMDGPUOperand> Reg = parseRegister();
1958   if (Reg) {
1959     Operands.push_back(std::move(Reg));
1960     return MatchOperand_Success;
1961   }
1962 
1963   const AsmToken &Tok = Parser.getTok();
1964   if (Tok.getString() == "off") {
1965     Operands.push_back(AMDGPUOperand::CreateImm(this, 0, Tok.getLoc(),
1966                                                 AMDGPUOperand::ImmTyOff, false));
1967     Parser.Lex();
1968     return MatchOperand_Success;
1969   }
1970 
1971   return MatchOperand_NoMatch;
1972 }
1973 
1974 unsigned AMDGPUAsmParser::checkTargetMatchPredicate(MCInst &Inst) {
1975   uint64_t TSFlags = MII.get(Inst.getOpcode()).TSFlags;
1976 
1977   if ((getForcedEncodingSize() == 32 && (TSFlags & SIInstrFlags::VOP3)) ||
1978       (getForcedEncodingSize() == 64 && !(TSFlags & SIInstrFlags::VOP3)) ||
1979       (isForcedDPP() && !(TSFlags & SIInstrFlags::DPP)) ||
1980       (isForcedSDWA() && !(TSFlags & SIInstrFlags::SDWA)) )
1981     return Match_InvalidOperand;
1982 
1983   if ((TSFlags & SIInstrFlags::VOP3) &&
1984       (TSFlags & SIInstrFlags::VOPAsmPrefer32Bit) &&
1985       getForcedEncodingSize() != 64)
1986     return Match_PreferE32;
1987 
1988   if (Inst.getOpcode() == AMDGPU::V_MAC_F32_sdwa_vi ||
1989       Inst.getOpcode() == AMDGPU::V_MAC_F16_sdwa_vi) {
1990     // v_mac_f32/16 allow only dst_sel == DWORD;
1991     auto OpNum =
1992         AMDGPU::getNamedOperandIdx(Inst.getOpcode(), AMDGPU::OpName::dst_sel);
1993     const auto &Op = Inst.getOperand(OpNum);
1994     if (!Op.isImm() || Op.getImm() != AMDGPU::SDWA::SdwaSel::DWORD) {
1995       return Match_InvalidOperand;
1996     }
1997   }
1998 
1999   if ((TSFlags & SIInstrFlags::FLAT) && !hasFlatOffsets()) {
2000     // FIXME: Produces error without correct column reported.
2001     auto OpNum =
2002         AMDGPU::getNamedOperandIdx(Inst.getOpcode(), AMDGPU::OpName::offset);
2003     const auto &Op = Inst.getOperand(OpNum);
2004     if (Op.getImm() != 0)
2005       return Match_InvalidOperand;
2006   }
2007 
2008   return Match_Success;
2009 }
2010 
2011 // What asm variants we should check
2012 ArrayRef<unsigned> AMDGPUAsmParser::getMatchedVariants() const {
2013   if (getForcedEncodingSize() == 32) {
2014     static const unsigned Variants[] = {AMDGPUAsmVariants::DEFAULT};
2015     return makeArrayRef(Variants);
2016   }
2017 
2018   if (isForcedVOP3()) {
2019     static const unsigned Variants[] = {AMDGPUAsmVariants::VOP3};
2020     return makeArrayRef(Variants);
2021   }
2022 
2023   if (isForcedSDWA()) {
2024     static const unsigned Variants[] = {AMDGPUAsmVariants::SDWA,
2025                                         AMDGPUAsmVariants::SDWA9};
2026     return makeArrayRef(Variants);
2027   }
2028 
2029   if (isForcedDPP()) {
2030     static const unsigned Variants[] = {AMDGPUAsmVariants::DPP};
2031     return makeArrayRef(Variants);
2032   }
2033 
2034   static const unsigned Variants[] = {
2035     AMDGPUAsmVariants::DEFAULT, AMDGPUAsmVariants::VOP3,
2036     AMDGPUAsmVariants::SDWA, AMDGPUAsmVariants::SDWA9, AMDGPUAsmVariants::DPP
2037   };
2038 
2039   return makeArrayRef(Variants);
2040 }
2041 
2042 unsigned AMDGPUAsmParser::findImplicitSGPRReadInVOP(const MCInst &Inst) const {
2043   const MCInstrDesc &Desc = MII.get(Inst.getOpcode());
2044   const unsigned Num = Desc.getNumImplicitUses();
2045   for (unsigned i = 0; i < Num; ++i) {
2046     unsigned Reg = Desc.ImplicitUses[i];
2047     switch (Reg) {
2048     case AMDGPU::FLAT_SCR:
2049     case AMDGPU::VCC:
2050     case AMDGPU::M0:
2051       return Reg;
2052     default:
2053       break;
2054     }
2055   }
2056   return AMDGPU::NoRegister;
2057 }
2058 
2059 // NB: This code is correct only when used to check constant
2060 // bus limitations because GFX7 support no f16 inline constants.
2061 // Note that there are no cases when a GFX7 opcode violates
2062 // constant bus limitations due to the use of an f16 constant.
2063 bool AMDGPUAsmParser::isInlineConstant(const MCInst &Inst,
2064                                        unsigned OpIdx) const {
2065   const MCInstrDesc &Desc = MII.get(Inst.getOpcode());
2066 
2067   if (!AMDGPU::isSISrcOperand(Desc, OpIdx)) {
2068     return false;
2069   }
2070 
2071   const MCOperand &MO = Inst.getOperand(OpIdx);
2072 
2073   int64_t Val = MO.getImm();
2074   auto OpSize = AMDGPU::getOperandSize(Desc, OpIdx);
2075 
2076   switch (OpSize) { // expected operand size
2077   case 8:
2078     return AMDGPU::isInlinableLiteral64(Val, hasInv2PiInlineImm());
2079   case 4:
2080     return AMDGPU::isInlinableLiteral32(Val, hasInv2PiInlineImm());
2081   case 2: {
2082     const unsigned OperandType = Desc.OpInfo[OpIdx].OperandType;
2083     if (OperandType == AMDGPU::OPERAND_REG_INLINE_C_V2INT16 ||
2084         OperandType == AMDGPU::OPERAND_REG_INLINE_C_V2FP16) {
2085       return AMDGPU::isInlinableLiteralV216(Val, hasInv2PiInlineImm());
2086     } else {
2087       return AMDGPU::isInlinableLiteral16(Val, hasInv2PiInlineImm());
2088     }
2089   }
2090   default:
2091     llvm_unreachable("invalid operand size");
2092   }
2093 }
2094 
2095 bool AMDGPUAsmParser::usesConstantBus(const MCInst &Inst, unsigned OpIdx) {
2096   const MCOperand &MO = Inst.getOperand(OpIdx);
2097   if (MO.isImm()) {
2098     return !isInlineConstant(Inst, OpIdx);
2099   }
2100   return !MO.isReg() ||
2101          isSGPR(mc2PseudoReg(MO.getReg()), getContext().getRegisterInfo());
2102 }
2103 
2104 bool AMDGPUAsmParser::validateConstantBusLimitations(const MCInst &Inst) {
2105   const unsigned Opcode = Inst.getOpcode();
2106   const MCInstrDesc &Desc = MII.get(Opcode);
2107   unsigned ConstantBusUseCount = 0;
2108 
2109   if (Desc.TSFlags &
2110       (SIInstrFlags::VOPC |
2111        SIInstrFlags::VOP1 | SIInstrFlags::VOP2 |
2112        SIInstrFlags::VOP3 | SIInstrFlags::VOP3P |
2113        SIInstrFlags::SDWA)) {
2114 
2115     // Check special imm operands (used by madmk, etc)
2116     if (AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::imm) != -1) {
2117       ++ConstantBusUseCount;
2118     }
2119 
2120     unsigned SGPRUsed = findImplicitSGPRReadInVOP(Inst);
2121     if (SGPRUsed != AMDGPU::NoRegister) {
2122       ++ConstantBusUseCount;
2123     }
2124 
2125     const int Src0Idx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::src0);
2126     const int Src1Idx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::src1);
2127     const int Src2Idx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::src2);
2128 
2129     const int OpIndices[] = { Src0Idx, Src1Idx, Src2Idx };
2130 
2131     for (int OpIdx : OpIndices) {
2132       if (OpIdx == -1) break;
2133 
2134       const MCOperand &MO = Inst.getOperand(OpIdx);
2135       if (usesConstantBus(Inst, OpIdx)) {
2136         if (MO.isReg()) {
2137           const unsigned Reg = mc2PseudoReg(MO.getReg());
2138           // Pairs of registers with a partial intersections like these
2139           //   s0, s[0:1]
2140           //   flat_scratch_lo, flat_scratch
2141           //   flat_scratch_lo, flat_scratch_hi
2142           // are theoretically valid but they are disabled anyway.
2143           // Note that this code mimics SIInstrInfo::verifyInstruction
2144           if (Reg != SGPRUsed) {
2145             ++ConstantBusUseCount;
2146           }
2147           SGPRUsed = Reg;
2148         } else { // Expression or a literal
2149           ++ConstantBusUseCount;
2150         }
2151       }
2152     }
2153   }
2154 
2155   return ConstantBusUseCount <= 1;
2156 }
2157 
2158 bool AMDGPUAsmParser::validateEarlyClobberLimitations(const MCInst &Inst) {
2159 
2160   const unsigned Opcode = Inst.getOpcode();
2161   const MCInstrDesc &Desc = MII.get(Opcode);
2162 
2163   const int DstIdx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::vdst);
2164   if (DstIdx == -1 ||
2165       Desc.getOperandConstraint(DstIdx, MCOI::EARLY_CLOBBER) == -1) {
2166     return true;
2167   }
2168 
2169   const MCRegisterInfo *TRI = getContext().getRegisterInfo();
2170 
2171   const int Src0Idx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::src0);
2172   const int Src1Idx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::src1);
2173   const int Src2Idx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::src2);
2174 
2175   assert(DstIdx != -1);
2176   const MCOperand &Dst = Inst.getOperand(DstIdx);
2177   assert(Dst.isReg());
2178   const unsigned DstReg = mc2PseudoReg(Dst.getReg());
2179 
2180   const int SrcIndices[] = { Src0Idx, Src1Idx, Src2Idx };
2181 
2182   for (int SrcIdx : SrcIndices) {
2183     if (SrcIdx == -1) break;
2184     const MCOperand &Src = Inst.getOperand(SrcIdx);
2185     if (Src.isReg()) {
2186       const unsigned SrcReg = mc2PseudoReg(Src.getReg());
2187       if (isRegIntersect(DstReg, SrcReg, TRI)) {
2188         return false;
2189       }
2190     }
2191   }
2192 
2193   return true;
2194 }
2195 
2196 bool AMDGPUAsmParser::validateInstruction(const MCInst &Inst,
2197                                           const SMLoc &IDLoc) {
2198   if (!validateConstantBusLimitations(Inst)) {
2199     Error(IDLoc,
2200       "invalid operand (violates constant bus restrictions)");
2201     return false;
2202   }
2203   if (!validateEarlyClobberLimitations(Inst)) {
2204     Error(IDLoc,
2205       "destination must be different than all sources");
2206     return false;
2207   }
2208 
2209   return true;
2210 }
2211 
2212 bool AMDGPUAsmParser::MatchAndEmitInstruction(SMLoc IDLoc, unsigned &Opcode,
2213                                               OperandVector &Operands,
2214                                               MCStreamer &Out,
2215                                               uint64_t &ErrorInfo,
2216                                               bool MatchingInlineAsm) {
2217   MCInst Inst;
2218   unsigned Result = Match_Success;
2219   for (auto Variant : getMatchedVariants()) {
2220     uint64_t EI;
2221     auto R = MatchInstructionImpl(Operands, Inst, EI, MatchingInlineAsm,
2222                                   Variant);
2223     // We order match statuses from least to most specific. We use most specific
2224     // status as resulting
2225     // Match_MnemonicFail < Match_InvalidOperand < Match_MissingFeature < Match_PreferE32
2226     if ((R == Match_Success) ||
2227         (R == Match_PreferE32) ||
2228         (R == Match_MissingFeature && Result != Match_PreferE32) ||
2229         (R == Match_InvalidOperand && Result != Match_MissingFeature
2230                                    && Result != Match_PreferE32) ||
2231         (R == Match_MnemonicFail   && Result != Match_InvalidOperand
2232                                    && Result != Match_MissingFeature
2233                                    && Result != Match_PreferE32)) {
2234       Result = R;
2235       ErrorInfo = EI;
2236     }
2237     if (R == Match_Success)
2238       break;
2239   }
2240 
2241   switch (Result) {
2242   default: break;
2243   case Match_Success:
2244     if (!validateInstruction(Inst, IDLoc)) {
2245       return true;
2246     }
2247     Inst.setLoc(IDLoc);
2248     Out.EmitInstruction(Inst, getSTI());
2249     return false;
2250 
2251   case Match_MissingFeature:
2252     return Error(IDLoc, "instruction not supported on this GPU");
2253 
2254   case Match_MnemonicFail:
2255     return Error(IDLoc, "unrecognized instruction mnemonic");
2256 
2257   case Match_InvalidOperand: {
2258     SMLoc ErrorLoc = IDLoc;
2259     if (ErrorInfo != ~0ULL) {
2260       if (ErrorInfo >= Operands.size()) {
2261         return Error(IDLoc, "too few operands for instruction");
2262       }
2263       ErrorLoc = ((AMDGPUOperand &)*Operands[ErrorInfo]).getStartLoc();
2264       if (ErrorLoc == SMLoc())
2265         ErrorLoc = IDLoc;
2266     }
2267     return Error(ErrorLoc, "invalid operand for instruction");
2268   }
2269 
2270   case Match_PreferE32:
2271     return Error(IDLoc, "internal error: instruction without _e64 suffix "
2272                         "should be encoded as e32");
2273   }
2274   llvm_unreachable("Implement any new match types added!");
2275 }
2276 
2277 bool AMDGPUAsmParser::ParseAsAbsoluteExpression(uint32_t &Ret) {
2278   int64_t Tmp = -1;
2279   if (getLexer().isNot(AsmToken::Integer) && getLexer().isNot(AsmToken::Identifier)) {
2280     return true;
2281   }
2282   if (getParser().parseAbsoluteExpression(Tmp)) {
2283     return true;
2284   }
2285   Ret = static_cast<uint32_t>(Tmp);
2286   return false;
2287 }
2288 
2289 bool AMDGPUAsmParser::ParseDirectiveMajorMinor(uint32_t &Major,
2290                                                uint32_t &Minor) {
2291   if (ParseAsAbsoluteExpression(Major))
2292     return TokError("invalid major version");
2293 
2294   if (getLexer().isNot(AsmToken::Comma))
2295     return TokError("minor version number required, comma expected");
2296   Lex();
2297 
2298   if (ParseAsAbsoluteExpression(Minor))
2299     return TokError("invalid minor version");
2300 
2301   return false;
2302 }
2303 
2304 bool AMDGPUAsmParser::ParseDirectiveHSACodeObjectVersion() {
2305   uint32_t Major;
2306   uint32_t Minor;
2307 
2308   if (ParseDirectiveMajorMinor(Major, Minor))
2309     return true;
2310 
2311   getTargetStreamer().EmitDirectiveHSACodeObjectVersion(Major, Minor);
2312   return false;
2313 }
2314 
2315 bool AMDGPUAsmParser::ParseDirectiveHSACodeObjectISA() {
2316   uint32_t Major;
2317   uint32_t Minor;
2318   uint32_t Stepping;
2319   StringRef VendorName;
2320   StringRef ArchName;
2321 
2322   // If this directive has no arguments, then use the ISA version for the
2323   // targeted GPU.
2324   if (getLexer().is(AsmToken::EndOfStatement)) {
2325     AMDGPU::IsaInfo::IsaVersion ISA =
2326         AMDGPU::IsaInfo::getIsaVersion(getFeatureBits());
2327     getTargetStreamer().EmitDirectiveHSACodeObjectISA(ISA.Major, ISA.Minor,
2328                                                       ISA.Stepping,
2329                                                       "AMD", "AMDGPU");
2330     return false;
2331   }
2332 
2333   if (ParseDirectiveMajorMinor(Major, Minor))
2334     return true;
2335 
2336   if (getLexer().isNot(AsmToken::Comma))
2337     return TokError("stepping version number required, comma expected");
2338   Lex();
2339 
2340   if (ParseAsAbsoluteExpression(Stepping))
2341     return TokError("invalid stepping version");
2342 
2343   if (getLexer().isNot(AsmToken::Comma))
2344     return TokError("vendor name required, comma expected");
2345   Lex();
2346 
2347   if (getLexer().isNot(AsmToken::String))
2348     return TokError("invalid vendor name");
2349 
2350   VendorName = getLexer().getTok().getStringContents();
2351   Lex();
2352 
2353   if (getLexer().isNot(AsmToken::Comma))
2354     return TokError("arch name required, comma expected");
2355   Lex();
2356 
2357   if (getLexer().isNot(AsmToken::String))
2358     return TokError("invalid arch name");
2359 
2360   ArchName = getLexer().getTok().getStringContents();
2361   Lex();
2362 
2363   getTargetStreamer().EmitDirectiveHSACodeObjectISA(Major, Minor, Stepping,
2364                                                     VendorName, ArchName);
2365   return false;
2366 }
2367 
2368 bool AMDGPUAsmParser::ParseDirectiveCodeObjectMetadata() {
2369   std::string YamlString;
2370   raw_string_ostream YamlStream(YamlString);
2371 
2372   getLexer().setSkipSpace(false);
2373 
2374   bool FoundEnd = false;
2375   while (!getLexer().is(AsmToken::Eof)) {
2376     while (getLexer().is(AsmToken::Space)) {
2377       YamlStream << getLexer().getTok().getString();
2378       Lex();
2379     }
2380 
2381     if (getLexer().is(AsmToken::Identifier)) {
2382       StringRef ID = getLexer().getTok().getIdentifier();
2383       if (ID == AMDGPU::CodeObject::MetadataAssemblerDirectiveEnd) {
2384         Lex();
2385         FoundEnd = true;
2386         break;
2387       }
2388     }
2389 
2390     YamlStream << Parser.parseStringToEndOfStatement()
2391                << getContext().getAsmInfo()->getSeparatorString();
2392 
2393     Parser.eatToEndOfStatement();
2394   }
2395 
2396   getLexer().setSkipSpace(true);
2397 
2398   if (getLexer().is(AsmToken::Eof) && !FoundEnd) {
2399     return TokError(
2400         "expected directive .end_amdgpu_code_object_metadata not found");
2401   }
2402 
2403   YamlStream.flush();
2404 
2405   if (!getTargetStreamer().EmitCodeObjectMetadata(YamlString))
2406     return Error(getParser().getTok().getLoc(), "invalid code object metadata");
2407 
2408   return false;
2409 }
2410 
2411 bool AMDGPUAsmParser::ParseAMDKernelCodeTValue(StringRef ID,
2412                                                amd_kernel_code_t &Header) {
2413   SmallString<40> ErrStr;
2414   raw_svector_ostream Err(ErrStr);
2415   if (!parseAmdKernelCodeField(ID, getParser(), Header, Err)) {
2416     return TokError(Err.str());
2417   }
2418   Lex();
2419   return false;
2420 }
2421 
2422 bool AMDGPUAsmParser::ParseDirectiveAMDKernelCodeT() {
2423   amd_kernel_code_t Header;
2424   AMDGPU::initDefaultAMDKernelCodeT(Header, getFeatureBits());
2425 
2426   while (true) {
2427     // Lex EndOfStatement.  This is in a while loop, because lexing a comment
2428     // will set the current token to EndOfStatement.
2429     while(getLexer().is(AsmToken::EndOfStatement))
2430       Lex();
2431 
2432     if (getLexer().isNot(AsmToken::Identifier))
2433       return TokError("expected value identifier or .end_amd_kernel_code_t");
2434 
2435     StringRef ID = getLexer().getTok().getIdentifier();
2436     Lex();
2437 
2438     if (ID == ".end_amd_kernel_code_t")
2439       break;
2440 
2441     if (ParseAMDKernelCodeTValue(ID, Header))
2442       return true;
2443   }
2444 
2445   getTargetStreamer().EmitAMDKernelCodeT(Header);
2446 
2447   return false;
2448 }
2449 
2450 bool AMDGPUAsmParser::ParseDirectiveAMDGPUHsaKernel() {
2451   if (getLexer().isNot(AsmToken::Identifier))
2452     return TokError("expected symbol name");
2453 
2454   StringRef KernelName = Parser.getTok().getString();
2455 
2456   getTargetStreamer().EmitAMDGPUSymbolType(KernelName,
2457                                            ELF::STT_AMDGPU_HSA_KERNEL);
2458   Lex();
2459   KernelScope.initialize(getContext());
2460   return false;
2461 }
2462 
2463 bool AMDGPUAsmParser::ParseDirective(AsmToken DirectiveID) {
2464   StringRef IDVal = DirectiveID.getString();
2465 
2466   if (IDVal == ".hsa_code_object_version")
2467     return ParseDirectiveHSACodeObjectVersion();
2468 
2469   if (IDVal == ".hsa_code_object_isa")
2470     return ParseDirectiveHSACodeObjectISA();
2471 
2472   if (IDVal == AMDGPU::CodeObject::MetadataAssemblerDirectiveBegin)
2473     return ParseDirectiveCodeObjectMetadata();
2474 
2475   if (IDVal == ".amd_kernel_code_t")
2476     return ParseDirectiveAMDKernelCodeT();
2477 
2478   if (IDVal == ".amdgpu_hsa_kernel")
2479     return ParseDirectiveAMDGPUHsaKernel();
2480 
2481   return true;
2482 }
2483 
2484 bool AMDGPUAsmParser::subtargetHasRegister(const MCRegisterInfo &MRI,
2485                                            unsigned RegNo) const {
2486   if (isCI())
2487     return true;
2488 
2489   if (isSI()) {
2490     // No flat_scr
2491     switch (RegNo) {
2492     case AMDGPU::FLAT_SCR:
2493     case AMDGPU::FLAT_SCR_LO:
2494     case AMDGPU::FLAT_SCR_HI:
2495       return false;
2496     default:
2497       return true;
2498     }
2499   }
2500 
2501   // VI only has 102 SGPRs, so make sure we aren't trying to use the 2 more that
2502   // SI/CI have.
2503   for (MCRegAliasIterator R(AMDGPU::SGPR102_SGPR103, &MRI, true);
2504        R.isValid(); ++R) {
2505     if (*R == RegNo)
2506       return false;
2507   }
2508 
2509   return true;
2510 }
2511 
2512 OperandMatchResultTy
2513 AMDGPUAsmParser::parseOperand(OperandVector &Operands, StringRef Mnemonic) {
2514   // Try to parse with a custom parser
2515   OperandMatchResultTy ResTy = MatchOperandParserImpl(Operands, Mnemonic);
2516 
2517   // If we successfully parsed the operand or if there as an error parsing,
2518   // we are done.
2519   //
2520   // If we are parsing after we reach EndOfStatement then this means we
2521   // are appending default values to the Operands list.  This is only done
2522   // by custom parser, so we shouldn't continue on to the generic parsing.
2523   if (ResTy == MatchOperand_Success || ResTy == MatchOperand_ParseFail ||
2524       getLexer().is(AsmToken::EndOfStatement))
2525     return ResTy;
2526 
2527   ResTy = parseRegOrImm(Operands);
2528 
2529   if (ResTy == MatchOperand_Success)
2530     return ResTy;
2531 
2532   if (getLexer().getKind() == AsmToken::Identifier) {
2533     // If this identifier is a symbol, we want to create an expression for it.
2534     // It is a little difficult to distinguish between a symbol name, and
2535     // an instruction flag like 'gds'.  In order to do this, we parse
2536     // all tokens as expressions and then treate the symbol name as the token
2537     // string when we want to interpret the operand as a token.
2538     const auto &Tok = Parser.getTok();
2539     SMLoc S = Tok.getLoc();
2540     const MCExpr *Expr = nullptr;
2541     if (!Parser.parseExpression(Expr)) {
2542       Operands.push_back(AMDGPUOperand::CreateExpr(this, Expr, S));
2543       return MatchOperand_Success;
2544     }
2545 
2546     Operands.push_back(AMDGPUOperand::CreateToken(this, Tok.getString(), Tok.getLoc()));
2547     Parser.Lex();
2548     return MatchOperand_Success;
2549   }
2550   return MatchOperand_NoMatch;
2551 }
2552 
2553 StringRef AMDGPUAsmParser::parseMnemonicSuffix(StringRef Name) {
2554   // Clear any forced encodings from the previous instruction.
2555   setForcedEncodingSize(0);
2556   setForcedDPP(false);
2557   setForcedSDWA(false);
2558 
2559   if (Name.endswith("_e64")) {
2560     setForcedEncodingSize(64);
2561     return Name.substr(0, Name.size() - 4);
2562   } else if (Name.endswith("_e32")) {
2563     setForcedEncodingSize(32);
2564     return Name.substr(0, Name.size() - 4);
2565   } else if (Name.endswith("_dpp")) {
2566     setForcedDPP(true);
2567     return Name.substr(0, Name.size() - 4);
2568   } else if (Name.endswith("_sdwa")) {
2569     setForcedSDWA(true);
2570     return Name.substr(0, Name.size() - 5);
2571   }
2572   return Name;
2573 }
2574 
2575 bool AMDGPUAsmParser::ParseInstruction(ParseInstructionInfo &Info,
2576                                        StringRef Name,
2577                                        SMLoc NameLoc, OperandVector &Operands) {
2578   // Add the instruction mnemonic
2579   Name = parseMnemonicSuffix(Name);
2580   Operands.push_back(AMDGPUOperand::CreateToken(this, Name, NameLoc));
2581 
2582   while (!getLexer().is(AsmToken::EndOfStatement)) {
2583     OperandMatchResultTy Res = parseOperand(Operands, Name);
2584 
2585     // Eat the comma or space if there is one.
2586     if (getLexer().is(AsmToken::Comma))
2587       Parser.Lex();
2588 
2589     switch (Res) {
2590       case MatchOperand_Success: break;
2591       case MatchOperand_ParseFail:
2592         Error(getLexer().getLoc(), "failed parsing operand.");
2593         while (!getLexer().is(AsmToken::EndOfStatement)) {
2594           Parser.Lex();
2595         }
2596         return true;
2597       case MatchOperand_NoMatch:
2598         Error(getLexer().getLoc(), "not a valid operand.");
2599         while (!getLexer().is(AsmToken::EndOfStatement)) {
2600           Parser.Lex();
2601         }
2602         return true;
2603     }
2604   }
2605 
2606   return false;
2607 }
2608 
2609 //===----------------------------------------------------------------------===//
2610 // Utility functions
2611 //===----------------------------------------------------------------------===//
2612 
2613 OperandMatchResultTy
2614 AMDGPUAsmParser::parseIntWithPrefix(const char *Prefix, int64_t &Int) {
2615   switch(getLexer().getKind()) {
2616     default: return MatchOperand_NoMatch;
2617     case AsmToken::Identifier: {
2618       StringRef Name = Parser.getTok().getString();
2619       if (!Name.equals(Prefix)) {
2620         return MatchOperand_NoMatch;
2621       }
2622 
2623       Parser.Lex();
2624       if (getLexer().isNot(AsmToken::Colon))
2625         return MatchOperand_ParseFail;
2626 
2627       Parser.Lex();
2628 
2629       bool IsMinus = false;
2630       if (getLexer().getKind() == AsmToken::Minus) {
2631         Parser.Lex();
2632         IsMinus = true;
2633       }
2634 
2635       if (getLexer().isNot(AsmToken::Integer))
2636         return MatchOperand_ParseFail;
2637 
2638       if (getParser().parseAbsoluteExpression(Int))
2639         return MatchOperand_ParseFail;
2640 
2641       if (IsMinus)
2642         Int = -Int;
2643       break;
2644     }
2645   }
2646   return MatchOperand_Success;
2647 }
2648 
2649 OperandMatchResultTy
2650 AMDGPUAsmParser::parseIntWithPrefix(const char *Prefix, OperandVector &Operands,
2651                                     AMDGPUOperand::ImmTy ImmTy,
2652                                     bool (*ConvertResult)(int64_t&)) {
2653   SMLoc S = Parser.getTok().getLoc();
2654   int64_t Value = 0;
2655 
2656   OperandMatchResultTy Res = parseIntWithPrefix(Prefix, Value);
2657   if (Res != MatchOperand_Success)
2658     return Res;
2659 
2660   if (ConvertResult && !ConvertResult(Value)) {
2661     return MatchOperand_ParseFail;
2662   }
2663 
2664   Operands.push_back(AMDGPUOperand::CreateImm(this, Value, S, ImmTy));
2665   return MatchOperand_Success;
2666 }
2667 
2668 OperandMatchResultTy AMDGPUAsmParser::parseOperandArrayWithPrefix(
2669   const char *Prefix,
2670   OperandVector &Operands,
2671   AMDGPUOperand::ImmTy ImmTy,
2672   bool (*ConvertResult)(int64_t&)) {
2673   StringRef Name = Parser.getTok().getString();
2674   if (!Name.equals(Prefix))
2675     return MatchOperand_NoMatch;
2676 
2677   Parser.Lex();
2678   if (getLexer().isNot(AsmToken::Colon))
2679     return MatchOperand_ParseFail;
2680 
2681   Parser.Lex();
2682   if (getLexer().isNot(AsmToken::LBrac))
2683     return MatchOperand_ParseFail;
2684   Parser.Lex();
2685 
2686   unsigned Val = 0;
2687   SMLoc S = Parser.getTok().getLoc();
2688 
2689   // FIXME: How to verify the number of elements matches the number of src
2690   // operands?
2691   for (int I = 0; I < 3; ++I) {
2692     if (I != 0) {
2693       if (getLexer().is(AsmToken::RBrac))
2694         break;
2695 
2696       if (getLexer().isNot(AsmToken::Comma))
2697         return MatchOperand_ParseFail;
2698       Parser.Lex();
2699     }
2700 
2701     if (getLexer().isNot(AsmToken::Integer))
2702       return MatchOperand_ParseFail;
2703 
2704     int64_t Op;
2705     if (getParser().parseAbsoluteExpression(Op))
2706       return MatchOperand_ParseFail;
2707 
2708     if (Op != 0 && Op != 1)
2709       return MatchOperand_ParseFail;
2710     Val |= (Op << I);
2711   }
2712 
2713   Parser.Lex();
2714   Operands.push_back(AMDGPUOperand::CreateImm(this, Val, S, ImmTy));
2715   return MatchOperand_Success;
2716 }
2717 
2718 OperandMatchResultTy
2719 AMDGPUAsmParser::parseNamedBit(const char *Name, OperandVector &Operands,
2720                                AMDGPUOperand::ImmTy ImmTy) {
2721   int64_t Bit = 0;
2722   SMLoc S = Parser.getTok().getLoc();
2723 
2724   // We are at the end of the statement, and this is a default argument, so
2725   // use a default value.
2726   if (getLexer().isNot(AsmToken::EndOfStatement)) {
2727     switch(getLexer().getKind()) {
2728       case AsmToken::Identifier: {
2729         StringRef Tok = Parser.getTok().getString();
2730         if (Tok == Name) {
2731           Bit = 1;
2732           Parser.Lex();
2733         } else if (Tok.startswith("no") && Tok.endswith(Name)) {
2734           Bit = 0;
2735           Parser.Lex();
2736         } else {
2737           return MatchOperand_NoMatch;
2738         }
2739         break;
2740       }
2741       default:
2742         return MatchOperand_NoMatch;
2743     }
2744   }
2745 
2746   Operands.push_back(AMDGPUOperand::CreateImm(this, Bit, S, ImmTy));
2747   return MatchOperand_Success;
2748 }
2749 
2750 static void addOptionalImmOperand(
2751   MCInst& Inst, const OperandVector& Operands,
2752   AMDGPUAsmParser::OptionalImmIndexMap& OptionalIdx,
2753   AMDGPUOperand::ImmTy ImmT,
2754   int64_t Default = 0) {
2755   auto i = OptionalIdx.find(ImmT);
2756   if (i != OptionalIdx.end()) {
2757     unsigned Idx = i->second;
2758     ((AMDGPUOperand &)*Operands[Idx]).addImmOperands(Inst, 1);
2759   } else {
2760     Inst.addOperand(MCOperand::createImm(Default));
2761   }
2762 }
2763 
2764 OperandMatchResultTy
2765 AMDGPUAsmParser::parseStringWithPrefix(StringRef Prefix, StringRef &Value) {
2766   if (getLexer().isNot(AsmToken::Identifier)) {
2767     return MatchOperand_NoMatch;
2768   }
2769   StringRef Tok = Parser.getTok().getString();
2770   if (Tok != Prefix) {
2771     return MatchOperand_NoMatch;
2772   }
2773 
2774   Parser.Lex();
2775   if (getLexer().isNot(AsmToken::Colon)) {
2776     return MatchOperand_ParseFail;
2777   }
2778 
2779   Parser.Lex();
2780   if (getLexer().isNot(AsmToken::Identifier)) {
2781     return MatchOperand_ParseFail;
2782   }
2783 
2784   Value = Parser.getTok().getString();
2785   return MatchOperand_Success;
2786 }
2787 
2788 //===----------------------------------------------------------------------===//
2789 // ds
2790 //===----------------------------------------------------------------------===//
2791 
2792 void AMDGPUAsmParser::cvtDSOffset01(MCInst &Inst,
2793                                     const OperandVector &Operands) {
2794   OptionalImmIndexMap OptionalIdx;
2795 
2796   for (unsigned i = 1, e = Operands.size(); i != e; ++i) {
2797     AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[i]);
2798 
2799     // Add the register arguments
2800     if (Op.isReg()) {
2801       Op.addRegOperands(Inst, 1);
2802       continue;
2803     }
2804 
2805     // Handle optional arguments
2806     OptionalIdx[Op.getImmTy()] = i;
2807   }
2808 
2809   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyOffset0);
2810   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyOffset1);
2811   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyGDS);
2812 
2813   Inst.addOperand(MCOperand::createReg(AMDGPU::M0)); // m0
2814 }
2815 
2816 void AMDGPUAsmParser::cvtDSImpl(MCInst &Inst, const OperandVector &Operands,
2817                                 bool IsGdsHardcoded) {
2818   OptionalImmIndexMap OptionalIdx;
2819 
2820   for (unsigned i = 1, e = Operands.size(); i != e; ++i) {
2821     AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[i]);
2822 
2823     // Add the register arguments
2824     if (Op.isReg()) {
2825       Op.addRegOperands(Inst, 1);
2826       continue;
2827     }
2828 
2829     if (Op.isToken() && Op.getToken() == "gds") {
2830       IsGdsHardcoded = true;
2831       continue;
2832     }
2833 
2834     // Handle optional arguments
2835     OptionalIdx[Op.getImmTy()] = i;
2836   }
2837 
2838   AMDGPUOperand::ImmTy OffsetType =
2839     (Inst.getOpcode() == AMDGPU::DS_SWIZZLE_B32_si ||
2840      Inst.getOpcode() == AMDGPU::DS_SWIZZLE_B32_vi) ? AMDGPUOperand::ImmTySwizzle :
2841                                                       AMDGPUOperand::ImmTyOffset;
2842 
2843   addOptionalImmOperand(Inst, Operands, OptionalIdx, OffsetType);
2844 
2845   if (!IsGdsHardcoded) {
2846     addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyGDS);
2847   }
2848   Inst.addOperand(MCOperand::createReg(AMDGPU::M0)); // m0
2849 }
2850 
2851 void AMDGPUAsmParser::cvtExp(MCInst &Inst, const OperandVector &Operands) {
2852   OptionalImmIndexMap OptionalIdx;
2853 
2854   unsigned OperandIdx[4];
2855   unsigned EnMask = 0;
2856   int SrcIdx = 0;
2857 
2858   for (unsigned i = 1, e = Operands.size(); i != e; ++i) {
2859     AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[i]);
2860 
2861     // Add the register arguments
2862     if (Op.isReg()) {
2863       assert(SrcIdx < 4);
2864       OperandIdx[SrcIdx] = Inst.size();
2865       Op.addRegOperands(Inst, 1);
2866       ++SrcIdx;
2867       continue;
2868     }
2869 
2870     if (Op.isOff()) {
2871       assert(SrcIdx < 4);
2872       OperandIdx[SrcIdx] = Inst.size();
2873       Inst.addOperand(MCOperand::createReg(AMDGPU::NoRegister));
2874       ++SrcIdx;
2875       continue;
2876     }
2877 
2878     if (Op.isImm() && Op.getImmTy() == AMDGPUOperand::ImmTyExpTgt) {
2879       Op.addImmOperands(Inst, 1);
2880       continue;
2881     }
2882 
2883     if (Op.isToken() && Op.getToken() == "done")
2884       continue;
2885 
2886     // Handle optional arguments
2887     OptionalIdx[Op.getImmTy()] = i;
2888   }
2889 
2890   assert(SrcIdx == 4);
2891 
2892   bool Compr = false;
2893   if (OptionalIdx.find(AMDGPUOperand::ImmTyExpCompr) != OptionalIdx.end()) {
2894     Compr = true;
2895     Inst.getOperand(OperandIdx[1]) = Inst.getOperand(OperandIdx[2]);
2896     Inst.getOperand(OperandIdx[2]).setReg(AMDGPU::NoRegister);
2897     Inst.getOperand(OperandIdx[3]).setReg(AMDGPU::NoRegister);
2898   }
2899 
2900   for (auto i = 0; i < SrcIdx; ++i) {
2901     if (Inst.getOperand(OperandIdx[i]).getReg() != AMDGPU::NoRegister) {
2902       EnMask |= Compr? (0x3 << i * 2) : (0x1 << i);
2903     }
2904   }
2905 
2906   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyExpVM);
2907   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyExpCompr);
2908 
2909   Inst.addOperand(MCOperand::createImm(EnMask));
2910 }
2911 
2912 //===----------------------------------------------------------------------===//
2913 // s_waitcnt
2914 //===----------------------------------------------------------------------===//
2915 
2916 static bool
2917 encodeCnt(
2918   const AMDGPU::IsaInfo::IsaVersion ISA,
2919   int64_t &IntVal,
2920   int64_t CntVal,
2921   bool Saturate,
2922   unsigned (*encode)(const IsaInfo::IsaVersion &Version, unsigned, unsigned),
2923   unsigned (*decode)(const IsaInfo::IsaVersion &Version, unsigned))
2924 {
2925   bool Failed = false;
2926 
2927   IntVal = encode(ISA, IntVal, CntVal);
2928   if (CntVal != decode(ISA, IntVal)) {
2929     if (Saturate) {
2930       IntVal = encode(ISA, IntVal, -1);
2931     } else {
2932       Failed = true;
2933     }
2934   }
2935   return Failed;
2936 }
2937 
2938 bool AMDGPUAsmParser::parseCnt(int64_t &IntVal) {
2939   StringRef CntName = Parser.getTok().getString();
2940   int64_t CntVal;
2941 
2942   Parser.Lex();
2943   if (getLexer().isNot(AsmToken::LParen))
2944     return true;
2945 
2946   Parser.Lex();
2947   if (getLexer().isNot(AsmToken::Integer))
2948     return true;
2949 
2950   SMLoc ValLoc = Parser.getTok().getLoc();
2951   if (getParser().parseAbsoluteExpression(CntVal))
2952     return true;
2953 
2954   AMDGPU::IsaInfo::IsaVersion ISA =
2955       AMDGPU::IsaInfo::getIsaVersion(getFeatureBits());
2956 
2957   bool Failed = true;
2958   bool Sat = CntName.endswith("_sat");
2959 
2960   if (CntName == "vmcnt" || CntName == "vmcnt_sat") {
2961     Failed = encodeCnt(ISA, IntVal, CntVal, Sat, encodeVmcnt, decodeVmcnt);
2962   } else if (CntName == "expcnt" || CntName == "expcnt_sat") {
2963     Failed = encodeCnt(ISA, IntVal, CntVal, Sat, encodeExpcnt, decodeExpcnt);
2964   } else if (CntName == "lgkmcnt" || CntName == "lgkmcnt_sat") {
2965     Failed = encodeCnt(ISA, IntVal, CntVal, Sat, encodeLgkmcnt, decodeLgkmcnt);
2966   }
2967 
2968   if (Failed) {
2969     Error(ValLoc, "too large value for " + CntName);
2970     return true;
2971   }
2972 
2973   if (getLexer().isNot(AsmToken::RParen)) {
2974     return true;
2975   }
2976 
2977   Parser.Lex();
2978   if (getLexer().is(AsmToken::Amp) || getLexer().is(AsmToken::Comma)) {
2979     const AsmToken NextToken = getLexer().peekTok();
2980     if (NextToken.is(AsmToken::Identifier)) {
2981       Parser.Lex();
2982     }
2983   }
2984 
2985   return false;
2986 }
2987 
2988 OperandMatchResultTy
2989 AMDGPUAsmParser::parseSWaitCntOps(OperandVector &Operands) {
2990   AMDGPU::IsaInfo::IsaVersion ISA =
2991       AMDGPU::IsaInfo::getIsaVersion(getFeatureBits());
2992   int64_t Waitcnt = getWaitcntBitMask(ISA);
2993   SMLoc S = Parser.getTok().getLoc();
2994 
2995   switch(getLexer().getKind()) {
2996     default: return MatchOperand_ParseFail;
2997     case AsmToken::Integer:
2998       // The operand can be an integer value.
2999       if (getParser().parseAbsoluteExpression(Waitcnt))
3000         return MatchOperand_ParseFail;
3001       break;
3002 
3003     case AsmToken::Identifier:
3004       do {
3005         if (parseCnt(Waitcnt))
3006           return MatchOperand_ParseFail;
3007       } while(getLexer().isNot(AsmToken::EndOfStatement));
3008       break;
3009   }
3010   Operands.push_back(AMDGPUOperand::CreateImm(this, Waitcnt, S));
3011   return MatchOperand_Success;
3012 }
3013 
3014 bool AMDGPUAsmParser::parseHwregConstruct(OperandInfoTy &HwReg, int64_t &Offset,
3015                                           int64_t &Width) {
3016   using namespace llvm::AMDGPU::Hwreg;
3017 
3018   if (Parser.getTok().getString() != "hwreg")
3019     return true;
3020   Parser.Lex();
3021 
3022   if (getLexer().isNot(AsmToken::LParen))
3023     return true;
3024   Parser.Lex();
3025 
3026   if (getLexer().is(AsmToken::Identifier)) {
3027     HwReg.IsSymbolic = true;
3028     HwReg.Id = ID_UNKNOWN_;
3029     const StringRef tok = Parser.getTok().getString();
3030     for (int i = ID_SYMBOLIC_FIRST_; i < ID_SYMBOLIC_LAST_; ++i) {
3031       if (tok == IdSymbolic[i]) {
3032         HwReg.Id = i;
3033         break;
3034       }
3035     }
3036     Parser.Lex();
3037   } else {
3038     HwReg.IsSymbolic = false;
3039     if (getLexer().isNot(AsmToken::Integer))
3040       return true;
3041     if (getParser().parseAbsoluteExpression(HwReg.Id))
3042       return true;
3043   }
3044 
3045   if (getLexer().is(AsmToken::RParen)) {
3046     Parser.Lex();
3047     return false;
3048   }
3049 
3050   // optional params
3051   if (getLexer().isNot(AsmToken::Comma))
3052     return true;
3053   Parser.Lex();
3054 
3055   if (getLexer().isNot(AsmToken::Integer))
3056     return true;
3057   if (getParser().parseAbsoluteExpression(Offset))
3058     return true;
3059 
3060   if (getLexer().isNot(AsmToken::Comma))
3061     return true;
3062   Parser.Lex();
3063 
3064   if (getLexer().isNot(AsmToken::Integer))
3065     return true;
3066   if (getParser().parseAbsoluteExpression(Width))
3067     return true;
3068 
3069   if (getLexer().isNot(AsmToken::RParen))
3070     return true;
3071   Parser.Lex();
3072 
3073   return false;
3074 }
3075 
3076 OperandMatchResultTy AMDGPUAsmParser::parseHwreg(OperandVector &Operands) {
3077   using namespace llvm::AMDGPU::Hwreg;
3078 
3079   int64_t Imm16Val = 0;
3080   SMLoc S = Parser.getTok().getLoc();
3081 
3082   switch(getLexer().getKind()) {
3083     default: return MatchOperand_NoMatch;
3084     case AsmToken::Integer:
3085       // The operand can be an integer value.
3086       if (getParser().parseAbsoluteExpression(Imm16Val))
3087         return MatchOperand_NoMatch;
3088       if (Imm16Val < 0 || !isUInt<16>(Imm16Val)) {
3089         Error(S, "invalid immediate: only 16-bit values are legal");
3090         // Do not return error code, but create an imm operand anyway and proceed
3091         // to the next operand, if any. That avoids unneccessary error messages.
3092       }
3093       break;
3094 
3095     case AsmToken::Identifier: {
3096         OperandInfoTy HwReg(ID_UNKNOWN_);
3097         int64_t Offset = OFFSET_DEFAULT_;
3098         int64_t Width = WIDTH_M1_DEFAULT_ + 1;
3099         if (parseHwregConstruct(HwReg, Offset, Width))
3100           return MatchOperand_ParseFail;
3101         if (HwReg.Id < 0 || !isUInt<ID_WIDTH_>(HwReg.Id)) {
3102           if (HwReg.IsSymbolic)
3103             Error(S, "invalid symbolic name of hardware register");
3104           else
3105             Error(S, "invalid code of hardware register: only 6-bit values are legal");
3106         }
3107         if (Offset < 0 || !isUInt<OFFSET_WIDTH_>(Offset))
3108           Error(S, "invalid bit offset: only 5-bit values are legal");
3109         if ((Width-1) < 0 || !isUInt<WIDTH_M1_WIDTH_>(Width-1))
3110           Error(S, "invalid bitfield width: only values from 1 to 32 are legal");
3111         Imm16Val = (HwReg.Id << ID_SHIFT_) | (Offset << OFFSET_SHIFT_) | ((Width-1) << WIDTH_M1_SHIFT_);
3112       }
3113       break;
3114   }
3115   Operands.push_back(AMDGPUOperand::CreateImm(this, Imm16Val, S, AMDGPUOperand::ImmTyHwreg));
3116   return MatchOperand_Success;
3117 }
3118 
3119 bool AMDGPUOperand::isSWaitCnt() const {
3120   return isImm();
3121 }
3122 
3123 bool AMDGPUOperand::isHwreg() const {
3124   return isImmTy(ImmTyHwreg);
3125 }
3126 
3127 bool AMDGPUAsmParser::parseSendMsgConstruct(OperandInfoTy &Msg, OperandInfoTy &Operation, int64_t &StreamId) {
3128   using namespace llvm::AMDGPU::SendMsg;
3129 
3130   if (Parser.getTok().getString() != "sendmsg")
3131     return true;
3132   Parser.Lex();
3133 
3134   if (getLexer().isNot(AsmToken::LParen))
3135     return true;
3136   Parser.Lex();
3137 
3138   if (getLexer().is(AsmToken::Identifier)) {
3139     Msg.IsSymbolic = true;
3140     Msg.Id = ID_UNKNOWN_;
3141     const std::string tok = Parser.getTok().getString();
3142     for (int i = ID_GAPS_FIRST_; i < ID_GAPS_LAST_; ++i) {
3143       switch(i) {
3144         default: continue; // Omit gaps.
3145         case ID_INTERRUPT: case ID_GS: case ID_GS_DONE:  case ID_SYSMSG: break;
3146       }
3147       if (tok == IdSymbolic[i]) {
3148         Msg.Id = i;
3149         break;
3150       }
3151     }
3152     Parser.Lex();
3153   } else {
3154     Msg.IsSymbolic = false;
3155     if (getLexer().isNot(AsmToken::Integer))
3156       return true;
3157     if (getParser().parseAbsoluteExpression(Msg.Id))
3158       return true;
3159     if (getLexer().is(AsmToken::Integer))
3160       if (getParser().parseAbsoluteExpression(Msg.Id))
3161         Msg.Id = ID_UNKNOWN_;
3162   }
3163   if (Msg.Id == ID_UNKNOWN_) // Don't know how to parse the rest.
3164     return false;
3165 
3166   if (!(Msg.Id == ID_GS || Msg.Id == ID_GS_DONE || Msg.Id == ID_SYSMSG)) {
3167     if (getLexer().isNot(AsmToken::RParen))
3168       return true;
3169     Parser.Lex();
3170     return false;
3171   }
3172 
3173   if (getLexer().isNot(AsmToken::Comma))
3174     return true;
3175   Parser.Lex();
3176 
3177   assert(Msg.Id == ID_GS || Msg.Id == ID_GS_DONE || Msg.Id == ID_SYSMSG);
3178   Operation.Id = ID_UNKNOWN_;
3179   if (getLexer().is(AsmToken::Identifier)) {
3180     Operation.IsSymbolic = true;
3181     const char* const *S = (Msg.Id == ID_SYSMSG) ? OpSysSymbolic : OpGsSymbolic;
3182     const int F = (Msg.Id == ID_SYSMSG) ? OP_SYS_FIRST_ : OP_GS_FIRST_;
3183     const int L = (Msg.Id == ID_SYSMSG) ? OP_SYS_LAST_ : OP_GS_LAST_;
3184     const StringRef Tok = Parser.getTok().getString();
3185     for (int i = F; i < L; ++i) {
3186       if (Tok == S[i]) {
3187         Operation.Id = i;
3188         break;
3189       }
3190     }
3191     Parser.Lex();
3192   } else {
3193     Operation.IsSymbolic = false;
3194     if (getLexer().isNot(AsmToken::Integer))
3195       return true;
3196     if (getParser().parseAbsoluteExpression(Operation.Id))
3197       return true;
3198   }
3199 
3200   if ((Msg.Id == ID_GS || Msg.Id == ID_GS_DONE) && Operation.Id != OP_GS_NOP) {
3201     // Stream id is optional.
3202     if (getLexer().is(AsmToken::RParen)) {
3203       Parser.Lex();
3204       return false;
3205     }
3206 
3207     if (getLexer().isNot(AsmToken::Comma))
3208       return true;
3209     Parser.Lex();
3210 
3211     if (getLexer().isNot(AsmToken::Integer))
3212       return true;
3213     if (getParser().parseAbsoluteExpression(StreamId))
3214       return true;
3215   }
3216 
3217   if (getLexer().isNot(AsmToken::RParen))
3218     return true;
3219   Parser.Lex();
3220   return false;
3221 }
3222 
3223 OperandMatchResultTy AMDGPUAsmParser::parseInterpSlot(OperandVector &Operands) {
3224   if (getLexer().getKind() != AsmToken::Identifier)
3225     return MatchOperand_NoMatch;
3226 
3227   StringRef Str = Parser.getTok().getString();
3228   int Slot = StringSwitch<int>(Str)
3229     .Case("p10", 0)
3230     .Case("p20", 1)
3231     .Case("p0", 2)
3232     .Default(-1);
3233 
3234   SMLoc S = Parser.getTok().getLoc();
3235   if (Slot == -1)
3236     return MatchOperand_ParseFail;
3237 
3238   Parser.Lex();
3239   Operands.push_back(AMDGPUOperand::CreateImm(this, Slot, S,
3240                                               AMDGPUOperand::ImmTyInterpSlot));
3241   return MatchOperand_Success;
3242 }
3243 
3244 OperandMatchResultTy AMDGPUAsmParser::parseInterpAttr(OperandVector &Operands) {
3245   if (getLexer().getKind() != AsmToken::Identifier)
3246     return MatchOperand_NoMatch;
3247 
3248   StringRef Str = Parser.getTok().getString();
3249   if (!Str.startswith("attr"))
3250     return MatchOperand_NoMatch;
3251 
3252   StringRef Chan = Str.take_back(2);
3253   int AttrChan = StringSwitch<int>(Chan)
3254     .Case(".x", 0)
3255     .Case(".y", 1)
3256     .Case(".z", 2)
3257     .Case(".w", 3)
3258     .Default(-1);
3259   if (AttrChan == -1)
3260     return MatchOperand_ParseFail;
3261 
3262   Str = Str.drop_back(2).drop_front(4);
3263 
3264   uint8_t Attr;
3265   if (Str.getAsInteger(10, Attr))
3266     return MatchOperand_ParseFail;
3267 
3268   SMLoc S = Parser.getTok().getLoc();
3269   Parser.Lex();
3270   if (Attr > 63) {
3271     Error(S, "out of bounds attr");
3272     return MatchOperand_Success;
3273   }
3274 
3275   SMLoc SChan = SMLoc::getFromPointer(Chan.data());
3276 
3277   Operands.push_back(AMDGPUOperand::CreateImm(this, Attr, S,
3278                                               AMDGPUOperand::ImmTyInterpAttr));
3279   Operands.push_back(AMDGPUOperand::CreateImm(this, AttrChan, SChan,
3280                                               AMDGPUOperand::ImmTyAttrChan));
3281   return MatchOperand_Success;
3282 }
3283 
3284 void AMDGPUAsmParser::errorExpTgt() {
3285   Error(Parser.getTok().getLoc(), "invalid exp target");
3286 }
3287 
3288 OperandMatchResultTy AMDGPUAsmParser::parseExpTgtImpl(StringRef Str,
3289                                                       uint8_t &Val) {
3290   if (Str == "null") {
3291     Val = 9;
3292     return MatchOperand_Success;
3293   }
3294 
3295   if (Str.startswith("mrt")) {
3296     Str = Str.drop_front(3);
3297     if (Str == "z") { // == mrtz
3298       Val = 8;
3299       return MatchOperand_Success;
3300     }
3301 
3302     if (Str.getAsInteger(10, Val))
3303       return MatchOperand_ParseFail;
3304 
3305     if (Val > 7)
3306       errorExpTgt();
3307 
3308     return MatchOperand_Success;
3309   }
3310 
3311   if (Str.startswith("pos")) {
3312     Str = Str.drop_front(3);
3313     if (Str.getAsInteger(10, Val))
3314       return MatchOperand_ParseFail;
3315 
3316     if (Val > 3)
3317       errorExpTgt();
3318 
3319     Val += 12;
3320     return MatchOperand_Success;
3321   }
3322 
3323   if (Str.startswith("param")) {
3324     Str = Str.drop_front(5);
3325     if (Str.getAsInteger(10, Val))
3326       return MatchOperand_ParseFail;
3327 
3328     if (Val >= 32)
3329       errorExpTgt();
3330 
3331     Val += 32;
3332     return MatchOperand_Success;
3333   }
3334 
3335   if (Str.startswith("invalid_target_")) {
3336     Str = Str.drop_front(15);
3337     if (Str.getAsInteger(10, Val))
3338       return MatchOperand_ParseFail;
3339 
3340     errorExpTgt();
3341     return MatchOperand_Success;
3342   }
3343 
3344   return MatchOperand_NoMatch;
3345 }
3346 
3347 OperandMatchResultTy AMDGPUAsmParser::parseExpTgt(OperandVector &Operands) {
3348   uint8_t Val;
3349   StringRef Str = Parser.getTok().getString();
3350 
3351   auto Res = parseExpTgtImpl(Str, Val);
3352   if (Res != MatchOperand_Success)
3353     return Res;
3354 
3355   SMLoc S = Parser.getTok().getLoc();
3356   Parser.Lex();
3357 
3358   Operands.push_back(AMDGPUOperand::CreateImm(this, Val, S,
3359                                               AMDGPUOperand::ImmTyExpTgt));
3360   return MatchOperand_Success;
3361 }
3362 
3363 OperandMatchResultTy
3364 AMDGPUAsmParser::parseSendMsgOp(OperandVector &Operands) {
3365   using namespace llvm::AMDGPU::SendMsg;
3366 
3367   int64_t Imm16Val = 0;
3368   SMLoc S = Parser.getTok().getLoc();
3369 
3370   switch(getLexer().getKind()) {
3371   default:
3372     return MatchOperand_NoMatch;
3373   case AsmToken::Integer:
3374     // The operand can be an integer value.
3375     if (getParser().parseAbsoluteExpression(Imm16Val))
3376       return MatchOperand_NoMatch;
3377     if (Imm16Val < 0 || !isUInt<16>(Imm16Val)) {
3378       Error(S, "invalid immediate: only 16-bit values are legal");
3379       // Do not return error code, but create an imm operand anyway and proceed
3380       // to the next operand, if any. That avoids unneccessary error messages.
3381     }
3382     break;
3383   case AsmToken::Identifier: {
3384       OperandInfoTy Msg(ID_UNKNOWN_);
3385       OperandInfoTy Operation(OP_UNKNOWN_);
3386       int64_t StreamId = STREAM_ID_DEFAULT_;
3387       if (parseSendMsgConstruct(Msg, Operation, StreamId))
3388         return MatchOperand_ParseFail;
3389       do {
3390         // Validate and encode message ID.
3391         if (! ((ID_INTERRUPT <= Msg.Id && Msg.Id <= ID_GS_DONE)
3392                 || Msg.Id == ID_SYSMSG)) {
3393           if (Msg.IsSymbolic)
3394             Error(S, "invalid/unsupported symbolic name of message");
3395           else
3396             Error(S, "invalid/unsupported code of message");
3397           break;
3398         }
3399         Imm16Val = (Msg.Id << ID_SHIFT_);
3400         // Validate and encode operation ID.
3401         if (Msg.Id == ID_GS || Msg.Id == ID_GS_DONE) {
3402           if (! (OP_GS_FIRST_ <= Operation.Id && Operation.Id < OP_GS_LAST_)) {
3403             if (Operation.IsSymbolic)
3404               Error(S, "invalid symbolic name of GS_OP");
3405             else
3406               Error(S, "invalid code of GS_OP: only 2-bit values are legal");
3407             break;
3408           }
3409           if (Operation.Id == OP_GS_NOP
3410               && Msg.Id != ID_GS_DONE) {
3411             Error(S, "invalid GS_OP: NOP is for GS_DONE only");
3412             break;
3413           }
3414           Imm16Val |= (Operation.Id << OP_SHIFT_);
3415         }
3416         if (Msg.Id == ID_SYSMSG) {
3417           if (! (OP_SYS_FIRST_ <= Operation.Id && Operation.Id < OP_SYS_LAST_)) {
3418             if (Operation.IsSymbolic)
3419               Error(S, "invalid/unsupported symbolic name of SYSMSG_OP");
3420             else
3421               Error(S, "invalid/unsupported code of SYSMSG_OP");
3422             break;
3423           }
3424           Imm16Val |= (Operation.Id << OP_SHIFT_);
3425         }
3426         // Validate and encode stream ID.
3427         if ((Msg.Id == ID_GS || Msg.Id == ID_GS_DONE) && Operation.Id != OP_GS_NOP) {
3428           if (! (STREAM_ID_FIRST_ <= StreamId && StreamId < STREAM_ID_LAST_)) {
3429             Error(S, "invalid stream id: only 2-bit values are legal");
3430             break;
3431           }
3432           Imm16Val |= (StreamId << STREAM_ID_SHIFT_);
3433         }
3434       } while (false);
3435     }
3436     break;
3437   }
3438   Operands.push_back(AMDGPUOperand::CreateImm(this, Imm16Val, S, AMDGPUOperand::ImmTySendMsg));
3439   return MatchOperand_Success;
3440 }
3441 
3442 bool AMDGPUOperand::isSendMsg() const {
3443   return isImmTy(ImmTySendMsg);
3444 }
3445 
3446 //===----------------------------------------------------------------------===//
3447 // parser helpers
3448 //===----------------------------------------------------------------------===//
3449 
3450 bool
3451 AMDGPUAsmParser::trySkipId(const StringRef Id) {
3452   if (getLexer().getKind() == AsmToken::Identifier &&
3453       Parser.getTok().getString() == Id) {
3454     Parser.Lex();
3455     return true;
3456   }
3457   return false;
3458 }
3459 
3460 bool
3461 AMDGPUAsmParser::trySkipToken(const AsmToken::TokenKind Kind) {
3462   if (getLexer().getKind() == Kind) {
3463     Parser.Lex();
3464     return true;
3465   }
3466   return false;
3467 }
3468 
3469 bool
3470 AMDGPUAsmParser::skipToken(const AsmToken::TokenKind Kind,
3471                            const StringRef ErrMsg) {
3472   if (!trySkipToken(Kind)) {
3473     Error(Parser.getTok().getLoc(), ErrMsg);
3474     return false;
3475   }
3476   return true;
3477 }
3478 
3479 bool
3480 AMDGPUAsmParser::parseExpr(int64_t &Imm) {
3481   return !getParser().parseAbsoluteExpression(Imm);
3482 }
3483 
3484 bool
3485 AMDGPUAsmParser::parseString(StringRef &Val, const StringRef ErrMsg) {
3486   SMLoc S = Parser.getTok().getLoc();
3487   if (getLexer().getKind() == AsmToken::String) {
3488     Val = Parser.getTok().getStringContents();
3489     Parser.Lex();
3490     return true;
3491   } else {
3492     Error(S, ErrMsg);
3493     return false;
3494   }
3495 }
3496 
3497 //===----------------------------------------------------------------------===//
3498 // swizzle
3499 //===----------------------------------------------------------------------===//
3500 
3501 LLVM_READNONE
3502 static unsigned
3503 encodeBitmaskPerm(const unsigned AndMask,
3504                   const unsigned OrMask,
3505                   const unsigned XorMask) {
3506   using namespace llvm::AMDGPU::Swizzle;
3507 
3508   return BITMASK_PERM_ENC |
3509          (AndMask << BITMASK_AND_SHIFT) |
3510          (OrMask  << BITMASK_OR_SHIFT)  |
3511          (XorMask << BITMASK_XOR_SHIFT);
3512 }
3513 
3514 bool
3515 AMDGPUAsmParser::parseSwizzleOperands(const unsigned OpNum, int64_t* Op,
3516                                       const unsigned MinVal,
3517                                       const unsigned MaxVal,
3518                                       const StringRef ErrMsg) {
3519   for (unsigned i = 0; i < OpNum; ++i) {
3520     if (!skipToken(AsmToken::Comma, "expected a comma")){
3521       return false;
3522     }
3523     SMLoc ExprLoc = Parser.getTok().getLoc();
3524     if (!parseExpr(Op[i])) {
3525       return false;
3526     }
3527     if (Op[i] < MinVal || Op[i] > MaxVal) {
3528       Error(ExprLoc, ErrMsg);
3529       return false;
3530     }
3531   }
3532 
3533   return true;
3534 }
3535 
3536 bool
3537 AMDGPUAsmParser::parseSwizzleQuadPerm(int64_t &Imm) {
3538   using namespace llvm::AMDGPU::Swizzle;
3539 
3540   int64_t Lane[LANE_NUM];
3541   if (parseSwizzleOperands(LANE_NUM, Lane, 0, LANE_MAX,
3542                            "expected a 2-bit lane id")) {
3543     Imm = QUAD_PERM_ENC;
3544     for (auto i = 0; i < LANE_NUM; ++i) {
3545       Imm |= Lane[i] << (LANE_SHIFT * i);
3546     }
3547     return true;
3548   }
3549   return false;
3550 }
3551 
3552 bool
3553 AMDGPUAsmParser::parseSwizzleBroadcast(int64_t &Imm) {
3554   using namespace llvm::AMDGPU::Swizzle;
3555 
3556   SMLoc S = Parser.getTok().getLoc();
3557   int64_t GroupSize;
3558   int64_t LaneIdx;
3559 
3560   if (!parseSwizzleOperands(1, &GroupSize,
3561                             2, 32,
3562                             "group size must be in the interval [2,32]")) {
3563     return false;
3564   }
3565   if (!isPowerOf2_64(GroupSize)) {
3566     Error(S, "group size must be a power of two");
3567     return false;
3568   }
3569   if (parseSwizzleOperands(1, &LaneIdx,
3570                            0, GroupSize - 1,
3571                            "lane id must be in the interval [0,group size - 1]")) {
3572     Imm = encodeBitmaskPerm(BITMASK_MAX - GroupSize + 1, LaneIdx, 0);
3573     return true;
3574   }
3575   return false;
3576 }
3577 
3578 bool
3579 AMDGPUAsmParser::parseSwizzleReverse(int64_t &Imm) {
3580   using namespace llvm::AMDGPU::Swizzle;
3581 
3582   SMLoc S = Parser.getTok().getLoc();
3583   int64_t GroupSize;
3584 
3585   if (!parseSwizzleOperands(1, &GroupSize,
3586       2, 32, "group size must be in the interval [2,32]")) {
3587     return false;
3588   }
3589   if (!isPowerOf2_64(GroupSize)) {
3590     Error(S, "group size must be a power of two");
3591     return false;
3592   }
3593 
3594   Imm = encodeBitmaskPerm(BITMASK_MAX, 0, GroupSize - 1);
3595   return true;
3596 }
3597 
3598 bool
3599 AMDGPUAsmParser::parseSwizzleSwap(int64_t &Imm) {
3600   using namespace llvm::AMDGPU::Swizzle;
3601 
3602   SMLoc S = Parser.getTok().getLoc();
3603   int64_t GroupSize;
3604 
3605   if (!parseSwizzleOperands(1, &GroupSize,
3606       1, 16, "group size must be in the interval [1,16]")) {
3607     return false;
3608   }
3609   if (!isPowerOf2_64(GroupSize)) {
3610     Error(S, "group size must be a power of two");
3611     return false;
3612   }
3613 
3614   Imm = encodeBitmaskPerm(BITMASK_MAX, 0, GroupSize);
3615   return true;
3616 }
3617 
3618 bool
3619 AMDGPUAsmParser::parseSwizzleBitmaskPerm(int64_t &Imm) {
3620   using namespace llvm::AMDGPU::Swizzle;
3621 
3622   if (!skipToken(AsmToken::Comma, "expected a comma")) {
3623     return false;
3624   }
3625 
3626   StringRef Ctl;
3627   SMLoc StrLoc = Parser.getTok().getLoc();
3628   if (!parseString(Ctl)) {
3629     return false;
3630   }
3631   if (Ctl.size() != BITMASK_WIDTH) {
3632     Error(StrLoc, "expected a 5-character mask");
3633     return false;
3634   }
3635 
3636   unsigned AndMask = 0;
3637   unsigned OrMask = 0;
3638   unsigned XorMask = 0;
3639 
3640   for (size_t i = 0; i < Ctl.size(); ++i) {
3641     unsigned Mask = 1 << (BITMASK_WIDTH - 1 - i);
3642     switch(Ctl[i]) {
3643     default:
3644       Error(StrLoc, "invalid mask");
3645       return false;
3646     case '0':
3647       break;
3648     case '1':
3649       OrMask |= Mask;
3650       break;
3651     case 'p':
3652       AndMask |= Mask;
3653       break;
3654     case 'i':
3655       AndMask |= Mask;
3656       XorMask |= Mask;
3657       break;
3658     }
3659   }
3660 
3661   Imm = encodeBitmaskPerm(AndMask, OrMask, XorMask);
3662   return true;
3663 }
3664 
3665 bool
3666 AMDGPUAsmParser::parseSwizzleOffset(int64_t &Imm) {
3667 
3668   SMLoc OffsetLoc = Parser.getTok().getLoc();
3669 
3670   if (!parseExpr(Imm)) {
3671     return false;
3672   }
3673   if (!isUInt<16>(Imm)) {
3674     Error(OffsetLoc, "expected a 16-bit offset");
3675     return false;
3676   }
3677   return true;
3678 }
3679 
3680 bool
3681 AMDGPUAsmParser::parseSwizzleMacro(int64_t &Imm) {
3682   using namespace llvm::AMDGPU::Swizzle;
3683 
3684   if (skipToken(AsmToken::LParen, "expected a left parentheses")) {
3685 
3686     SMLoc ModeLoc = Parser.getTok().getLoc();
3687     bool Ok = false;
3688 
3689     if (trySkipId(IdSymbolic[ID_QUAD_PERM])) {
3690       Ok = parseSwizzleQuadPerm(Imm);
3691     } else if (trySkipId(IdSymbolic[ID_BITMASK_PERM])) {
3692       Ok = parseSwizzleBitmaskPerm(Imm);
3693     } else if (trySkipId(IdSymbolic[ID_BROADCAST])) {
3694       Ok = parseSwizzleBroadcast(Imm);
3695     } else if (trySkipId(IdSymbolic[ID_SWAP])) {
3696       Ok = parseSwizzleSwap(Imm);
3697     } else if (trySkipId(IdSymbolic[ID_REVERSE])) {
3698       Ok = parseSwizzleReverse(Imm);
3699     } else {
3700       Error(ModeLoc, "expected a swizzle mode");
3701     }
3702 
3703     return Ok && skipToken(AsmToken::RParen, "expected a closing parentheses");
3704   }
3705 
3706   return false;
3707 }
3708 
3709 OperandMatchResultTy
3710 AMDGPUAsmParser::parseSwizzleOp(OperandVector &Operands) {
3711   SMLoc S = Parser.getTok().getLoc();
3712   int64_t Imm = 0;
3713 
3714   if (trySkipId("offset")) {
3715 
3716     bool Ok = false;
3717     if (skipToken(AsmToken::Colon, "expected a colon")) {
3718       if (trySkipId("swizzle")) {
3719         Ok = parseSwizzleMacro(Imm);
3720       } else {
3721         Ok = parseSwizzleOffset(Imm);
3722       }
3723     }
3724 
3725     Operands.push_back(AMDGPUOperand::CreateImm(this, Imm, S, AMDGPUOperand::ImmTySwizzle));
3726 
3727     return Ok? MatchOperand_Success : MatchOperand_ParseFail;
3728   } else {
3729     return MatchOperand_NoMatch;
3730   }
3731 }
3732 
3733 bool
3734 AMDGPUOperand::isSwizzle() const {
3735   return isImmTy(ImmTySwizzle);
3736 }
3737 
3738 //===----------------------------------------------------------------------===//
3739 // sopp branch targets
3740 //===----------------------------------------------------------------------===//
3741 
3742 OperandMatchResultTy
3743 AMDGPUAsmParser::parseSOppBrTarget(OperandVector &Operands) {
3744   SMLoc S = Parser.getTok().getLoc();
3745 
3746   switch (getLexer().getKind()) {
3747     default: return MatchOperand_ParseFail;
3748     case AsmToken::Integer: {
3749       int64_t Imm;
3750       if (getParser().parseAbsoluteExpression(Imm))
3751         return MatchOperand_ParseFail;
3752       Operands.push_back(AMDGPUOperand::CreateImm(this, Imm, S));
3753       return MatchOperand_Success;
3754     }
3755 
3756     case AsmToken::Identifier:
3757       Operands.push_back(AMDGPUOperand::CreateExpr(this,
3758           MCSymbolRefExpr::create(getContext().getOrCreateSymbol(
3759                                   Parser.getTok().getString()), getContext()), S));
3760       Parser.Lex();
3761       return MatchOperand_Success;
3762   }
3763 }
3764 
3765 //===----------------------------------------------------------------------===//
3766 // mubuf
3767 //===----------------------------------------------------------------------===//
3768 
3769 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultGLC() const {
3770   return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTyGLC);
3771 }
3772 
3773 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultSLC() const {
3774   return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTySLC);
3775 }
3776 
3777 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultTFE() const {
3778   return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTyTFE);
3779 }
3780 
3781 void AMDGPUAsmParser::cvtMubufImpl(MCInst &Inst,
3782                                const OperandVector &Operands,
3783                                bool IsAtomic, bool IsAtomicReturn) {
3784   OptionalImmIndexMap OptionalIdx;
3785   assert(IsAtomicReturn ? IsAtomic : true);
3786 
3787   for (unsigned i = 1, e = Operands.size(); i != e; ++i) {
3788     AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[i]);
3789 
3790     // Add the register arguments
3791     if (Op.isReg()) {
3792       Op.addRegOperands(Inst, 1);
3793       continue;
3794     }
3795 
3796     // Handle the case where soffset is an immediate
3797     if (Op.isImm() && Op.getImmTy() == AMDGPUOperand::ImmTyNone) {
3798       Op.addImmOperands(Inst, 1);
3799       continue;
3800     }
3801 
3802     // Handle tokens like 'offen' which are sometimes hard-coded into the
3803     // asm string.  There are no MCInst operands for these.
3804     if (Op.isToken()) {
3805       continue;
3806     }
3807     assert(Op.isImm());
3808 
3809     // Handle optional arguments
3810     OptionalIdx[Op.getImmTy()] = i;
3811   }
3812 
3813   // Copy $vdata_in operand and insert as $vdata for MUBUF_Atomic RTN insns.
3814   if (IsAtomicReturn) {
3815     MCInst::iterator I = Inst.begin(); // $vdata_in is always at the beginning.
3816     Inst.insert(I, *I);
3817   }
3818 
3819   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyOffset);
3820   if (!IsAtomic) { // glc is hard-coded.
3821     addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyGLC);
3822   }
3823   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySLC);
3824   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyTFE);
3825 }
3826 
3827 void AMDGPUAsmParser::cvtMtbuf(MCInst &Inst, const OperandVector &Operands) {
3828   OptionalImmIndexMap OptionalIdx;
3829 
3830   for (unsigned i = 1, e = Operands.size(); i != e; ++i) {
3831     AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[i]);
3832 
3833     // Add the register arguments
3834     if (Op.isReg()) {
3835       Op.addRegOperands(Inst, 1);
3836       continue;
3837     }
3838 
3839     // Handle the case where soffset is an immediate
3840     if (Op.isImm() && Op.getImmTy() == AMDGPUOperand::ImmTyNone) {
3841       Op.addImmOperands(Inst, 1);
3842       continue;
3843     }
3844 
3845     // Handle tokens like 'offen' which are sometimes hard-coded into the
3846     // asm string.  There are no MCInst operands for these.
3847     if (Op.isToken()) {
3848       continue;
3849     }
3850     assert(Op.isImm());
3851 
3852     // Handle optional arguments
3853     OptionalIdx[Op.getImmTy()] = i;
3854   }
3855 
3856   addOptionalImmOperand(Inst, Operands, OptionalIdx,
3857                         AMDGPUOperand::ImmTyOffset);
3858   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyDFMT);
3859   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyNFMT);
3860   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyGLC);
3861   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySLC);
3862   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyTFE);
3863 }
3864 
3865 //===----------------------------------------------------------------------===//
3866 // mimg
3867 //===----------------------------------------------------------------------===//
3868 
3869 void AMDGPUAsmParser::cvtMIMG(MCInst &Inst, const OperandVector &Operands,
3870                               bool IsAtomic) {
3871   unsigned I = 1;
3872   const MCInstrDesc &Desc = MII.get(Inst.getOpcode());
3873   for (unsigned J = 0; J < Desc.getNumDefs(); ++J) {
3874     ((AMDGPUOperand &)*Operands[I++]).addRegOperands(Inst, 1);
3875   }
3876 
3877   if (IsAtomic) {
3878     // Add src, same as dst
3879     ((AMDGPUOperand &)*Operands[I]).addRegOperands(Inst, 1);
3880   }
3881 
3882   OptionalImmIndexMap OptionalIdx;
3883 
3884   for (unsigned E = Operands.size(); I != E; ++I) {
3885     AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[I]);
3886 
3887     // Add the register arguments
3888     if (Op.isRegOrImm()) {
3889       Op.addRegOrImmOperands(Inst, 1);
3890       continue;
3891     } else if (Op.isImmModifier()) {
3892       OptionalIdx[Op.getImmTy()] = I;
3893     } else {
3894       llvm_unreachable("unexpected operand type");
3895     }
3896   }
3897 
3898   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyDMask);
3899   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyUNorm);
3900   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyGLC);
3901   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyDA);
3902   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyR128);
3903   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyTFE);
3904   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyLWE);
3905   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySLC);
3906 }
3907 
3908 void AMDGPUAsmParser::cvtMIMGAtomic(MCInst &Inst, const OperandVector &Operands) {
3909   cvtMIMG(Inst, Operands, true);
3910 }
3911 
3912 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultDMask() const {
3913   return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTyDMask);
3914 }
3915 
3916 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultUNorm() const {
3917   return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTyUNorm);
3918 }
3919 
3920 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultDA() const {
3921   return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTyDA);
3922 }
3923 
3924 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultR128() const {
3925   return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTyR128);
3926 }
3927 
3928 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultLWE() const {
3929   return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTyLWE);
3930 }
3931 
3932 //===----------------------------------------------------------------------===//
3933 // smrd
3934 //===----------------------------------------------------------------------===//
3935 
3936 bool AMDGPUOperand::isSMRDOffset8() const {
3937   return isImm() && isUInt<8>(getImm());
3938 }
3939 
3940 bool AMDGPUOperand::isSMRDOffset20() const {
3941   return isImm() && isUInt<20>(getImm());
3942 }
3943 
3944 bool AMDGPUOperand::isSMRDLiteralOffset() const {
3945   // 32-bit literals are only supported on CI and we only want to use them
3946   // when the offset is > 8-bits.
3947   return isImm() && !isUInt<8>(getImm()) && isUInt<32>(getImm());
3948 }
3949 
3950 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultSMRDOffset8() const {
3951   return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTyOffset);
3952 }
3953 
3954 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultSMRDOffset20() const {
3955   return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTyOffset);
3956 }
3957 
3958 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultSMRDLiteralOffset() const {
3959   return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTyOffset);
3960 }
3961 
3962 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultOffsetU12() const {
3963   return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTyOffset);
3964 }
3965 
3966 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultOffsetS13() const {
3967   return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTyOffset);
3968 }
3969 
3970 //===----------------------------------------------------------------------===//
3971 // vop3
3972 //===----------------------------------------------------------------------===//
3973 
3974 static bool ConvertOmodMul(int64_t &Mul) {
3975   if (Mul != 1 && Mul != 2 && Mul != 4)
3976     return false;
3977 
3978   Mul >>= 1;
3979   return true;
3980 }
3981 
3982 static bool ConvertOmodDiv(int64_t &Div) {
3983   if (Div == 1) {
3984     Div = 0;
3985     return true;
3986   }
3987 
3988   if (Div == 2) {
3989     Div = 3;
3990     return true;
3991   }
3992 
3993   return false;
3994 }
3995 
3996 static bool ConvertBoundCtrl(int64_t &BoundCtrl) {
3997   if (BoundCtrl == 0) {
3998     BoundCtrl = 1;
3999     return true;
4000   }
4001 
4002   if (BoundCtrl == -1) {
4003     BoundCtrl = 0;
4004     return true;
4005   }
4006 
4007   return false;
4008 }
4009 
4010 // Note: the order in this table matches the order of operands in AsmString.
4011 static const OptionalOperand AMDGPUOptionalOperandTable[] = {
4012   {"offen",   AMDGPUOperand::ImmTyOffen, true, nullptr},
4013   {"idxen",   AMDGPUOperand::ImmTyIdxen, true, nullptr},
4014   {"addr64",  AMDGPUOperand::ImmTyAddr64, true, nullptr},
4015   {"offset0", AMDGPUOperand::ImmTyOffset0, false, nullptr},
4016   {"offset1", AMDGPUOperand::ImmTyOffset1, false, nullptr},
4017   {"gds",     AMDGPUOperand::ImmTyGDS, true, nullptr},
4018   {"offset",  AMDGPUOperand::ImmTyOffset, false, nullptr},
4019   {"dfmt",    AMDGPUOperand::ImmTyDFMT, false, nullptr},
4020   {"nfmt",    AMDGPUOperand::ImmTyNFMT, false, nullptr},
4021   {"glc",     AMDGPUOperand::ImmTyGLC, true, nullptr},
4022   {"slc",     AMDGPUOperand::ImmTySLC, true, nullptr},
4023   {"tfe",     AMDGPUOperand::ImmTyTFE, true, nullptr},
4024   {"clamp",   AMDGPUOperand::ImmTyClampSI, true, nullptr},
4025   {"omod",    AMDGPUOperand::ImmTyOModSI, false, ConvertOmodMul},
4026   {"unorm",   AMDGPUOperand::ImmTyUNorm, true, nullptr},
4027   {"da",      AMDGPUOperand::ImmTyDA,    true, nullptr},
4028   {"r128",    AMDGPUOperand::ImmTyR128,  true, nullptr},
4029   {"lwe",     AMDGPUOperand::ImmTyLWE,   true, nullptr},
4030   {"dmask",   AMDGPUOperand::ImmTyDMask, false, nullptr},
4031   {"row_mask",   AMDGPUOperand::ImmTyDppRowMask, false, nullptr},
4032   {"bank_mask",  AMDGPUOperand::ImmTyDppBankMask, false, nullptr},
4033   {"bound_ctrl", AMDGPUOperand::ImmTyDppBoundCtrl, false, ConvertBoundCtrl},
4034   {"dst_sel",    AMDGPUOperand::ImmTySdwaDstSel, false, nullptr},
4035   {"src0_sel",   AMDGPUOperand::ImmTySdwaSrc0Sel, false, nullptr},
4036   {"src1_sel",   AMDGPUOperand::ImmTySdwaSrc1Sel, false, nullptr},
4037   {"dst_unused", AMDGPUOperand::ImmTySdwaDstUnused, false, nullptr},
4038   {"compr", AMDGPUOperand::ImmTyExpCompr, true, nullptr },
4039   {"vm", AMDGPUOperand::ImmTyExpVM, true, nullptr},
4040   {"op_sel", AMDGPUOperand::ImmTyOpSel, false, nullptr},
4041   {"op_sel_hi", AMDGPUOperand::ImmTyOpSelHi, false, nullptr},
4042   {"neg_lo", AMDGPUOperand::ImmTyNegLo, false, nullptr},
4043   {"neg_hi", AMDGPUOperand::ImmTyNegHi, false, nullptr}
4044 };
4045 
4046 OperandMatchResultTy AMDGPUAsmParser::parseOptionalOperand(OperandVector &Operands) {
4047   OperandMatchResultTy res;
4048   for (const OptionalOperand &Op : AMDGPUOptionalOperandTable) {
4049     // try to parse any optional operand here
4050     if (Op.IsBit) {
4051       res = parseNamedBit(Op.Name, Operands, Op.Type);
4052     } else if (Op.Type == AMDGPUOperand::ImmTyOModSI) {
4053       res = parseOModOperand(Operands);
4054     } else if (Op.Type == AMDGPUOperand::ImmTySdwaDstSel ||
4055                Op.Type == AMDGPUOperand::ImmTySdwaSrc0Sel ||
4056                Op.Type == AMDGPUOperand::ImmTySdwaSrc1Sel) {
4057       res = parseSDWASel(Operands, Op.Name, Op.Type);
4058     } else if (Op.Type == AMDGPUOperand::ImmTySdwaDstUnused) {
4059       res = parseSDWADstUnused(Operands);
4060     } else if (Op.Type == AMDGPUOperand::ImmTyOpSel ||
4061                Op.Type == AMDGPUOperand::ImmTyOpSelHi ||
4062                Op.Type == AMDGPUOperand::ImmTyNegLo ||
4063                Op.Type == AMDGPUOperand::ImmTyNegHi) {
4064       res = parseOperandArrayWithPrefix(Op.Name, Operands, Op.Type,
4065                                         Op.ConvertResult);
4066     } else {
4067       res = parseIntWithPrefix(Op.Name, Operands, Op.Type, Op.ConvertResult);
4068     }
4069     if (res != MatchOperand_NoMatch) {
4070       return res;
4071     }
4072   }
4073   return MatchOperand_NoMatch;
4074 }
4075 
4076 OperandMatchResultTy AMDGPUAsmParser::parseOModOperand(OperandVector &Operands) {
4077   StringRef Name = Parser.getTok().getString();
4078   if (Name == "mul") {
4079     return parseIntWithPrefix("mul", Operands,
4080                               AMDGPUOperand::ImmTyOModSI, ConvertOmodMul);
4081   }
4082 
4083   if (Name == "div") {
4084     return parseIntWithPrefix("div", Operands,
4085                               AMDGPUOperand::ImmTyOModSI, ConvertOmodDiv);
4086   }
4087 
4088   return MatchOperand_NoMatch;
4089 }
4090 
4091 static bool isRegOrImmWithInputMods(const MCInstrDesc &Desc, unsigned OpNum) {
4092       // 1. This operand is input modifiers
4093   return Desc.OpInfo[OpNum].OperandType == AMDGPU::OPERAND_INPUT_MODS
4094       // 2. This is not last operand
4095       && Desc.NumOperands > (OpNum + 1)
4096       // 3. Next operand is register class
4097       && Desc.OpInfo[OpNum + 1].RegClass != -1
4098       // 4. Next register is not tied to any other operand
4099       && Desc.getOperandConstraint(OpNum + 1, MCOI::OperandConstraint::TIED_TO) == -1;
4100 }
4101 
4102 void AMDGPUAsmParser::cvtVOP3(MCInst &Inst, const OperandVector &Operands,
4103                               OptionalImmIndexMap &OptionalIdx) {
4104   unsigned Opc = Inst.getOpcode();
4105 
4106   unsigned I = 1;
4107   const MCInstrDesc &Desc = MII.get(Inst.getOpcode());
4108   for (unsigned J = 0; J < Desc.getNumDefs(); ++J) {
4109     ((AMDGPUOperand &)*Operands[I++]).addRegOperands(Inst, 1);
4110   }
4111 
4112   if (AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src0_modifiers) != -1) {
4113     // This instruction has src modifiers
4114     for (unsigned E = Operands.size(); I != E; ++I) {
4115       AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[I]);
4116       if (isRegOrImmWithInputMods(Desc, Inst.getNumOperands())) {
4117         Op.addRegOrImmWithFPInputModsOperands(Inst, 2);
4118       } else if (Op.isImmModifier()) {
4119         OptionalIdx[Op.getImmTy()] = I;
4120       } else if (Op.isRegOrImm()) {
4121         Op.addRegOrImmOperands(Inst, 1);
4122       } else {
4123         llvm_unreachable("unhandled operand type");
4124       }
4125     }
4126   } else {
4127     // No src modifiers
4128     for (unsigned E = Operands.size(); I != E; ++I) {
4129       AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[I]);
4130       if (Op.isMod()) {
4131         OptionalIdx[Op.getImmTy()] = I;
4132       } else {
4133         Op.addRegOrImmOperands(Inst, 1);
4134       }
4135     }
4136   }
4137 
4138   if (AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::clamp) != -1) {
4139     addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyClampSI);
4140   }
4141 
4142   if (AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::omod) != -1) {
4143     addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyOModSI);
4144   }
4145 
4146   // special case v_mac_{f16, f32}:
4147   // it has src2 register operand that is tied to dst operand
4148   // we don't allow modifiers for this operand in assembler so src2_modifiers
4149   // should be 0
4150   if (Opc == AMDGPU::V_MAC_F32_e64_si || Opc == AMDGPU::V_MAC_F32_e64_vi ||
4151       Opc == AMDGPU::V_MAC_F16_e64_vi) {
4152     auto it = Inst.begin();
4153     std::advance(it, AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src2_modifiers));
4154     it = Inst.insert(it, MCOperand::createImm(0)); // no modifiers for src2
4155     ++it;
4156     Inst.insert(it, Inst.getOperand(0)); // src2 = dst
4157   }
4158 }
4159 
4160 void AMDGPUAsmParser::cvtVOP3(MCInst &Inst, const OperandVector &Operands) {
4161   OptionalImmIndexMap OptionalIdx;
4162   cvtVOP3(Inst, Operands, OptionalIdx);
4163 }
4164 
4165 void AMDGPUAsmParser::cvtVOP3P(MCInst &Inst, const OperandVector &Operands) {
4166   OptionalImmIndexMap OptIdx;
4167 
4168   cvtVOP3(Inst, Operands, OptIdx);
4169 
4170   // FIXME: This is messy. Parse the modifiers as if it was a normal VOP3
4171   // instruction, and then figure out where to actually put the modifiers
4172   int Opc = Inst.getOpcode();
4173 
4174   addOptionalImmOperand(Inst, Operands, OptIdx, AMDGPUOperand::ImmTyOpSel);
4175   addOptionalImmOperand(Inst, Operands, OptIdx, AMDGPUOperand::ImmTyOpSelHi, -1);
4176 
4177   int NegLoIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::neg_lo);
4178   if (NegLoIdx != -1) {
4179     addOptionalImmOperand(Inst, Operands, OptIdx, AMDGPUOperand::ImmTyNegLo);
4180     addOptionalImmOperand(Inst, Operands, OptIdx, AMDGPUOperand::ImmTyNegHi);
4181   }
4182 
4183   const int Ops[] = { AMDGPU::OpName::src0,
4184                       AMDGPU::OpName::src1,
4185                       AMDGPU::OpName::src2 };
4186   const int ModOps[] = { AMDGPU::OpName::src0_modifiers,
4187                          AMDGPU::OpName::src1_modifiers,
4188                          AMDGPU::OpName::src2_modifiers };
4189 
4190   int OpSelIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::op_sel);
4191   int OpSelHiIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::op_sel_hi);
4192 
4193   unsigned OpSel = Inst.getOperand(OpSelIdx).getImm();
4194   unsigned OpSelHi = Inst.getOperand(OpSelHiIdx).getImm();
4195   unsigned NegLo = 0;
4196   unsigned NegHi = 0;
4197 
4198   if (NegLoIdx != -1) {
4199     int NegHiIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::neg_hi);
4200     NegLo = Inst.getOperand(NegLoIdx).getImm();
4201     NegHi = Inst.getOperand(NegHiIdx).getImm();
4202   }
4203 
4204   for (int J = 0; J < 3; ++J) {
4205     int OpIdx = AMDGPU::getNamedOperandIdx(Opc, Ops[J]);
4206     if (OpIdx == -1)
4207       break;
4208 
4209     uint32_t ModVal = 0;
4210 
4211     if ((OpSel & (1 << J)) != 0)
4212       ModVal |= SISrcMods::OP_SEL_0;
4213 
4214     if ((OpSelHi & (1 << J)) != 0)
4215       ModVal |= SISrcMods::OP_SEL_1;
4216 
4217     if ((NegLo & (1 << J)) != 0)
4218       ModVal |= SISrcMods::NEG;
4219 
4220     if ((NegHi & (1 << J)) != 0)
4221       ModVal |= SISrcMods::NEG_HI;
4222 
4223     int ModIdx = AMDGPU::getNamedOperandIdx(Opc, ModOps[J]);
4224 
4225     Inst.getOperand(ModIdx).setImm(Inst.getOperand(ModIdx).getImm() | ModVal);
4226   }
4227 }
4228 
4229 //===----------------------------------------------------------------------===//
4230 // dpp
4231 //===----------------------------------------------------------------------===//
4232 
4233 bool AMDGPUOperand::isDPPCtrl() const {
4234   bool result = isImm() && getImmTy() == ImmTyDppCtrl && isUInt<9>(getImm());
4235   if (result) {
4236     int64_t Imm = getImm();
4237     return ((Imm >= 0x000) && (Imm <= 0x0ff)) ||
4238            ((Imm >= 0x101) && (Imm <= 0x10f)) ||
4239            ((Imm >= 0x111) && (Imm <= 0x11f)) ||
4240            ((Imm >= 0x121) && (Imm <= 0x12f)) ||
4241            (Imm == 0x130) ||
4242            (Imm == 0x134) ||
4243            (Imm == 0x138) ||
4244            (Imm == 0x13c) ||
4245            (Imm == 0x140) ||
4246            (Imm == 0x141) ||
4247            (Imm == 0x142) ||
4248            (Imm == 0x143);
4249   }
4250   return false;
4251 }
4252 
4253 bool AMDGPUOperand::isGPRIdxMode() const {
4254   return isImm() && isUInt<4>(getImm());
4255 }
4256 
4257 bool AMDGPUOperand::isS16Imm() const {
4258   return isImm() && (isInt<16>(getImm()) || isUInt<16>(getImm()));
4259 }
4260 
4261 bool AMDGPUOperand::isU16Imm() const {
4262   return isImm() && isUInt<16>(getImm());
4263 }
4264 
4265 OperandMatchResultTy
4266 AMDGPUAsmParser::parseDPPCtrl(OperandVector &Operands) {
4267   SMLoc S = Parser.getTok().getLoc();
4268   StringRef Prefix;
4269   int64_t Int;
4270 
4271   if (getLexer().getKind() == AsmToken::Identifier) {
4272     Prefix = Parser.getTok().getString();
4273   } else {
4274     return MatchOperand_NoMatch;
4275   }
4276 
4277   if (Prefix == "row_mirror") {
4278     Int = 0x140;
4279     Parser.Lex();
4280   } else if (Prefix == "row_half_mirror") {
4281     Int = 0x141;
4282     Parser.Lex();
4283   } else {
4284     // Check to prevent parseDPPCtrlOps from eating invalid tokens
4285     if (Prefix != "quad_perm"
4286         && Prefix != "row_shl"
4287         && Prefix != "row_shr"
4288         && Prefix != "row_ror"
4289         && Prefix != "wave_shl"
4290         && Prefix != "wave_rol"
4291         && Prefix != "wave_shr"
4292         && Prefix != "wave_ror"
4293         && Prefix != "row_bcast") {
4294       return MatchOperand_NoMatch;
4295     }
4296 
4297     Parser.Lex();
4298     if (getLexer().isNot(AsmToken::Colon))
4299       return MatchOperand_ParseFail;
4300 
4301     if (Prefix == "quad_perm") {
4302       // quad_perm:[%d,%d,%d,%d]
4303       Parser.Lex();
4304       if (getLexer().isNot(AsmToken::LBrac))
4305         return MatchOperand_ParseFail;
4306       Parser.Lex();
4307 
4308       if (getParser().parseAbsoluteExpression(Int) || !(0 <= Int && Int <=3))
4309         return MatchOperand_ParseFail;
4310 
4311       for (int i = 0; i < 3; ++i) {
4312         if (getLexer().isNot(AsmToken::Comma))
4313           return MatchOperand_ParseFail;
4314         Parser.Lex();
4315 
4316         int64_t Temp;
4317         if (getParser().parseAbsoluteExpression(Temp) || !(0 <= Temp && Temp <=3))
4318           return MatchOperand_ParseFail;
4319         const int shift = i*2 + 2;
4320         Int += (Temp << shift);
4321       }
4322 
4323       if (getLexer().isNot(AsmToken::RBrac))
4324         return MatchOperand_ParseFail;
4325       Parser.Lex();
4326 
4327     } else {
4328       // sel:%d
4329       Parser.Lex();
4330       if (getParser().parseAbsoluteExpression(Int))
4331         return MatchOperand_ParseFail;
4332 
4333       if (Prefix == "row_shl" && 1 <= Int && Int <= 15) {
4334         Int |= 0x100;
4335       } else if (Prefix == "row_shr" && 1 <= Int && Int <= 15) {
4336         Int |= 0x110;
4337       } else if (Prefix == "row_ror" && 1 <= Int && Int <= 15) {
4338         Int |= 0x120;
4339       } else if (Prefix == "wave_shl" && 1 == Int) {
4340         Int = 0x130;
4341       } else if (Prefix == "wave_rol" && 1 == Int) {
4342         Int = 0x134;
4343       } else if (Prefix == "wave_shr" && 1 == Int) {
4344         Int = 0x138;
4345       } else if (Prefix == "wave_ror" && 1 == Int) {
4346         Int = 0x13C;
4347       } else if (Prefix == "row_bcast") {
4348         if (Int == 15) {
4349           Int = 0x142;
4350         } else if (Int == 31) {
4351           Int = 0x143;
4352         } else {
4353           return MatchOperand_ParseFail;
4354         }
4355       } else {
4356         return MatchOperand_ParseFail;
4357       }
4358     }
4359   }
4360 
4361   Operands.push_back(AMDGPUOperand::CreateImm(this, Int, S, AMDGPUOperand::ImmTyDppCtrl));
4362   return MatchOperand_Success;
4363 }
4364 
4365 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultRowMask() const {
4366   return AMDGPUOperand::CreateImm(this, 0xf, SMLoc(), AMDGPUOperand::ImmTyDppRowMask);
4367 }
4368 
4369 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultBankMask() const {
4370   return AMDGPUOperand::CreateImm(this, 0xf, SMLoc(), AMDGPUOperand::ImmTyDppBankMask);
4371 }
4372 
4373 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultBoundCtrl() const {
4374   return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTyDppBoundCtrl);
4375 }
4376 
4377 void AMDGPUAsmParser::cvtDPP(MCInst &Inst, const OperandVector &Operands) {
4378   OptionalImmIndexMap OptionalIdx;
4379 
4380   unsigned I = 1;
4381   const MCInstrDesc &Desc = MII.get(Inst.getOpcode());
4382   for (unsigned J = 0; J < Desc.getNumDefs(); ++J) {
4383     ((AMDGPUOperand &)*Operands[I++]).addRegOperands(Inst, 1);
4384   }
4385 
4386   for (unsigned E = Operands.size(); I != E; ++I) {
4387     AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[I]);
4388     // Add the register arguments
4389     if (Op.isReg() && Op.Reg.RegNo == AMDGPU::VCC) {
4390       // VOP2b (v_add_u32, v_sub_u32 ...) dpp use "vcc" token.
4391       // Skip it.
4392       continue;
4393     } if (isRegOrImmWithInputMods(Desc, Inst.getNumOperands())) {
4394       Op.addRegWithFPInputModsOperands(Inst, 2);
4395     } else if (Op.isDPPCtrl()) {
4396       Op.addImmOperands(Inst, 1);
4397     } else if (Op.isImm()) {
4398       // Handle optional arguments
4399       OptionalIdx[Op.getImmTy()] = I;
4400     } else {
4401       llvm_unreachable("Invalid operand type");
4402     }
4403   }
4404 
4405   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyDppRowMask, 0xf);
4406   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyDppBankMask, 0xf);
4407   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyDppBoundCtrl);
4408 
4409   // special case v_mac_{f16, f32}:
4410   // it has src2 register operand that is tied to dst operand
4411   if (Inst.getOpcode() == AMDGPU::V_MAC_F32_dpp ||
4412       Inst.getOpcode() == AMDGPU::V_MAC_F16_dpp) {
4413     auto it = Inst.begin();
4414     std::advance(
4415         it, AMDGPU::getNamedOperandIdx(Inst.getOpcode(), AMDGPU::OpName::src2));
4416     Inst.insert(it, Inst.getOperand(0)); // src2 = dst
4417   }
4418 }
4419 
4420 //===----------------------------------------------------------------------===//
4421 // sdwa
4422 //===----------------------------------------------------------------------===//
4423 
4424 OperandMatchResultTy
4425 AMDGPUAsmParser::parseSDWASel(OperandVector &Operands, StringRef Prefix,
4426                               AMDGPUOperand::ImmTy Type) {
4427   using namespace llvm::AMDGPU::SDWA;
4428 
4429   SMLoc S = Parser.getTok().getLoc();
4430   StringRef Value;
4431   OperandMatchResultTy res;
4432 
4433   res = parseStringWithPrefix(Prefix, Value);
4434   if (res != MatchOperand_Success) {
4435     return res;
4436   }
4437 
4438   int64_t Int;
4439   Int = StringSwitch<int64_t>(Value)
4440         .Case("BYTE_0", SdwaSel::BYTE_0)
4441         .Case("BYTE_1", SdwaSel::BYTE_1)
4442         .Case("BYTE_2", SdwaSel::BYTE_2)
4443         .Case("BYTE_3", SdwaSel::BYTE_3)
4444         .Case("WORD_0", SdwaSel::WORD_0)
4445         .Case("WORD_1", SdwaSel::WORD_1)
4446         .Case("DWORD", SdwaSel::DWORD)
4447         .Default(0xffffffff);
4448   Parser.Lex(); // eat last token
4449 
4450   if (Int == 0xffffffff) {
4451     return MatchOperand_ParseFail;
4452   }
4453 
4454   Operands.push_back(AMDGPUOperand::CreateImm(this, Int, S, Type));
4455   return MatchOperand_Success;
4456 }
4457 
4458 OperandMatchResultTy
4459 AMDGPUAsmParser::parseSDWADstUnused(OperandVector &Operands) {
4460   using namespace llvm::AMDGPU::SDWA;
4461 
4462   SMLoc S = Parser.getTok().getLoc();
4463   StringRef Value;
4464   OperandMatchResultTy res;
4465 
4466   res = parseStringWithPrefix("dst_unused", Value);
4467   if (res != MatchOperand_Success) {
4468     return res;
4469   }
4470 
4471   int64_t Int;
4472   Int = StringSwitch<int64_t>(Value)
4473         .Case("UNUSED_PAD", DstUnused::UNUSED_PAD)
4474         .Case("UNUSED_SEXT", DstUnused::UNUSED_SEXT)
4475         .Case("UNUSED_PRESERVE", DstUnused::UNUSED_PRESERVE)
4476         .Default(0xffffffff);
4477   Parser.Lex(); // eat last token
4478 
4479   if (Int == 0xffffffff) {
4480     return MatchOperand_ParseFail;
4481   }
4482 
4483   Operands.push_back(AMDGPUOperand::CreateImm(this, Int, S, AMDGPUOperand::ImmTySdwaDstUnused));
4484   return MatchOperand_Success;
4485 }
4486 
4487 void AMDGPUAsmParser::cvtSdwaVOP1(MCInst &Inst, const OperandVector &Operands) {
4488   cvtSDWA(Inst, Operands, SIInstrFlags::VOP1);
4489 }
4490 
4491 void AMDGPUAsmParser::cvtSdwaVOP2(MCInst &Inst, const OperandVector &Operands) {
4492   cvtSDWA(Inst, Operands, SIInstrFlags::VOP2);
4493 }
4494 
4495 void AMDGPUAsmParser::cvtSdwaVOP2b(MCInst &Inst, const OperandVector &Operands) {
4496   cvtSDWA(Inst, Operands, SIInstrFlags::VOP2, true);
4497 }
4498 
4499 void AMDGPUAsmParser::cvtSdwaVOPC(MCInst &Inst, const OperandVector &Operands) {
4500   cvtSDWA(Inst, Operands, SIInstrFlags::VOPC, isVI());
4501 }
4502 
4503 void AMDGPUAsmParser::cvtSDWA(MCInst &Inst, const OperandVector &Operands,
4504                               uint64_t BasicInstType, bool skipVcc) {
4505   using namespace llvm::AMDGPU::SDWA;
4506   OptionalImmIndexMap OptionalIdx;
4507   bool skippedVcc = false;
4508 
4509   unsigned I = 1;
4510   const MCInstrDesc &Desc = MII.get(Inst.getOpcode());
4511   for (unsigned J = 0; J < Desc.getNumDefs(); ++J) {
4512     ((AMDGPUOperand &)*Operands[I++]).addRegOperands(Inst, 1);
4513   }
4514 
4515   for (unsigned E = Operands.size(); I != E; ++I) {
4516     AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[I]);
4517     if (skipVcc && !skippedVcc && Op.isReg() && Op.Reg.RegNo == AMDGPU::VCC) {
4518       // VOP2b (v_add_u32, v_sub_u32 ...) sdwa use "vcc" token as dst.
4519       // Skip it if it's 2nd (e.g. v_add_i32_sdwa v1, vcc, v2, v3)
4520       // or 4th (v_addc_u32_sdwa v1, vcc, v2, v3, vcc) operand.
4521       // Skip VCC only if we didn't skip it on previous iteration.
4522       if (BasicInstType == SIInstrFlags::VOP2 &&
4523           (Inst.getNumOperands() == 1 || Inst.getNumOperands() == 5)) {
4524         skippedVcc = true;
4525         continue;
4526       } else if (BasicInstType == SIInstrFlags::VOPC &&
4527                  Inst.getNumOperands() == 0) {
4528         skippedVcc = true;
4529         continue;
4530       }
4531     }
4532     if (isRegOrImmWithInputMods(Desc, Inst.getNumOperands())) {
4533       Op.addRegWithInputModsOperands(Inst, 2);
4534     } else if (Op.isImm()) {
4535       // Handle optional arguments
4536       OptionalIdx[Op.getImmTy()] = I;
4537     } else {
4538       llvm_unreachable("Invalid operand type");
4539     }
4540     skippedVcc = false;
4541   }
4542 
4543   if (Inst.getOpcode() != AMDGPU::V_NOP_sdwa_gfx9 &&
4544       Inst.getOpcode() != AMDGPU::V_NOP_sdwa_vi) {
4545     // v_nop_sdwa_sdwa_vi/gfx9 has no optional sdwa arguments
4546     switch (BasicInstType) {
4547     case SIInstrFlags::VOP1:
4548       addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyClampSI, 0);
4549       if (AMDGPU::getNamedOperandIdx(Inst.getOpcode(), AMDGPU::OpName::omod) != -1) {
4550         addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyOModSI, 0);
4551       }
4552       addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySdwaDstSel, SdwaSel::DWORD);
4553       addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySdwaDstUnused, DstUnused::UNUSED_PRESERVE);
4554       addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySdwaSrc0Sel, SdwaSel::DWORD);
4555       break;
4556 
4557     case SIInstrFlags::VOP2:
4558       addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyClampSI, 0);
4559       if (AMDGPU::getNamedOperandIdx(Inst.getOpcode(), AMDGPU::OpName::omod) != -1) {
4560         addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyOModSI, 0);
4561       }
4562       addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySdwaDstSel, SdwaSel::DWORD);
4563       addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySdwaDstUnused, DstUnused::UNUSED_PRESERVE);
4564       addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySdwaSrc0Sel, SdwaSel::DWORD);
4565       addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySdwaSrc1Sel, SdwaSel::DWORD);
4566       break;
4567 
4568     case SIInstrFlags::VOPC:
4569       addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyClampSI, 0);
4570       addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySdwaSrc0Sel, SdwaSel::DWORD);
4571       addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySdwaSrc1Sel, SdwaSel::DWORD);
4572       break;
4573 
4574     default:
4575       llvm_unreachable("Invalid instruction type. Only VOP1, VOP2 and VOPC allowed");
4576     }
4577   }
4578 
4579   // special case v_mac_{f16, f32}:
4580   // it has src2 register operand that is tied to dst operand
4581   if (Inst.getOpcode() == AMDGPU::V_MAC_F32_sdwa_vi ||
4582       Inst.getOpcode() == AMDGPU::V_MAC_F16_sdwa_vi)  {
4583     auto it = Inst.begin();
4584     std::advance(
4585       it, AMDGPU::getNamedOperandIdx(Inst.getOpcode(), AMDGPU::OpName::src2));
4586     Inst.insert(it, Inst.getOperand(0)); // src2 = dst
4587   }
4588 }
4589 
4590 /// Force static initialization.
4591 extern "C" void LLVMInitializeAMDGPUAsmParser() {
4592   RegisterMCAsmParser<AMDGPUAsmParser> A(getTheAMDGPUTarget());
4593   RegisterMCAsmParser<AMDGPUAsmParser> B(getTheGCNTarget());
4594 }
4595 
4596 #define GET_REGISTER_MATCHER
4597 #define GET_MATCHER_IMPLEMENTATION
4598 #include "AMDGPUGenAsmMatcher.inc"
4599 
4600 // This fuction should be defined after auto-generated include so that we have
4601 // MatchClassKind enum defined
4602 unsigned AMDGPUAsmParser::validateTargetOperandClass(MCParsedAsmOperand &Op,
4603                                                      unsigned Kind) {
4604   // Tokens like "glc" would be parsed as immediate operands in ParseOperand().
4605   // But MatchInstructionImpl() expects to meet token and fails to validate
4606   // operand. This method checks if we are given immediate operand but expect to
4607   // get corresponding token.
4608   AMDGPUOperand &Operand = (AMDGPUOperand&)Op;
4609   switch (Kind) {
4610   case MCK_addr64:
4611     return Operand.isAddr64() ? Match_Success : Match_InvalidOperand;
4612   case MCK_gds:
4613     return Operand.isGDS() ? Match_Success : Match_InvalidOperand;
4614   case MCK_glc:
4615     return Operand.isGLC() ? Match_Success : Match_InvalidOperand;
4616   case MCK_idxen:
4617     return Operand.isIdxen() ? Match_Success : Match_InvalidOperand;
4618   case MCK_offen:
4619     return Operand.isOffen() ? Match_Success : Match_InvalidOperand;
4620   case MCK_SSrcB32:
4621     // When operands have expression values, they will return true for isToken,
4622     // because it is not possible to distinguish between a token and an
4623     // expression at parse time. MatchInstructionImpl() will always try to
4624     // match an operand as a token, when isToken returns true, and when the
4625     // name of the expression is not a valid token, the match will fail,
4626     // so we need to handle it here.
4627     return Operand.isSSrcB32() ? Match_Success : Match_InvalidOperand;
4628   case MCK_SSrcF32:
4629     return Operand.isSSrcF32() ? Match_Success : Match_InvalidOperand;
4630   case MCK_SoppBrTarget:
4631     return Operand.isSoppBrTarget() ? Match_Success : Match_InvalidOperand;
4632   case MCK_VReg32OrOff:
4633     return Operand.isVReg32OrOff() ? Match_Success : Match_InvalidOperand;
4634   case MCK_InterpSlot:
4635     return Operand.isInterpSlot() ? Match_Success : Match_InvalidOperand;
4636   case MCK_Attr:
4637     return Operand.isInterpAttr() ? Match_Success : Match_InvalidOperand;
4638   case MCK_AttrChan:
4639     return Operand.isAttrChan() ? Match_Success : Match_InvalidOperand;
4640   default:
4641     return Match_InvalidOperand;
4642   }
4643 }
4644