1 //===-- AMDGPUAsmParser.cpp - Parse SI asm to MCInst instructions ---------===//
2 //
3 //                     The LLVM Compiler Infrastructure
4 //
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
7 //
8 //===----------------------------------------------------------------------===//
9 
10 #include "AMDKernelCodeT.h"
11 #include "MCTargetDesc/AMDGPUMCTargetDesc.h"
12 #include "MCTargetDesc/AMDGPUTargetStreamer.h"
13 #include "SIDefines.h"
14 #include "Utils/AMDGPUAsmUtils.h"
15 #include "Utils/AMDGPUBaseInfo.h"
16 #include "Utils/AMDKernelCodeTUtils.h"
17 #include "llvm/ADT/APFloat.h"
18 #include "llvm/ADT/APInt.h"
19 #include "llvm/ADT/ArrayRef.h"
20 #include "llvm/ADT/STLExtras.h"
21 #include "llvm/ADT/SmallBitVector.h"
22 #include "llvm/ADT/SmallString.h"
23 #include "llvm/ADT/StringRef.h"
24 #include "llvm/ADT/StringSwitch.h"
25 #include "llvm/ADT/Twine.h"
26 #include "llvm/BinaryFormat/ELF.h"
27 #include "llvm/CodeGen/MachineValueType.h"
28 #include "llvm/MC/MCAsmInfo.h"
29 #include "llvm/MC/MCContext.h"
30 #include "llvm/MC/MCExpr.h"
31 #include "llvm/MC/MCInst.h"
32 #include "llvm/MC/MCInstrDesc.h"
33 #include "llvm/MC/MCInstrInfo.h"
34 #include "llvm/MC/MCParser/MCAsmLexer.h"
35 #include "llvm/MC/MCParser/MCAsmParser.h"
36 #include "llvm/MC/MCParser/MCAsmParserExtension.h"
37 #include "llvm/MC/MCParser/MCParsedAsmOperand.h"
38 #include "llvm/MC/MCParser/MCTargetAsmParser.h"
39 #include "llvm/MC/MCRegisterInfo.h"
40 #include "llvm/MC/MCStreamer.h"
41 #include "llvm/MC/MCSubtargetInfo.h"
42 #include "llvm/MC/MCSymbol.h"
43 #include "llvm/Support/Casting.h"
44 #include "llvm/Support/ErrorHandling.h"
45 #include "llvm/Support/MathExtras.h"
46 #include "llvm/Support/SMLoc.h"
47 #include "llvm/Support/TargetRegistry.h"
48 #include "llvm/Support/raw_ostream.h"
49 #include <algorithm>
50 #include <cassert>
51 #include <cstdint>
52 #include <cstring>
53 #include <iterator>
54 #include <map>
55 #include <memory>
56 #include <string>
57 
58 using namespace llvm;
59 using namespace llvm::AMDGPU;
60 
61 namespace {
62 
63 class AMDGPUAsmParser;
64 
65 enum RegisterKind { IS_UNKNOWN, IS_VGPR, IS_SGPR, IS_TTMP, IS_SPECIAL };
66 
67 //===----------------------------------------------------------------------===//
68 // Operand
69 //===----------------------------------------------------------------------===//
70 
71 class AMDGPUOperand : public MCParsedAsmOperand {
72   enum KindTy {
73     Token,
74     Immediate,
75     Register,
76     Expression
77   } Kind;
78 
79   SMLoc StartLoc, EndLoc;
80   const AMDGPUAsmParser *AsmParser;
81 
82 public:
83   AMDGPUOperand(KindTy Kind_, const AMDGPUAsmParser *AsmParser_)
84     : MCParsedAsmOperand(), Kind(Kind_), AsmParser(AsmParser_) {}
85 
86   typedef std::unique_ptr<AMDGPUOperand> Ptr;
87 
88   struct Modifiers {
89     bool Abs = false;
90     bool Neg = false;
91     bool Sext = false;
92 
93     bool hasFPModifiers() const { return Abs || Neg; }
94     bool hasIntModifiers() const { return Sext; }
95     bool hasModifiers() const { return hasFPModifiers() || hasIntModifiers(); }
96 
97     int64_t getFPModifiersOperand() const {
98       int64_t Operand = 0;
99       Operand |= Abs ? SISrcMods::ABS : 0;
100       Operand |= Neg ? SISrcMods::NEG : 0;
101       return Operand;
102     }
103 
104     int64_t getIntModifiersOperand() const {
105       int64_t Operand = 0;
106       Operand |= Sext ? SISrcMods::SEXT : 0;
107       return Operand;
108     }
109 
110     int64_t getModifiersOperand() const {
111       assert(!(hasFPModifiers() && hasIntModifiers())
112            && "fp and int modifiers should not be used simultaneously");
113       if (hasFPModifiers()) {
114         return getFPModifiersOperand();
115       } else if (hasIntModifiers()) {
116         return getIntModifiersOperand();
117       } else {
118         return 0;
119       }
120     }
121 
122     friend raw_ostream &operator <<(raw_ostream &OS, AMDGPUOperand::Modifiers Mods);
123   };
124 
125   enum ImmTy {
126     ImmTyNone,
127     ImmTyGDS,
128     ImmTyOffen,
129     ImmTyIdxen,
130     ImmTyAddr64,
131     ImmTyOffset,
132     ImmTyOffset0,
133     ImmTyOffset1,
134     ImmTyGLC,
135     ImmTySLC,
136     ImmTyTFE,
137     ImmTyClampSI,
138     ImmTyOModSI,
139     ImmTyDppCtrl,
140     ImmTyDppRowMask,
141     ImmTyDppBankMask,
142     ImmTyDppBoundCtrl,
143     ImmTySdwaDstSel,
144     ImmTySdwaSrc0Sel,
145     ImmTySdwaSrc1Sel,
146     ImmTySdwaDstUnused,
147     ImmTyDMask,
148     ImmTyUNorm,
149     ImmTyDA,
150     ImmTyR128,
151     ImmTyLWE,
152     ImmTyExpTgt,
153     ImmTyExpCompr,
154     ImmTyExpVM,
155     ImmTyDFMT,
156     ImmTyNFMT,
157     ImmTyHwreg,
158     ImmTyOff,
159     ImmTySendMsg,
160     ImmTyInterpSlot,
161     ImmTyInterpAttr,
162     ImmTyAttrChan,
163     ImmTyOpSel,
164     ImmTyOpSelHi,
165     ImmTyNegLo,
166     ImmTyNegHi,
167     ImmTySwizzle
168   };
169 
170   struct TokOp {
171     const char *Data;
172     unsigned Length;
173   };
174 
175   struct ImmOp {
176     int64_t Val;
177     ImmTy Type;
178     bool IsFPImm;
179     Modifiers Mods;
180   };
181 
182   struct RegOp {
183     unsigned RegNo;
184     bool IsForcedVOP3;
185     Modifiers Mods;
186   };
187 
188   union {
189     TokOp Tok;
190     ImmOp Imm;
191     RegOp Reg;
192     const MCExpr *Expr;
193   };
194 
195   bool isToken() const override {
196     if (Kind == Token)
197       return true;
198 
199     if (Kind != Expression || !Expr)
200       return false;
201 
202     // When parsing operands, we can't always tell if something was meant to be
203     // a token, like 'gds', or an expression that references a global variable.
204     // In this case, we assume the string is an expression, and if we need to
205     // interpret is a token, then we treat the symbol name as the token.
206     return isa<MCSymbolRefExpr>(Expr);
207   }
208 
209   bool isImm() const override {
210     return Kind == Immediate;
211   }
212 
213   bool isInlinableImm(MVT type) const;
214   bool isLiteralImm(MVT type) const;
215 
216   bool isRegKind() const {
217     return Kind == Register;
218   }
219 
220   bool isReg() const override {
221     return isRegKind() && !hasModifiers();
222   }
223 
224   bool isRegOrImmWithInputMods(MVT type) const {
225     return isRegKind() || isInlinableImm(type);
226   }
227 
228   bool isRegOrImmWithInt16InputMods() const {
229     return isRegOrImmWithInputMods(MVT::i16);
230   }
231 
232   bool isRegOrImmWithInt32InputMods() const {
233     return isRegOrImmWithInputMods(MVT::i32);
234   }
235 
236   bool isRegOrImmWithInt64InputMods() const {
237     return isRegOrImmWithInputMods(MVT::i64);
238   }
239 
240   bool isRegOrImmWithFP16InputMods() const {
241     return isRegOrImmWithInputMods(MVT::f16);
242   }
243 
244   bool isRegOrImmWithFP32InputMods() const {
245     return isRegOrImmWithInputMods(MVT::f32);
246   }
247 
248   bool isRegOrImmWithFP64InputMods() const {
249     return isRegOrImmWithInputMods(MVT::f64);
250   }
251 
252   bool isVReg() const {
253     return isRegClass(AMDGPU::VGPR_32RegClassID) ||
254            isRegClass(AMDGPU::VReg_64RegClassID) ||
255            isRegClass(AMDGPU::VReg_96RegClassID) ||
256            isRegClass(AMDGPU::VReg_128RegClassID) ||
257            isRegClass(AMDGPU::VReg_256RegClassID) ||
258            isRegClass(AMDGPU::VReg_512RegClassID);
259   }
260 
261   bool isVReg32OrOff() const {
262     return isOff() || isRegClass(AMDGPU::VGPR_32RegClassID);
263   }
264 
265   bool isSDWARegKind() const;
266 
267   bool isImmTy(ImmTy ImmT) const {
268     return isImm() && Imm.Type == ImmT;
269   }
270 
271   bool isImmModifier() const {
272     return isImm() && Imm.Type != ImmTyNone;
273   }
274 
275   bool isClampSI() const { return isImmTy(ImmTyClampSI); }
276   bool isOModSI() const { return isImmTy(ImmTyOModSI); }
277   bool isDMask() const { return isImmTy(ImmTyDMask); }
278   bool isUNorm() const { return isImmTy(ImmTyUNorm); }
279   bool isDA() const { return isImmTy(ImmTyDA); }
280   bool isR128() const { return isImmTy(ImmTyUNorm); }
281   bool isLWE() const { return isImmTy(ImmTyLWE); }
282   bool isOff() const { return isImmTy(ImmTyOff); }
283   bool isExpTgt() const { return isImmTy(ImmTyExpTgt); }
284   bool isExpVM() const { return isImmTy(ImmTyExpVM); }
285   bool isExpCompr() const { return isImmTy(ImmTyExpCompr); }
286   bool isOffen() const { return isImmTy(ImmTyOffen); }
287   bool isIdxen() const { return isImmTy(ImmTyIdxen); }
288   bool isAddr64() const { return isImmTy(ImmTyAddr64); }
289   bool isOffset() const { return isImmTy(ImmTyOffset) && isUInt<16>(getImm()); }
290   bool isOffset0() const { return isImmTy(ImmTyOffset0) && isUInt<16>(getImm()); }
291   bool isOffset1() const { return isImmTy(ImmTyOffset1) && isUInt<8>(getImm()); }
292 
293   bool isOffsetU12() const { return isImmTy(ImmTyOffset) && isUInt<12>(getImm()); }
294   bool isOffsetS13() const { return isImmTy(ImmTyOffset) && isInt<13>(getImm()); }
295   bool isGDS() const { return isImmTy(ImmTyGDS); }
296   bool isGLC() const { return isImmTy(ImmTyGLC); }
297   bool isSLC() const { return isImmTy(ImmTySLC); }
298   bool isTFE() const { return isImmTy(ImmTyTFE); }
299   bool isDFMT() const { return isImmTy(ImmTyDFMT) && isUInt<8>(getImm()); }
300   bool isNFMT() const { return isImmTy(ImmTyNFMT) && isUInt<8>(getImm()); }
301   bool isBankMask() const { return isImmTy(ImmTyDppBankMask); }
302   bool isRowMask() const { return isImmTy(ImmTyDppRowMask); }
303   bool isBoundCtrl() const { return isImmTy(ImmTyDppBoundCtrl); }
304   bool isSDWADstSel() const { return isImmTy(ImmTySdwaDstSel); }
305   bool isSDWASrc0Sel() const { return isImmTy(ImmTySdwaSrc0Sel); }
306   bool isSDWASrc1Sel() const { return isImmTy(ImmTySdwaSrc1Sel); }
307   bool isSDWADstUnused() const { return isImmTy(ImmTySdwaDstUnused); }
308   bool isInterpSlot() const { return isImmTy(ImmTyInterpSlot); }
309   bool isInterpAttr() const { return isImmTy(ImmTyInterpAttr); }
310   bool isAttrChan() const { return isImmTy(ImmTyAttrChan); }
311   bool isOpSel() const { return isImmTy(ImmTyOpSel); }
312   bool isOpSelHi() const { return isImmTy(ImmTyOpSelHi); }
313   bool isNegLo() const { return isImmTy(ImmTyNegLo); }
314   bool isNegHi() const { return isImmTy(ImmTyNegHi); }
315 
316   bool isMod() const {
317     return isClampSI() || isOModSI();
318   }
319 
320   bool isRegOrImm() const {
321     return isReg() || isImm();
322   }
323 
324   bool isRegClass(unsigned RCID) const;
325 
326   bool isRegOrInlineNoMods(unsigned RCID, MVT type) const {
327     return (isRegClass(RCID) || isInlinableImm(type)) && !hasModifiers();
328   }
329 
330   bool isSCSrcB16() const {
331     return isRegOrInlineNoMods(AMDGPU::SReg_32RegClassID, MVT::i16);
332   }
333 
334   bool isSCSrcV2B16() const {
335     return isSCSrcB16();
336   }
337 
338   bool isSCSrcB32() const {
339     return isRegOrInlineNoMods(AMDGPU::SReg_32RegClassID, MVT::i32);
340   }
341 
342   bool isSCSrcB64() const {
343     return isRegOrInlineNoMods(AMDGPU::SReg_64RegClassID, MVT::i64);
344   }
345 
346   bool isSCSrcF16() const {
347     return isRegOrInlineNoMods(AMDGPU::SReg_32RegClassID, MVT::f16);
348   }
349 
350   bool isSCSrcV2F16() const {
351     return isSCSrcF16();
352   }
353 
354   bool isSCSrcF32() const {
355     return isRegOrInlineNoMods(AMDGPU::SReg_32RegClassID, MVT::f32);
356   }
357 
358   bool isSCSrcF64() const {
359     return isRegOrInlineNoMods(AMDGPU::SReg_64RegClassID, MVT::f64);
360   }
361 
362   bool isSSrcB32() const {
363     return isSCSrcB32() || isLiteralImm(MVT::i32) || isExpr();
364   }
365 
366   bool isSSrcB16() const {
367     return isSCSrcB16() || isLiteralImm(MVT::i16);
368   }
369 
370   bool isSSrcV2B16() const {
371     llvm_unreachable("cannot happen");
372     return isSSrcB16();
373   }
374 
375   bool isSSrcB64() const {
376     // TODO: Find out how SALU supports extension of 32-bit literals to 64 bits.
377     // See isVSrc64().
378     return isSCSrcB64() || isLiteralImm(MVT::i64);
379   }
380 
381   bool isSSrcF32() const {
382     return isSCSrcB32() || isLiteralImm(MVT::f32) || isExpr();
383   }
384 
385   bool isSSrcF64() const {
386     return isSCSrcB64() || isLiteralImm(MVT::f64);
387   }
388 
389   bool isSSrcF16() const {
390     return isSCSrcB16() || isLiteralImm(MVT::f16);
391   }
392 
393   bool isSSrcV2F16() const {
394     llvm_unreachable("cannot happen");
395     return isSSrcF16();
396   }
397 
398   bool isVCSrcB32() const {
399     return isRegOrInlineNoMods(AMDGPU::VS_32RegClassID, MVT::i32);
400   }
401 
402   bool isVCSrcB64() const {
403     return isRegOrInlineNoMods(AMDGPU::VS_64RegClassID, MVT::i64);
404   }
405 
406   bool isVCSrcB16() const {
407     return isRegOrInlineNoMods(AMDGPU::VS_32RegClassID, MVT::i16);
408   }
409 
410   bool isVCSrcV2B16() const {
411     return isVCSrcB16();
412   }
413 
414   bool isVCSrcF32() const {
415     return isRegOrInlineNoMods(AMDGPU::VS_32RegClassID, MVT::f32);
416   }
417 
418   bool isVCSrcF64() const {
419     return isRegOrInlineNoMods(AMDGPU::VS_64RegClassID, MVT::f64);
420   }
421 
422   bool isVCSrcF16() const {
423     return isRegOrInlineNoMods(AMDGPU::VS_32RegClassID, MVT::f16);
424   }
425 
426   bool isVCSrcV2F16() const {
427     return isVCSrcF16();
428   }
429 
430   bool isVSrcB32() const {
431     return isVCSrcF32() || isLiteralImm(MVT::i32);
432   }
433 
434   bool isVSrcB64() const {
435     return isVCSrcF64() || isLiteralImm(MVT::i64);
436   }
437 
438   bool isVSrcB16() const {
439     return isVCSrcF16() || isLiteralImm(MVT::i16);
440   }
441 
442   bool isVSrcV2B16() const {
443     llvm_unreachable("cannot happen");
444     return isVSrcB16();
445   }
446 
447   bool isVSrcF32() const {
448     return isVCSrcF32() || isLiteralImm(MVT::f32);
449   }
450 
451   bool isVSrcF64() const {
452     return isVCSrcF64() || isLiteralImm(MVT::f64);
453   }
454 
455   bool isVSrcF16() const {
456     return isVCSrcF16() || isLiteralImm(MVT::f16);
457   }
458 
459   bool isVSrcV2F16() const {
460     llvm_unreachable("cannot happen");
461     return isVSrcF16();
462   }
463 
464   bool isKImmFP32() const {
465     return isLiteralImm(MVT::f32);
466   }
467 
468   bool isKImmFP16() const {
469     return isLiteralImm(MVT::f16);
470   }
471 
472   bool isMem() const override {
473     return false;
474   }
475 
476   bool isExpr() const {
477     return Kind == Expression;
478   }
479 
480   bool isSoppBrTarget() const {
481     return isExpr() || isImm();
482   }
483 
484   bool isSWaitCnt() const;
485   bool isHwreg() const;
486   bool isSendMsg() const;
487   bool isSwizzle() const;
488   bool isSMRDOffset8() const;
489   bool isSMRDOffset20() const;
490   bool isSMRDLiteralOffset() const;
491   bool isDPPCtrl() const;
492   bool isGPRIdxMode() const;
493   bool isS16Imm() const;
494   bool isU16Imm() const;
495 
496   StringRef getExpressionAsToken() const {
497     assert(isExpr());
498     const MCSymbolRefExpr *S = cast<MCSymbolRefExpr>(Expr);
499     return S->getSymbol().getName();
500   }
501 
502   StringRef getToken() const {
503     assert(isToken());
504 
505     if (Kind == Expression)
506       return getExpressionAsToken();
507 
508     return StringRef(Tok.Data, Tok.Length);
509   }
510 
511   int64_t getImm() const {
512     assert(isImm());
513     return Imm.Val;
514   }
515 
516   ImmTy getImmTy() const {
517     assert(isImm());
518     return Imm.Type;
519   }
520 
521   unsigned getReg() const override {
522     return Reg.RegNo;
523   }
524 
525   SMLoc getStartLoc() const override {
526     return StartLoc;
527   }
528 
529   SMLoc getEndLoc() const override {
530     return EndLoc;
531   }
532 
533   Modifiers getModifiers() const {
534     assert(isRegKind() || isImmTy(ImmTyNone));
535     return isRegKind() ? Reg.Mods : Imm.Mods;
536   }
537 
538   void setModifiers(Modifiers Mods) {
539     assert(isRegKind() || isImmTy(ImmTyNone));
540     if (isRegKind())
541       Reg.Mods = Mods;
542     else
543       Imm.Mods = Mods;
544   }
545 
546   bool hasModifiers() const {
547     return getModifiers().hasModifiers();
548   }
549 
550   bool hasFPModifiers() const {
551     return getModifiers().hasFPModifiers();
552   }
553 
554   bool hasIntModifiers() const {
555     return getModifiers().hasIntModifiers();
556   }
557 
558   uint64_t applyInputFPModifiers(uint64_t Val, unsigned Size) const;
559 
560   void addImmOperands(MCInst &Inst, unsigned N, bool ApplyModifiers = true) const;
561 
562   void addLiteralImmOperand(MCInst &Inst, int64_t Val, bool ApplyModifiers) const;
563 
564   template <unsigned Bitwidth>
565   void addKImmFPOperands(MCInst &Inst, unsigned N) const;
566 
567   void addKImmFP16Operands(MCInst &Inst, unsigned N) const {
568     addKImmFPOperands<16>(Inst, N);
569   }
570 
571   void addKImmFP32Operands(MCInst &Inst, unsigned N) const {
572     addKImmFPOperands<32>(Inst, N);
573   }
574 
575   void addRegOperands(MCInst &Inst, unsigned N) const;
576 
577   void addRegOrImmOperands(MCInst &Inst, unsigned N) const {
578     if (isRegKind())
579       addRegOperands(Inst, N);
580     else if (isExpr())
581       Inst.addOperand(MCOperand::createExpr(Expr));
582     else
583       addImmOperands(Inst, N);
584   }
585 
586   void addRegOrImmWithInputModsOperands(MCInst &Inst, unsigned N) const {
587     Modifiers Mods = getModifiers();
588     Inst.addOperand(MCOperand::createImm(Mods.getModifiersOperand()));
589     if (isRegKind()) {
590       addRegOperands(Inst, N);
591     } else {
592       addImmOperands(Inst, N, false);
593     }
594   }
595 
596   void addRegOrImmWithFPInputModsOperands(MCInst &Inst, unsigned N) const {
597     assert(!hasIntModifiers());
598     addRegOrImmWithInputModsOperands(Inst, N);
599   }
600 
601   void addRegOrImmWithIntInputModsOperands(MCInst &Inst, unsigned N) const {
602     assert(!hasFPModifiers());
603     addRegOrImmWithInputModsOperands(Inst, N);
604   }
605 
606   void addRegWithInputModsOperands(MCInst &Inst, unsigned N) const {
607     Modifiers Mods = getModifiers();
608     Inst.addOperand(MCOperand::createImm(Mods.getModifiersOperand()));
609     assert(isRegKind());
610     addRegOperands(Inst, N);
611   }
612 
613   void addRegWithFPInputModsOperands(MCInst &Inst, unsigned N) const {
614     assert(!hasIntModifiers());
615     addRegWithInputModsOperands(Inst, N);
616   }
617 
618   void addRegWithIntInputModsOperands(MCInst &Inst, unsigned N) const {
619     assert(!hasFPModifiers());
620     addRegWithInputModsOperands(Inst, N);
621   }
622 
623   void addSoppBrTargetOperands(MCInst &Inst, unsigned N) const {
624     if (isImm())
625       addImmOperands(Inst, N);
626     else {
627       assert(isExpr());
628       Inst.addOperand(MCOperand::createExpr(Expr));
629     }
630   }
631 
632   static void printImmTy(raw_ostream& OS, ImmTy Type) {
633     switch (Type) {
634     case ImmTyNone: OS << "None"; break;
635     case ImmTyGDS: OS << "GDS"; break;
636     case ImmTyOffen: OS << "Offen"; break;
637     case ImmTyIdxen: OS << "Idxen"; break;
638     case ImmTyAddr64: OS << "Addr64"; break;
639     case ImmTyOffset: OS << "Offset"; break;
640     case ImmTyOffset0: OS << "Offset0"; break;
641     case ImmTyOffset1: OS << "Offset1"; break;
642     case ImmTyGLC: OS << "GLC"; break;
643     case ImmTySLC: OS << "SLC"; break;
644     case ImmTyTFE: OS << "TFE"; break;
645     case ImmTyDFMT: OS << "DFMT"; break;
646     case ImmTyNFMT: OS << "NFMT"; break;
647     case ImmTyClampSI: OS << "ClampSI"; break;
648     case ImmTyOModSI: OS << "OModSI"; break;
649     case ImmTyDppCtrl: OS << "DppCtrl"; break;
650     case ImmTyDppRowMask: OS << "DppRowMask"; break;
651     case ImmTyDppBankMask: OS << "DppBankMask"; break;
652     case ImmTyDppBoundCtrl: OS << "DppBoundCtrl"; break;
653     case ImmTySdwaDstSel: OS << "SdwaDstSel"; break;
654     case ImmTySdwaSrc0Sel: OS << "SdwaSrc0Sel"; break;
655     case ImmTySdwaSrc1Sel: OS << "SdwaSrc1Sel"; break;
656     case ImmTySdwaDstUnused: OS << "SdwaDstUnused"; break;
657     case ImmTyDMask: OS << "DMask"; break;
658     case ImmTyUNorm: OS << "UNorm"; break;
659     case ImmTyDA: OS << "DA"; break;
660     case ImmTyR128: OS << "R128"; break;
661     case ImmTyLWE: OS << "LWE"; break;
662     case ImmTyOff: OS << "Off"; break;
663     case ImmTyExpTgt: OS << "ExpTgt"; break;
664     case ImmTyExpCompr: OS << "ExpCompr"; break;
665     case ImmTyExpVM: OS << "ExpVM"; break;
666     case ImmTyHwreg: OS << "Hwreg"; break;
667     case ImmTySendMsg: OS << "SendMsg"; break;
668     case ImmTyInterpSlot: OS << "InterpSlot"; break;
669     case ImmTyInterpAttr: OS << "InterpAttr"; break;
670     case ImmTyAttrChan: OS << "AttrChan"; break;
671     case ImmTyOpSel: OS << "OpSel"; break;
672     case ImmTyOpSelHi: OS << "OpSelHi"; break;
673     case ImmTyNegLo: OS << "NegLo"; break;
674     case ImmTyNegHi: OS << "NegHi"; break;
675     case ImmTySwizzle: OS << "Swizzle"; break;
676     }
677   }
678 
679   void print(raw_ostream &OS) const override {
680     switch (Kind) {
681     case Register:
682       OS << "<register " << getReg() << " mods: " << Reg.Mods << '>';
683       break;
684     case Immediate:
685       OS << '<' << getImm();
686       if (getImmTy() != ImmTyNone) {
687         OS << " type: "; printImmTy(OS, getImmTy());
688       }
689       OS << " mods: " << Imm.Mods << '>';
690       break;
691     case Token:
692       OS << '\'' << getToken() << '\'';
693       break;
694     case Expression:
695       OS << "<expr " << *Expr << '>';
696       break;
697     }
698   }
699 
700   static AMDGPUOperand::Ptr CreateImm(const AMDGPUAsmParser *AsmParser,
701                                       int64_t Val, SMLoc Loc,
702                                       ImmTy Type = ImmTyNone,
703                                       bool IsFPImm = false) {
704     auto Op = llvm::make_unique<AMDGPUOperand>(Immediate, AsmParser);
705     Op->Imm.Val = Val;
706     Op->Imm.IsFPImm = IsFPImm;
707     Op->Imm.Type = Type;
708     Op->Imm.Mods = Modifiers();
709     Op->StartLoc = Loc;
710     Op->EndLoc = Loc;
711     return Op;
712   }
713 
714   static AMDGPUOperand::Ptr CreateToken(const AMDGPUAsmParser *AsmParser,
715                                         StringRef Str, SMLoc Loc,
716                                         bool HasExplicitEncodingSize = true) {
717     auto Res = llvm::make_unique<AMDGPUOperand>(Token, AsmParser);
718     Res->Tok.Data = Str.data();
719     Res->Tok.Length = Str.size();
720     Res->StartLoc = Loc;
721     Res->EndLoc = Loc;
722     return Res;
723   }
724 
725   static AMDGPUOperand::Ptr CreateReg(const AMDGPUAsmParser *AsmParser,
726                                       unsigned RegNo, SMLoc S,
727                                       SMLoc E,
728                                       bool ForceVOP3) {
729     auto Op = llvm::make_unique<AMDGPUOperand>(Register, AsmParser);
730     Op->Reg.RegNo = RegNo;
731     Op->Reg.Mods = Modifiers();
732     Op->Reg.IsForcedVOP3 = ForceVOP3;
733     Op->StartLoc = S;
734     Op->EndLoc = E;
735     return Op;
736   }
737 
738   static AMDGPUOperand::Ptr CreateExpr(const AMDGPUAsmParser *AsmParser,
739                                        const class MCExpr *Expr, SMLoc S) {
740     auto Op = llvm::make_unique<AMDGPUOperand>(Expression, AsmParser);
741     Op->Expr = Expr;
742     Op->StartLoc = S;
743     Op->EndLoc = S;
744     return Op;
745   }
746 };
747 
748 raw_ostream &operator <<(raw_ostream &OS, AMDGPUOperand::Modifiers Mods) {
749   OS << "abs:" << Mods.Abs << " neg: " << Mods.Neg << " sext:" << Mods.Sext;
750   return OS;
751 }
752 
753 //===----------------------------------------------------------------------===//
754 // AsmParser
755 //===----------------------------------------------------------------------===//
756 
757 // Holds info related to the current kernel, e.g. count of SGPRs used.
758 // Kernel scope begins at .amdgpu_hsa_kernel directive, ends at next
759 // .amdgpu_hsa_kernel or at EOF.
760 class KernelScopeInfo {
761   int SgprIndexUnusedMin = -1;
762   int VgprIndexUnusedMin = -1;
763   MCContext *Ctx = nullptr;
764 
765   void usesSgprAt(int i) {
766     if (i >= SgprIndexUnusedMin) {
767       SgprIndexUnusedMin = ++i;
768       if (Ctx) {
769         MCSymbol * const Sym = Ctx->getOrCreateSymbol(Twine(".kernel.sgpr_count"));
770         Sym->setVariableValue(MCConstantExpr::create(SgprIndexUnusedMin, *Ctx));
771       }
772     }
773   }
774 
775   void usesVgprAt(int i) {
776     if (i >= VgprIndexUnusedMin) {
777       VgprIndexUnusedMin = ++i;
778       if (Ctx) {
779         MCSymbol * const Sym = Ctx->getOrCreateSymbol(Twine(".kernel.vgpr_count"));
780         Sym->setVariableValue(MCConstantExpr::create(VgprIndexUnusedMin, *Ctx));
781       }
782     }
783   }
784 
785 public:
786   KernelScopeInfo() = default;
787 
788   void initialize(MCContext &Context) {
789     Ctx = &Context;
790     usesSgprAt(SgprIndexUnusedMin = -1);
791     usesVgprAt(VgprIndexUnusedMin = -1);
792   }
793 
794   void usesRegister(RegisterKind RegKind, unsigned DwordRegIndex, unsigned RegWidth) {
795     switch (RegKind) {
796       case IS_SGPR: usesSgprAt(DwordRegIndex + RegWidth - 1); break;
797       case IS_VGPR: usesVgprAt(DwordRegIndex + RegWidth - 1); break;
798       default: break;
799     }
800   }
801 };
802 
803 class AMDGPUAsmParser : public MCTargetAsmParser {
804   const MCInstrInfo &MII;
805   MCAsmParser &Parser;
806 
807   unsigned ForcedEncodingSize = 0;
808   bool ForcedDPP = false;
809   bool ForcedSDWA = false;
810   KernelScopeInfo KernelScope;
811 
812   /// @name Auto-generated Match Functions
813   /// {
814 
815 #define GET_ASSEMBLER_HEADER
816 #include "AMDGPUGenAsmMatcher.inc"
817 
818   /// }
819 
820 private:
821   bool ParseAsAbsoluteExpression(uint32_t &Ret);
822   bool ParseDirectiveMajorMinor(uint32_t &Major, uint32_t &Minor);
823   bool ParseDirectiveHSACodeObjectVersion();
824   bool ParseDirectiveHSACodeObjectISA();
825   bool ParseDirectiveCodeObjectMetadata();
826   bool ParseAMDKernelCodeTValue(StringRef ID, amd_kernel_code_t &Header);
827   bool ParseDirectiveAMDKernelCodeT();
828   bool subtargetHasRegister(const MCRegisterInfo &MRI, unsigned RegNo) const;
829   bool ParseDirectiveAMDGPUHsaKernel();
830   bool AddNextRegisterToList(unsigned& Reg, unsigned& RegWidth,
831                              RegisterKind RegKind, unsigned Reg1,
832                              unsigned RegNum);
833   bool ParseAMDGPURegister(RegisterKind& RegKind, unsigned& Reg,
834                            unsigned& RegNum, unsigned& RegWidth,
835                            unsigned *DwordRegIndex);
836   void cvtMubufImpl(MCInst &Inst, const OperandVector &Operands,
837                     bool IsAtomic, bool IsAtomicReturn);
838   void cvtDSImpl(MCInst &Inst, const OperandVector &Operands,
839                  bool IsGdsHardcoded);
840 
841 public:
842   enum AMDGPUMatchResultTy {
843     Match_PreferE32 = FIRST_TARGET_MATCH_RESULT_TY
844   };
845 
846   typedef std::map<AMDGPUOperand::ImmTy, unsigned> OptionalImmIndexMap;
847 
848   AMDGPUAsmParser(const MCSubtargetInfo &STI, MCAsmParser &_Parser,
849                const MCInstrInfo &MII,
850                const MCTargetOptions &Options)
851       : MCTargetAsmParser(Options, STI), MII(MII), Parser(_Parser) {
852     MCAsmParserExtension::Initialize(Parser);
853 
854     if (getFeatureBits().none()) {
855       // Set default features.
856       copySTI().ToggleFeature("SOUTHERN_ISLANDS");
857     }
858 
859     setAvailableFeatures(ComputeAvailableFeatures(getFeatureBits()));
860 
861     {
862       // TODO: make those pre-defined variables read-only.
863       // Currently there is none suitable machinery in the core llvm-mc for this.
864       // MCSymbol::isRedefinable is intended for another purpose, and
865       // AsmParser::parseDirectiveSet() cannot be specialized for specific target.
866       AMDGPU::IsaInfo::IsaVersion ISA =
867           AMDGPU::IsaInfo::getIsaVersion(getFeatureBits());
868       MCContext &Ctx = getContext();
869       MCSymbol *Sym =
870           Ctx.getOrCreateSymbol(Twine(".option.machine_version_major"));
871       Sym->setVariableValue(MCConstantExpr::create(ISA.Major, Ctx));
872       Sym = Ctx.getOrCreateSymbol(Twine(".option.machine_version_minor"));
873       Sym->setVariableValue(MCConstantExpr::create(ISA.Minor, Ctx));
874       Sym = Ctx.getOrCreateSymbol(Twine(".option.machine_version_stepping"));
875       Sym->setVariableValue(MCConstantExpr::create(ISA.Stepping, Ctx));
876     }
877     KernelScope.initialize(getContext());
878   }
879 
880   bool isSI() const {
881     return AMDGPU::isSI(getSTI());
882   }
883 
884   bool isCI() const {
885     return AMDGPU::isCI(getSTI());
886   }
887 
888   bool isVI() const {
889     return AMDGPU::isVI(getSTI());
890   }
891 
892   bool isGFX9() const {
893     return AMDGPU::isGFX9(getSTI());
894   }
895 
896   bool hasInv2PiInlineImm() const {
897     return getFeatureBits()[AMDGPU::FeatureInv2PiInlineImm];
898   }
899 
900   bool hasFlatOffsets() const {
901     return getFeatureBits()[AMDGPU::FeatureFlatInstOffsets];
902   }
903 
904   bool hasSGPR102_SGPR103() const {
905     return !isVI();
906   }
907 
908   AMDGPUTargetStreamer &getTargetStreamer() {
909     MCTargetStreamer &TS = *getParser().getStreamer().getTargetStreamer();
910     return static_cast<AMDGPUTargetStreamer &>(TS);
911   }
912 
913   const MCRegisterInfo *getMRI() const {
914     // We need this const_cast because for some reason getContext() is not const
915     // in MCAsmParser.
916     return const_cast<AMDGPUAsmParser*>(this)->getContext().getRegisterInfo();
917   }
918 
919   const MCInstrInfo *getMII() const {
920     return &MII;
921   }
922 
923   const FeatureBitset &getFeatureBits() const {
924     return getSTI().getFeatureBits();
925   }
926 
927   void setForcedEncodingSize(unsigned Size) { ForcedEncodingSize = Size; }
928   void setForcedDPP(bool ForceDPP_) { ForcedDPP = ForceDPP_; }
929   void setForcedSDWA(bool ForceSDWA_) { ForcedSDWA = ForceSDWA_; }
930 
931   unsigned getForcedEncodingSize() const { return ForcedEncodingSize; }
932   bool isForcedVOP3() const { return ForcedEncodingSize == 64; }
933   bool isForcedDPP() const { return ForcedDPP; }
934   bool isForcedSDWA() const { return ForcedSDWA; }
935   ArrayRef<unsigned> getMatchedVariants() const;
936 
937   std::unique_ptr<AMDGPUOperand> parseRegister();
938   bool ParseRegister(unsigned &RegNo, SMLoc &StartLoc, SMLoc &EndLoc) override;
939   unsigned checkTargetMatchPredicate(MCInst &Inst) override;
940   unsigned validateTargetOperandClass(MCParsedAsmOperand &Op,
941                                       unsigned Kind) override;
942   bool MatchAndEmitInstruction(SMLoc IDLoc, unsigned &Opcode,
943                                OperandVector &Operands, MCStreamer &Out,
944                                uint64_t &ErrorInfo,
945                                bool MatchingInlineAsm) override;
946   bool ParseDirective(AsmToken DirectiveID) override;
947   OperandMatchResultTy parseOperand(OperandVector &Operands, StringRef Mnemonic);
948   StringRef parseMnemonicSuffix(StringRef Name);
949   bool ParseInstruction(ParseInstructionInfo &Info, StringRef Name,
950                         SMLoc NameLoc, OperandVector &Operands) override;
951   //bool ProcessInstruction(MCInst &Inst);
952 
953   OperandMatchResultTy parseIntWithPrefix(const char *Prefix, int64_t &Int);
954 
955   OperandMatchResultTy
956   parseIntWithPrefix(const char *Prefix, OperandVector &Operands,
957                      AMDGPUOperand::ImmTy ImmTy = AMDGPUOperand::ImmTyNone,
958                      bool (*ConvertResult)(int64_t &) = nullptr);
959 
960   OperandMatchResultTy parseOperandArrayWithPrefix(
961     const char *Prefix,
962     OperandVector &Operands,
963     AMDGPUOperand::ImmTy ImmTy = AMDGPUOperand::ImmTyNone,
964     bool (*ConvertResult)(int64_t&) = nullptr);
965 
966   OperandMatchResultTy
967   parseNamedBit(const char *Name, OperandVector &Operands,
968                 AMDGPUOperand::ImmTy ImmTy = AMDGPUOperand::ImmTyNone);
969   OperandMatchResultTy parseStringWithPrefix(StringRef Prefix,
970                                              StringRef &Value);
971 
972   bool parseAbsoluteExpr(int64_t &Val, bool AbsMod = false);
973   OperandMatchResultTy parseImm(OperandVector &Operands, bool AbsMod = false);
974   OperandMatchResultTy parseReg(OperandVector &Operands);
975   OperandMatchResultTy parseRegOrImm(OperandVector &Operands, bool AbsMod = false);
976   OperandMatchResultTy parseRegOrImmWithFPInputMods(OperandVector &Operands, bool AllowImm = true);
977   OperandMatchResultTy parseRegOrImmWithIntInputMods(OperandVector &Operands, bool AllowImm = true);
978   OperandMatchResultTy parseRegWithFPInputMods(OperandVector &Operands);
979   OperandMatchResultTy parseRegWithIntInputMods(OperandVector &Operands);
980   OperandMatchResultTy parseVReg32OrOff(OperandVector &Operands);
981 
982   void cvtDSOffset01(MCInst &Inst, const OperandVector &Operands);
983   void cvtDS(MCInst &Inst, const OperandVector &Operands) { cvtDSImpl(Inst, Operands, false); }
984   void cvtDSGds(MCInst &Inst, const OperandVector &Operands) { cvtDSImpl(Inst, Operands, true); }
985   void cvtExp(MCInst &Inst, const OperandVector &Operands);
986 
987   bool parseCnt(int64_t &IntVal);
988   OperandMatchResultTy parseSWaitCntOps(OperandVector &Operands);
989   OperandMatchResultTy parseHwreg(OperandVector &Operands);
990 
991 private:
992   struct OperandInfoTy {
993     int64_t Id;
994     bool IsSymbolic;
995     OperandInfoTy(int64_t Id_) : Id(Id_), IsSymbolic(false) { }
996   };
997 
998   bool parseSendMsgConstruct(OperandInfoTy &Msg, OperandInfoTy &Operation, int64_t &StreamId);
999   bool parseHwregConstruct(OperandInfoTy &HwReg, int64_t &Offset, int64_t &Width);
1000 
1001   void errorExpTgt();
1002   OperandMatchResultTy parseExpTgtImpl(StringRef Str, uint8_t &Val);
1003 
1004   bool validateInstruction(const MCInst &Inst, const SMLoc &IDLoc);
1005   bool validateConstantBusLimitations(const MCInst &Inst);
1006   bool validateEarlyClobberLimitations(const MCInst &Inst);
1007   bool usesConstantBus(const MCInst &Inst, unsigned OpIdx);
1008   bool isInlineConstant(const MCInst &Inst, unsigned OpIdx) const;
1009   unsigned findImplicitSGPRReadInVOP(const MCInst &Inst) const;
1010 
1011   bool trySkipId(const StringRef Id);
1012   bool trySkipToken(const AsmToken::TokenKind Kind);
1013   bool skipToken(const AsmToken::TokenKind Kind, const StringRef ErrMsg);
1014   bool parseString(StringRef &Val, const StringRef ErrMsg = "expected a string");
1015   bool parseExpr(int64_t &Imm);
1016 
1017 public:
1018   OperandMatchResultTy parseOptionalOperand(OperandVector &Operands);
1019 
1020   OperandMatchResultTy parseExpTgt(OperandVector &Operands);
1021   OperandMatchResultTy parseSendMsgOp(OperandVector &Operands);
1022   OperandMatchResultTy parseInterpSlot(OperandVector &Operands);
1023   OperandMatchResultTy parseInterpAttr(OperandVector &Operands);
1024   OperandMatchResultTy parseSOppBrTarget(OperandVector &Operands);
1025 
1026   bool parseSwizzleOperands(const unsigned OpNum, int64_t* Op,
1027                             const unsigned MinVal,
1028                             const unsigned MaxVal,
1029                             const StringRef ErrMsg);
1030   OperandMatchResultTy parseSwizzleOp(OperandVector &Operands);
1031   bool parseSwizzleOffset(int64_t &Imm);
1032   bool parseSwizzleMacro(int64_t &Imm);
1033   bool parseSwizzleQuadPerm(int64_t &Imm);
1034   bool parseSwizzleBitmaskPerm(int64_t &Imm);
1035   bool parseSwizzleBroadcast(int64_t &Imm);
1036   bool parseSwizzleSwap(int64_t &Imm);
1037   bool parseSwizzleReverse(int64_t &Imm);
1038 
1039   void cvtMubuf(MCInst &Inst, const OperandVector &Operands) { cvtMubufImpl(Inst, Operands, false, false); }
1040   void cvtMubufAtomic(MCInst &Inst, const OperandVector &Operands) { cvtMubufImpl(Inst, Operands, true, false); }
1041   void cvtMubufAtomicReturn(MCInst &Inst, const OperandVector &Operands) { cvtMubufImpl(Inst, Operands, true, true); }
1042   void cvtMtbuf(MCInst &Inst, const OperandVector &Operands);
1043 
1044   AMDGPUOperand::Ptr defaultGLC() const;
1045   AMDGPUOperand::Ptr defaultSLC() const;
1046   AMDGPUOperand::Ptr defaultTFE() const;
1047 
1048   AMDGPUOperand::Ptr defaultDMask() const;
1049   AMDGPUOperand::Ptr defaultUNorm() const;
1050   AMDGPUOperand::Ptr defaultDA() const;
1051   AMDGPUOperand::Ptr defaultR128() const;
1052   AMDGPUOperand::Ptr defaultLWE() const;
1053   AMDGPUOperand::Ptr defaultSMRDOffset8() const;
1054   AMDGPUOperand::Ptr defaultSMRDOffset20() const;
1055   AMDGPUOperand::Ptr defaultSMRDLiteralOffset() const;
1056   AMDGPUOperand::Ptr defaultOffsetU12() const;
1057   AMDGPUOperand::Ptr defaultOffsetS13() const;
1058 
1059   OperandMatchResultTy parseOModOperand(OperandVector &Operands);
1060 
1061   void cvtVOP3(MCInst &Inst, const OperandVector &Operands,
1062                OptionalImmIndexMap &OptionalIdx);
1063   void cvtVOP3OpSel(MCInst &Inst, const OperandVector &Operands);
1064   void cvtVOP3(MCInst &Inst, const OperandVector &Operands);
1065   void cvtVOP3P(MCInst &Inst, const OperandVector &Operands);
1066 
1067   void cvtMIMG(MCInst &Inst, const OperandVector &Operands,
1068                bool IsAtomic = false);
1069   void cvtMIMGAtomic(MCInst &Inst, const OperandVector &Operands);
1070 
1071   OperandMatchResultTy parseDPPCtrl(OperandVector &Operands);
1072   AMDGPUOperand::Ptr defaultRowMask() const;
1073   AMDGPUOperand::Ptr defaultBankMask() const;
1074   AMDGPUOperand::Ptr defaultBoundCtrl() const;
1075   void cvtDPP(MCInst &Inst, const OperandVector &Operands);
1076 
1077   OperandMatchResultTy parseSDWASel(OperandVector &Operands, StringRef Prefix,
1078                                     AMDGPUOperand::ImmTy Type);
1079   OperandMatchResultTy parseSDWADstUnused(OperandVector &Operands);
1080   void cvtSdwaVOP1(MCInst &Inst, const OperandVector &Operands);
1081   void cvtSdwaVOP2(MCInst &Inst, const OperandVector &Operands);
1082   void cvtSdwaVOP2b(MCInst &Inst, const OperandVector &Operands);
1083   void cvtSdwaVOPC(MCInst &Inst, const OperandVector &Operands);
1084   void cvtSDWA(MCInst &Inst, const OperandVector &Operands,
1085                 uint64_t BasicInstType, bool skipVcc = false);
1086 };
1087 
1088 struct OptionalOperand {
1089   const char *Name;
1090   AMDGPUOperand::ImmTy Type;
1091   bool IsBit;
1092   bool (*ConvertResult)(int64_t&);
1093 };
1094 
1095 } // end anonymous namespace
1096 
1097 // May be called with integer type with equivalent bitwidth.
1098 static const fltSemantics *getFltSemantics(unsigned Size) {
1099   switch (Size) {
1100   case 4:
1101     return &APFloat::IEEEsingle();
1102   case 8:
1103     return &APFloat::IEEEdouble();
1104   case 2:
1105     return &APFloat::IEEEhalf();
1106   default:
1107     llvm_unreachable("unsupported fp type");
1108   }
1109 }
1110 
1111 static const fltSemantics *getFltSemantics(MVT VT) {
1112   return getFltSemantics(VT.getSizeInBits() / 8);
1113 }
1114 
1115 static const fltSemantics *getOpFltSemantics(uint8_t OperandType) {
1116   switch (OperandType) {
1117   case AMDGPU::OPERAND_REG_IMM_INT32:
1118   case AMDGPU::OPERAND_REG_IMM_FP32:
1119   case AMDGPU::OPERAND_REG_INLINE_C_INT32:
1120   case AMDGPU::OPERAND_REG_INLINE_C_FP32:
1121     return &APFloat::IEEEsingle();
1122   case AMDGPU::OPERAND_REG_IMM_INT64:
1123   case AMDGPU::OPERAND_REG_IMM_FP64:
1124   case AMDGPU::OPERAND_REG_INLINE_C_INT64:
1125   case AMDGPU::OPERAND_REG_INLINE_C_FP64:
1126     return &APFloat::IEEEdouble();
1127   case AMDGPU::OPERAND_REG_IMM_INT16:
1128   case AMDGPU::OPERAND_REG_IMM_FP16:
1129   case AMDGPU::OPERAND_REG_INLINE_C_INT16:
1130   case AMDGPU::OPERAND_REG_INLINE_C_FP16:
1131   case AMDGPU::OPERAND_REG_INLINE_C_V2INT16:
1132   case AMDGPU::OPERAND_REG_INLINE_C_V2FP16:
1133     return &APFloat::IEEEhalf();
1134   default:
1135     llvm_unreachable("unsupported fp type");
1136   }
1137 }
1138 
1139 //===----------------------------------------------------------------------===//
1140 // Operand
1141 //===----------------------------------------------------------------------===//
1142 
1143 static bool canLosslesslyConvertToFPType(APFloat &FPLiteral, MVT VT) {
1144   bool Lost;
1145 
1146   // Convert literal to single precision
1147   APFloat::opStatus Status = FPLiteral.convert(*getFltSemantics(VT),
1148                                                APFloat::rmNearestTiesToEven,
1149                                                &Lost);
1150   // We allow precision lost but not overflow or underflow
1151   if (Status != APFloat::opOK &&
1152       Lost &&
1153       ((Status & APFloat::opOverflow)  != 0 ||
1154        (Status & APFloat::opUnderflow) != 0)) {
1155     return false;
1156   }
1157 
1158   return true;
1159 }
1160 
1161 bool AMDGPUOperand::isInlinableImm(MVT type) const {
1162   if (!isImmTy(ImmTyNone)) {
1163     // Only plain immediates are inlinable (e.g. "clamp" attribute is not)
1164     return false;
1165   }
1166   // TODO: We should avoid using host float here. It would be better to
1167   // check the float bit values which is what a few other places do.
1168   // We've had bot failures before due to weird NaN support on mips hosts.
1169 
1170   APInt Literal(64, Imm.Val);
1171 
1172   if (Imm.IsFPImm) { // We got fp literal token
1173     if (type == MVT::f64 || type == MVT::i64) { // Expected 64-bit operand
1174       return AMDGPU::isInlinableLiteral64(Imm.Val,
1175                                           AsmParser->hasInv2PiInlineImm());
1176     }
1177 
1178     APFloat FPLiteral(APFloat::IEEEdouble(), APInt(64, Imm.Val));
1179     if (!canLosslesslyConvertToFPType(FPLiteral, type))
1180       return false;
1181 
1182     if (type.getScalarSizeInBits() == 16) {
1183       return AMDGPU::isInlinableLiteral16(
1184         static_cast<int16_t>(FPLiteral.bitcastToAPInt().getZExtValue()),
1185         AsmParser->hasInv2PiInlineImm());
1186     }
1187 
1188     // Check if single precision literal is inlinable
1189     return AMDGPU::isInlinableLiteral32(
1190       static_cast<int32_t>(FPLiteral.bitcastToAPInt().getZExtValue()),
1191       AsmParser->hasInv2PiInlineImm());
1192   }
1193 
1194   // We got int literal token.
1195   if (type == MVT::f64 || type == MVT::i64) { // Expected 64-bit operand
1196     return AMDGPU::isInlinableLiteral64(Imm.Val,
1197                                         AsmParser->hasInv2PiInlineImm());
1198   }
1199 
1200   if (type.getScalarSizeInBits() == 16) {
1201     return AMDGPU::isInlinableLiteral16(
1202       static_cast<int16_t>(Literal.getLoBits(16).getSExtValue()),
1203       AsmParser->hasInv2PiInlineImm());
1204   }
1205 
1206   return AMDGPU::isInlinableLiteral32(
1207     static_cast<int32_t>(Literal.getLoBits(32).getZExtValue()),
1208     AsmParser->hasInv2PiInlineImm());
1209 }
1210 
1211 bool AMDGPUOperand::isLiteralImm(MVT type) const {
1212   // Check that this immediate can be added as literal
1213   if (!isImmTy(ImmTyNone)) {
1214     return false;
1215   }
1216 
1217   if (!Imm.IsFPImm) {
1218     // We got int literal token.
1219 
1220     if (type == MVT::f64 && hasFPModifiers()) {
1221       // Cannot apply fp modifiers to int literals preserving the same semantics
1222       // for VOP1/2/C and VOP3 because of integer truncation. To avoid ambiguity,
1223       // disable these cases.
1224       return false;
1225     }
1226 
1227     unsigned Size = type.getSizeInBits();
1228     if (Size == 64)
1229       Size = 32;
1230 
1231     // FIXME: 64-bit operands can zero extend, sign extend, or pad zeroes for FP
1232     // types.
1233     return isUIntN(Size, Imm.Val) || isIntN(Size, Imm.Val);
1234   }
1235 
1236   // We got fp literal token
1237   if (type == MVT::f64) { // Expected 64-bit fp operand
1238     // We would set low 64-bits of literal to zeroes but we accept this literals
1239     return true;
1240   }
1241 
1242   if (type == MVT::i64) { // Expected 64-bit int operand
1243     // We don't allow fp literals in 64-bit integer instructions. It is
1244     // unclear how we should encode them.
1245     return false;
1246   }
1247 
1248   APFloat FPLiteral(APFloat::IEEEdouble(), APInt(64, Imm.Val));
1249   return canLosslesslyConvertToFPType(FPLiteral, type);
1250 }
1251 
1252 bool AMDGPUOperand::isRegClass(unsigned RCID) const {
1253   return isRegKind() && AsmParser->getMRI()->getRegClass(RCID).contains(getReg());
1254 }
1255 
1256 bool AMDGPUOperand::isSDWARegKind() const {
1257   if (AsmParser->isVI())
1258     return isVReg();
1259   else if (AsmParser->isGFX9())
1260     return isRegKind();
1261   else
1262     return false;
1263 }
1264 
1265 uint64_t AMDGPUOperand::applyInputFPModifiers(uint64_t Val, unsigned Size) const
1266 {
1267   assert(isImmTy(ImmTyNone) && Imm.Mods.hasFPModifiers());
1268   assert(Size == 2 || Size == 4 || Size == 8);
1269 
1270   const uint64_t FpSignMask = (1ULL << (Size * 8 - 1));
1271 
1272   if (Imm.Mods.Abs) {
1273     Val &= ~FpSignMask;
1274   }
1275   if (Imm.Mods.Neg) {
1276     Val ^= FpSignMask;
1277   }
1278 
1279   return Val;
1280 }
1281 
1282 void AMDGPUOperand::addImmOperands(MCInst &Inst, unsigned N, bool ApplyModifiers) const {
1283 
1284   if (AMDGPU::isSISrcOperand(AsmParser->getMII()->get(Inst.getOpcode()),
1285                              Inst.getNumOperands())) {
1286     addLiteralImmOperand(Inst, Imm.Val,
1287                          ApplyModifiers &
1288                          isImmTy(ImmTyNone) && Imm.Mods.hasFPModifiers());
1289   } else {
1290     assert(!isImmTy(ImmTyNone) || !hasModifiers());
1291     Inst.addOperand(MCOperand::createImm(Imm.Val));
1292   }
1293 }
1294 
1295 void AMDGPUOperand::addLiteralImmOperand(MCInst &Inst, int64_t Val, bool ApplyModifiers) const {
1296   const auto& InstDesc = AsmParser->getMII()->get(Inst.getOpcode());
1297   auto OpNum = Inst.getNumOperands();
1298   // Check that this operand accepts literals
1299   assert(AMDGPU::isSISrcOperand(InstDesc, OpNum));
1300 
1301   if (ApplyModifiers) {
1302     assert(AMDGPU::isSISrcFPOperand(InstDesc, OpNum));
1303     const unsigned Size = Imm.IsFPImm ? sizeof(double) : getOperandSize(InstDesc, OpNum);
1304     Val = applyInputFPModifiers(Val, Size);
1305   }
1306 
1307   APInt Literal(64, Val);
1308   uint8_t OpTy = InstDesc.OpInfo[OpNum].OperandType;
1309 
1310   if (Imm.IsFPImm) { // We got fp literal token
1311     switch (OpTy) {
1312     case AMDGPU::OPERAND_REG_IMM_INT64:
1313     case AMDGPU::OPERAND_REG_IMM_FP64:
1314     case AMDGPU::OPERAND_REG_INLINE_C_INT64:
1315     case AMDGPU::OPERAND_REG_INLINE_C_FP64: {
1316       if (AMDGPU::isInlinableLiteral64(Literal.getZExtValue(),
1317                                        AsmParser->hasInv2PiInlineImm())) {
1318         Inst.addOperand(MCOperand::createImm(Literal.getZExtValue()));
1319         return;
1320       }
1321 
1322       // Non-inlineable
1323       if (AMDGPU::isSISrcFPOperand(InstDesc, OpNum)) { // Expected 64-bit fp operand
1324         // For fp operands we check if low 32 bits are zeros
1325         if (Literal.getLoBits(32) != 0) {
1326           const_cast<AMDGPUAsmParser *>(AsmParser)->Warning(Inst.getLoc(),
1327           "Can't encode literal as exact 64-bit floating-point operand. "
1328           "Low 32-bits will be set to zero");
1329         }
1330 
1331         Inst.addOperand(MCOperand::createImm(Literal.lshr(32).getZExtValue()));
1332         return;
1333       }
1334 
1335       // We don't allow fp literals in 64-bit integer instructions. It is
1336       // unclear how we should encode them. This case should be checked earlier
1337       // in predicate methods (isLiteralImm())
1338       llvm_unreachable("fp literal in 64-bit integer instruction.");
1339     }
1340     case AMDGPU::OPERAND_REG_IMM_INT32:
1341     case AMDGPU::OPERAND_REG_IMM_FP32:
1342     case AMDGPU::OPERAND_REG_INLINE_C_INT32:
1343     case AMDGPU::OPERAND_REG_INLINE_C_FP32:
1344     case AMDGPU::OPERAND_REG_IMM_INT16:
1345     case AMDGPU::OPERAND_REG_IMM_FP16:
1346     case AMDGPU::OPERAND_REG_INLINE_C_INT16:
1347     case AMDGPU::OPERAND_REG_INLINE_C_FP16:
1348     case AMDGPU::OPERAND_REG_INLINE_C_V2INT16:
1349     case AMDGPU::OPERAND_REG_INLINE_C_V2FP16: {
1350       bool lost;
1351       APFloat FPLiteral(APFloat::IEEEdouble(), Literal);
1352       // Convert literal to single precision
1353       FPLiteral.convert(*getOpFltSemantics(OpTy),
1354                         APFloat::rmNearestTiesToEven, &lost);
1355       // We allow precision lost but not overflow or underflow. This should be
1356       // checked earlier in isLiteralImm()
1357 
1358       uint64_t ImmVal = FPLiteral.bitcastToAPInt().getZExtValue();
1359       if (OpTy == AMDGPU::OPERAND_REG_INLINE_C_V2INT16 ||
1360           OpTy == AMDGPU::OPERAND_REG_INLINE_C_V2FP16) {
1361         ImmVal |= (ImmVal << 16);
1362       }
1363 
1364       Inst.addOperand(MCOperand::createImm(ImmVal));
1365       return;
1366     }
1367     default:
1368       llvm_unreachable("invalid operand size");
1369     }
1370 
1371     return;
1372   }
1373 
1374    // We got int literal token.
1375   // Only sign extend inline immediates.
1376   // FIXME: No errors on truncation
1377   switch (OpTy) {
1378   case AMDGPU::OPERAND_REG_IMM_INT32:
1379   case AMDGPU::OPERAND_REG_IMM_FP32:
1380   case AMDGPU::OPERAND_REG_INLINE_C_INT32:
1381   case AMDGPU::OPERAND_REG_INLINE_C_FP32: {
1382     if (isInt<32>(Val) &&
1383         AMDGPU::isInlinableLiteral32(static_cast<int32_t>(Val),
1384                                      AsmParser->hasInv2PiInlineImm())) {
1385       Inst.addOperand(MCOperand::createImm(Val));
1386       return;
1387     }
1388 
1389     Inst.addOperand(MCOperand::createImm(Val & 0xffffffff));
1390     return;
1391   }
1392   case AMDGPU::OPERAND_REG_IMM_INT64:
1393   case AMDGPU::OPERAND_REG_IMM_FP64:
1394   case AMDGPU::OPERAND_REG_INLINE_C_INT64:
1395   case AMDGPU::OPERAND_REG_INLINE_C_FP64: {
1396     if (AMDGPU::isInlinableLiteral64(Val, AsmParser->hasInv2PiInlineImm())) {
1397       Inst.addOperand(MCOperand::createImm(Val));
1398       return;
1399     }
1400 
1401     Inst.addOperand(MCOperand::createImm(Lo_32(Val)));
1402     return;
1403   }
1404   case AMDGPU::OPERAND_REG_IMM_INT16:
1405   case AMDGPU::OPERAND_REG_IMM_FP16:
1406   case AMDGPU::OPERAND_REG_INLINE_C_INT16:
1407   case AMDGPU::OPERAND_REG_INLINE_C_FP16: {
1408     if (isInt<16>(Val) &&
1409         AMDGPU::isInlinableLiteral16(static_cast<int16_t>(Val),
1410                                      AsmParser->hasInv2PiInlineImm())) {
1411       Inst.addOperand(MCOperand::createImm(Val));
1412       return;
1413     }
1414 
1415     Inst.addOperand(MCOperand::createImm(Val & 0xffff));
1416     return;
1417   }
1418   case AMDGPU::OPERAND_REG_INLINE_C_V2INT16:
1419   case AMDGPU::OPERAND_REG_INLINE_C_V2FP16: {
1420     auto LiteralVal = static_cast<uint16_t>(Literal.getLoBits(16).getZExtValue());
1421     assert(AMDGPU::isInlinableLiteral16(LiteralVal,
1422                                         AsmParser->hasInv2PiInlineImm()));
1423 
1424     uint32_t ImmVal = static_cast<uint32_t>(LiteralVal) << 16 |
1425                       static_cast<uint32_t>(LiteralVal);
1426     Inst.addOperand(MCOperand::createImm(ImmVal));
1427     return;
1428   }
1429   default:
1430     llvm_unreachable("invalid operand size");
1431   }
1432 }
1433 
1434 template <unsigned Bitwidth>
1435 void AMDGPUOperand::addKImmFPOperands(MCInst &Inst, unsigned N) const {
1436   APInt Literal(64, Imm.Val);
1437 
1438   if (!Imm.IsFPImm) {
1439     // We got int literal token.
1440     Inst.addOperand(MCOperand::createImm(Literal.getLoBits(Bitwidth).getZExtValue()));
1441     return;
1442   }
1443 
1444   bool Lost;
1445   APFloat FPLiteral(APFloat::IEEEdouble(), Literal);
1446   FPLiteral.convert(*getFltSemantics(Bitwidth / 8),
1447                     APFloat::rmNearestTiesToEven, &Lost);
1448   Inst.addOperand(MCOperand::createImm(FPLiteral.bitcastToAPInt().getZExtValue()));
1449 }
1450 
1451 void AMDGPUOperand::addRegOperands(MCInst &Inst, unsigned N) const {
1452   Inst.addOperand(MCOperand::createReg(AMDGPU::getMCReg(getReg(), AsmParser->getSTI())));
1453 }
1454 
1455 //===----------------------------------------------------------------------===//
1456 // AsmParser
1457 //===----------------------------------------------------------------------===//
1458 
1459 static int getRegClass(RegisterKind Is, unsigned RegWidth) {
1460   if (Is == IS_VGPR) {
1461     switch (RegWidth) {
1462       default: return -1;
1463       case 1: return AMDGPU::VGPR_32RegClassID;
1464       case 2: return AMDGPU::VReg_64RegClassID;
1465       case 3: return AMDGPU::VReg_96RegClassID;
1466       case 4: return AMDGPU::VReg_128RegClassID;
1467       case 8: return AMDGPU::VReg_256RegClassID;
1468       case 16: return AMDGPU::VReg_512RegClassID;
1469     }
1470   } else if (Is == IS_TTMP) {
1471     switch (RegWidth) {
1472       default: return -1;
1473       case 1: return AMDGPU::TTMP_32RegClassID;
1474       case 2: return AMDGPU::TTMP_64RegClassID;
1475       case 4: return AMDGPU::TTMP_128RegClassID;
1476     }
1477   } else if (Is == IS_SGPR) {
1478     switch (RegWidth) {
1479       default: return -1;
1480       case 1: return AMDGPU::SGPR_32RegClassID;
1481       case 2: return AMDGPU::SGPR_64RegClassID;
1482       case 4: return AMDGPU::SGPR_128RegClassID;
1483       case 8: return AMDGPU::SReg_256RegClassID;
1484       case 16: return AMDGPU::SReg_512RegClassID;
1485     }
1486   }
1487   return -1;
1488 }
1489 
1490 static unsigned getSpecialRegForName(StringRef RegName) {
1491   return StringSwitch<unsigned>(RegName)
1492     .Case("exec", AMDGPU::EXEC)
1493     .Case("vcc", AMDGPU::VCC)
1494     .Case("flat_scratch", AMDGPU::FLAT_SCR)
1495     .Case("m0", AMDGPU::M0)
1496     .Case("scc", AMDGPU::SCC)
1497     .Case("tba", AMDGPU::TBA)
1498     .Case("tma", AMDGPU::TMA)
1499     .Case("flat_scratch_lo", AMDGPU::FLAT_SCR_LO)
1500     .Case("flat_scratch_hi", AMDGPU::FLAT_SCR_HI)
1501     .Case("vcc_lo", AMDGPU::VCC_LO)
1502     .Case("vcc_hi", AMDGPU::VCC_HI)
1503     .Case("exec_lo", AMDGPU::EXEC_LO)
1504     .Case("exec_hi", AMDGPU::EXEC_HI)
1505     .Case("tma_lo", AMDGPU::TMA_LO)
1506     .Case("tma_hi", AMDGPU::TMA_HI)
1507     .Case("tba_lo", AMDGPU::TBA_LO)
1508     .Case("tba_hi", AMDGPU::TBA_HI)
1509     .Default(0);
1510 }
1511 
1512 bool AMDGPUAsmParser::ParseRegister(unsigned &RegNo, SMLoc &StartLoc,
1513                                     SMLoc &EndLoc) {
1514   auto R = parseRegister();
1515   if (!R) return true;
1516   assert(R->isReg());
1517   RegNo = R->getReg();
1518   StartLoc = R->getStartLoc();
1519   EndLoc = R->getEndLoc();
1520   return false;
1521 }
1522 
1523 bool AMDGPUAsmParser::AddNextRegisterToList(unsigned &Reg, unsigned &RegWidth,
1524                                             RegisterKind RegKind, unsigned Reg1,
1525                                             unsigned RegNum) {
1526   switch (RegKind) {
1527   case IS_SPECIAL:
1528     if (Reg == AMDGPU::EXEC_LO && Reg1 == AMDGPU::EXEC_HI) {
1529       Reg = AMDGPU::EXEC;
1530       RegWidth = 2;
1531       return true;
1532     }
1533     if (Reg == AMDGPU::FLAT_SCR_LO && Reg1 == AMDGPU::FLAT_SCR_HI) {
1534       Reg = AMDGPU::FLAT_SCR;
1535       RegWidth = 2;
1536       return true;
1537     }
1538     if (Reg == AMDGPU::VCC_LO && Reg1 == AMDGPU::VCC_HI) {
1539       Reg = AMDGPU::VCC;
1540       RegWidth = 2;
1541       return true;
1542     }
1543     if (Reg == AMDGPU::TBA_LO && Reg1 == AMDGPU::TBA_HI) {
1544       Reg = AMDGPU::TBA;
1545       RegWidth = 2;
1546       return true;
1547     }
1548     if (Reg == AMDGPU::TMA_LO && Reg1 == AMDGPU::TMA_HI) {
1549       Reg = AMDGPU::TMA;
1550       RegWidth = 2;
1551       return true;
1552     }
1553     return false;
1554   case IS_VGPR:
1555   case IS_SGPR:
1556   case IS_TTMP:
1557     if (Reg1 != Reg + RegWidth) {
1558       return false;
1559     }
1560     RegWidth++;
1561     return true;
1562   default:
1563     llvm_unreachable("unexpected register kind");
1564   }
1565 }
1566 
1567 bool AMDGPUAsmParser::ParseAMDGPURegister(RegisterKind &RegKind, unsigned &Reg,
1568                                           unsigned &RegNum, unsigned &RegWidth,
1569                                           unsigned *DwordRegIndex) {
1570   if (DwordRegIndex) { *DwordRegIndex = 0; }
1571   const MCRegisterInfo *TRI = getContext().getRegisterInfo();
1572   if (getLexer().is(AsmToken::Identifier)) {
1573     StringRef RegName = Parser.getTok().getString();
1574     if ((Reg = getSpecialRegForName(RegName))) {
1575       Parser.Lex();
1576       RegKind = IS_SPECIAL;
1577     } else {
1578       unsigned RegNumIndex = 0;
1579       if (RegName[0] == 'v') {
1580         RegNumIndex = 1;
1581         RegKind = IS_VGPR;
1582       } else if (RegName[0] == 's') {
1583         RegNumIndex = 1;
1584         RegKind = IS_SGPR;
1585       } else if (RegName.startswith("ttmp")) {
1586         RegNumIndex = strlen("ttmp");
1587         RegKind = IS_TTMP;
1588       } else {
1589         return false;
1590       }
1591       if (RegName.size() > RegNumIndex) {
1592         // Single 32-bit register: vXX.
1593         if (RegName.substr(RegNumIndex).getAsInteger(10, RegNum))
1594           return false;
1595         Parser.Lex();
1596         RegWidth = 1;
1597       } else {
1598         // Range of registers: v[XX:YY]. ":YY" is optional.
1599         Parser.Lex();
1600         int64_t RegLo, RegHi;
1601         if (getLexer().isNot(AsmToken::LBrac))
1602           return false;
1603         Parser.Lex();
1604 
1605         if (getParser().parseAbsoluteExpression(RegLo))
1606           return false;
1607 
1608         const bool isRBrace = getLexer().is(AsmToken::RBrac);
1609         if (!isRBrace && getLexer().isNot(AsmToken::Colon))
1610           return false;
1611         Parser.Lex();
1612 
1613         if (isRBrace) {
1614           RegHi = RegLo;
1615         } else {
1616           if (getParser().parseAbsoluteExpression(RegHi))
1617             return false;
1618 
1619           if (getLexer().isNot(AsmToken::RBrac))
1620             return false;
1621           Parser.Lex();
1622         }
1623         RegNum = (unsigned) RegLo;
1624         RegWidth = (RegHi - RegLo) + 1;
1625       }
1626     }
1627   } else if (getLexer().is(AsmToken::LBrac)) {
1628     // List of consecutive registers: [s0,s1,s2,s3]
1629     Parser.Lex();
1630     if (!ParseAMDGPURegister(RegKind, Reg, RegNum, RegWidth, nullptr))
1631       return false;
1632     if (RegWidth != 1)
1633       return false;
1634     RegisterKind RegKind1;
1635     unsigned Reg1, RegNum1, RegWidth1;
1636     do {
1637       if (getLexer().is(AsmToken::Comma)) {
1638         Parser.Lex();
1639       } else if (getLexer().is(AsmToken::RBrac)) {
1640         Parser.Lex();
1641         break;
1642       } else if (ParseAMDGPURegister(RegKind1, Reg1, RegNum1, RegWidth1, nullptr)) {
1643         if (RegWidth1 != 1) {
1644           return false;
1645         }
1646         if (RegKind1 != RegKind) {
1647           return false;
1648         }
1649         if (!AddNextRegisterToList(Reg, RegWidth, RegKind1, Reg1, RegNum1)) {
1650           return false;
1651         }
1652       } else {
1653         return false;
1654       }
1655     } while (true);
1656   } else {
1657     return false;
1658   }
1659   switch (RegKind) {
1660   case IS_SPECIAL:
1661     RegNum = 0;
1662     RegWidth = 1;
1663     break;
1664   case IS_VGPR:
1665   case IS_SGPR:
1666   case IS_TTMP:
1667   {
1668     unsigned Size = 1;
1669     if (RegKind == IS_SGPR || RegKind == IS_TTMP) {
1670       // SGPR and TTMP registers must be aligned. Max required alignment is 4 dwords.
1671       Size = std::min(RegWidth, 4u);
1672     }
1673     if (RegNum % Size != 0)
1674       return false;
1675     if (DwordRegIndex) { *DwordRegIndex = RegNum; }
1676     RegNum = RegNum / Size;
1677     int RCID = getRegClass(RegKind, RegWidth);
1678     if (RCID == -1)
1679       return false;
1680     const MCRegisterClass RC = TRI->getRegClass(RCID);
1681     if (RegNum >= RC.getNumRegs())
1682       return false;
1683     Reg = RC.getRegister(RegNum);
1684     break;
1685   }
1686 
1687   default:
1688     llvm_unreachable("unexpected register kind");
1689   }
1690 
1691   if (!subtargetHasRegister(*TRI, Reg))
1692     return false;
1693   return true;
1694 }
1695 
1696 std::unique_ptr<AMDGPUOperand> AMDGPUAsmParser::parseRegister() {
1697   const auto &Tok = Parser.getTok();
1698   SMLoc StartLoc = Tok.getLoc();
1699   SMLoc EndLoc = Tok.getEndLoc();
1700   RegisterKind RegKind;
1701   unsigned Reg, RegNum, RegWidth, DwordRegIndex;
1702 
1703   if (!ParseAMDGPURegister(RegKind, Reg, RegNum, RegWidth, &DwordRegIndex)) {
1704     return nullptr;
1705   }
1706   KernelScope.usesRegister(RegKind, DwordRegIndex, RegWidth);
1707   return AMDGPUOperand::CreateReg(this, Reg, StartLoc, EndLoc, false);
1708 }
1709 
1710 bool
1711 AMDGPUAsmParser::parseAbsoluteExpr(int64_t &Val, bool AbsMod) {
1712   if (AbsMod && getLexer().peekTok().is(AsmToken::Pipe) &&
1713       (getLexer().getKind() == AsmToken::Integer ||
1714        getLexer().getKind() == AsmToken::Real)) {
1715 
1716     // This is a workaround for handling operands like these:
1717     //     |1.0|
1718     //     |-1|
1719     // This syntax is not compatible with syntax of standard
1720     // MC expressions (due to the trailing '|').
1721 
1722     SMLoc EndLoc;
1723     const MCExpr *Expr;
1724 
1725     if (getParser().parsePrimaryExpr(Expr, EndLoc)) {
1726       return true;
1727     }
1728 
1729     return !Expr->evaluateAsAbsolute(Val);
1730   }
1731 
1732   return getParser().parseAbsoluteExpression(Val);
1733 }
1734 
1735 OperandMatchResultTy
1736 AMDGPUAsmParser::parseImm(OperandVector &Operands, bool AbsMod) {
1737   // TODO: add syntactic sugar for 1/(2*PI)
1738   bool Minus = false;
1739   if (getLexer().getKind() == AsmToken::Minus) {
1740     Minus = true;
1741     Parser.Lex();
1742   }
1743 
1744   SMLoc S = Parser.getTok().getLoc();
1745   switch(getLexer().getKind()) {
1746   case AsmToken::Integer: {
1747     int64_t IntVal;
1748     if (parseAbsoluteExpr(IntVal, AbsMod))
1749       return MatchOperand_ParseFail;
1750     if (Minus)
1751       IntVal *= -1;
1752     Operands.push_back(AMDGPUOperand::CreateImm(this, IntVal, S));
1753     return MatchOperand_Success;
1754   }
1755   case AsmToken::Real: {
1756     int64_t IntVal;
1757     if (parseAbsoluteExpr(IntVal, AbsMod))
1758       return MatchOperand_ParseFail;
1759 
1760     APFloat F(BitsToDouble(IntVal));
1761     if (Minus)
1762       F.changeSign();
1763     Operands.push_back(
1764         AMDGPUOperand::CreateImm(this, F.bitcastToAPInt().getZExtValue(), S,
1765                                  AMDGPUOperand::ImmTyNone, true));
1766     return MatchOperand_Success;
1767   }
1768   default:
1769     return Minus ? MatchOperand_ParseFail : MatchOperand_NoMatch;
1770   }
1771 }
1772 
1773 OperandMatchResultTy
1774 AMDGPUAsmParser::parseReg(OperandVector &Operands) {
1775   if (auto R = parseRegister()) {
1776     assert(R->isReg());
1777     R->Reg.IsForcedVOP3 = isForcedVOP3();
1778     Operands.push_back(std::move(R));
1779     return MatchOperand_Success;
1780   }
1781   return MatchOperand_NoMatch;
1782 }
1783 
1784 OperandMatchResultTy
1785 AMDGPUAsmParser::parseRegOrImm(OperandVector &Operands, bool AbsMod) {
1786   auto res = parseImm(Operands, AbsMod);
1787   if (res != MatchOperand_NoMatch) {
1788     return res;
1789   }
1790 
1791   return parseReg(Operands);
1792 }
1793 
1794 OperandMatchResultTy
1795 AMDGPUAsmParser::parseRegOrImmWithFPInputMods(OperandVector &Operands,
1796                                               bool AllowImm) {
1797   bool Negate = false, Negate2 = false, Abs = false, Abs2 = false;
1798 
1799   if (getLexer().getKind()== AsmToken::Minus) {
1800     const AsmToken NextToken = getLexer().peekTok();
1801 
1802     // Disable ambiguous constructs like '--1' etc. Should use neg(-1) instead.
1803     if (NextToken.is(AsmToken::Minus)) {
1804       Error(Parser.getTok().getLoc(), "invalid syntax, expected 'neg' modifier");
1805       return MatchOperand_ParseFail;
1806     }
1807 
1808     // '-' followed by an integer literal N should be interpreted as integer
1809     // negation rather than a floating-point NEG modifier applied to N.
1810     // Beside being contr-intuitive, such use of floating-point NEG modifier
1811     // results in different meaning of integer literals used with VOP1/2/C
1812     // and VOP3, for example:
1813     //    v_exp_f32_e32 v5, -1 // VOP1: src0 = 0xFFFFFFFF
1814     //    v_exp_f32_e64 v5, -1 // VOP3: src0 = 0x80000001
1815     // Negative fp literals should be handled likewise for unifomtity
1816     if (!NextToken.is(AsmToken::Integer) && !NextToken.is(AsmToken::Real)) {
1817       Parser.Lex();
1818       Negate = true;
1819     }
1820   }
1821 
1822   if (getLexer().getKind() == AsmToken::Identifier &&
1823       Parser.getTok().getString() == "neg") {
1824     if (Negate) {
1825       Error(Parser.getTok().getLoc(), "expected register or immediate");
1826       return MatchOperand_ParseFail;
1827     }
1828     Parser.Lex();
1829     Negate2 = true;
1830     if (getLexer().isNot(AsmToken::LParen)) {
1831       Error(Parser.getTok().getLoc(), "expected left paren after neg");
1832       return MatchOperand_ParseFail;
1833     }
1834     Parser.Lex();
1835   }
1836 
1837   if (getLexer().getKind() == AsmToken::Identifier &&
1838       Parser.getTok().getString() == "abs") {
1839     Parser.Lex();
1840     Abs2 = true;
1841     if (getLexer().isNot(AsmToken::LParen)) {
1842       Error(Parser.getTok().getLoc(), "expected left paren after abs");
1843       return MatchOperand_ParseFail;
1844     }
1845     Parser.Lex();
1846   }
1847 
1848   if (getLexer().getKind() == AsmToken::Pipe) {
1849     if (Abs2) {
1850       Error(Parser.getTok().getLoc(), "expected register or immediate");
1851       return MatchOperand_ParseFail;
1852     }
1853     Parser.Lex();
1854     Abs = true;
1855   }
1856 
1857   OperandMatchResultTy Res;
1858   if (AllowImm) {
1859     Res = parseRegOrImm(Operands, Abs);
1860   } else {
1861     Res = parseReg(Operands);
1862   }
1863   if (Res != MatchOperand_Success) {
1864     return Res;
1865   }
1866 
1867   AMDGPUOperand::Modifiers Mods;
1868   if (Abs) {
1869     if (getLexer().getKind() != AsmToken::Pipe) {
1870       Error(Parser.getTok().getLoc(), "expected vertical bar");
1871       return MatchOperand_ParseFail;
1872     }
1873     Parser.Lex();
1874     Mods.Abs = true;
1875   }
1876   if (Abs2) {
1877     if (getLexer().isNot(AsmToken::RParen)) {
1878       Error(Parser.getTok().getLoc(), "expected closing parentheses");
1879       return MatchOperand_ParseFail;
1880     }
1881     Parser.Lex();
1882     Mods.Abs = true;
1883   }
1884 
1885   if (Negate) {
1886     Mods.Neg = true;
1887   } else if (Negate2) {
1888     if (getLexer().isNot(AsmToken::RParen)) {
1889       Error(Parser.getTok().getLoc(), "expected closing parentheses");
1890       return MatchOperand_ParseFail;
1891     }
1892     Parser.Lex();
1893     Mods.Neg = true;
1894   }
1895 
1896   if (Mods.hasFPModifiers()) {
1897     AMDGPUOperand &Op = static_cast<AMDGPUOperand &>(*Operands.back());
1898     Op.setModifiers(Mods);
1899   }
1900   return MatchOperand_Success;
1901 }
1902 
1903 OperandMatchResultTy
1904 AMDGPUAsmParser::parseRegOrImmWithIntInputMods(OperandVector &Operands,
1905                                                bool AllowImm) {
1906   bool Sext = false;
1907 
1908   if (getLexer().getKind() == AsmToken::Identifier &&
1909       Parser.getTok().getString() == "sext") {
1910     Parser.Lex();
1911     Sext = true;
1912     if (getLexer().isNot(AsmToken::LParen)) {
1913       Error(Parser.getTok().getLoc(), "expected left paren after sext");
1914       return MatchOperand_ParseFail;
1915     }
1916     Parser.Lex();
1917   }
1918 
1919   OperandMatchResultTy Res;
1920   if (AllowImm) {
1921     Res = parseRegOrImm(Operands);
1922   } else {
1923     Res = parseReg(Operands);
1924   }
1925   if (Res != MatchOperand_Success) {
1926     return Res;
1927   }
1928 
1929   AMDGPUOperand::Modifiers Mods;
1930   if (Sext) {
1931     if (getLexer().isNot(AsmToken::RParen)) {
1932       Error(Parser.getTok().getLoc(), "expected closing parentheses");
1933       return MatchOperand_ParseFail;
1934     }
1935     Parser.Lex();
1936     Mods.Sext = true;
1937   }
1938 
1939   if (Mods.hasIntModifiers()) {
1940     AMDGPUOperand &Op = static_cast<AMDGPUOperand &>(*Operands.back());
1941     Op.setModifiers(Mods);
1942   }
1943 
1944   return MatchOperand_Success;
1945 }
1946 
1947 OperandMatchResultTy
1948 AMDGPUAsmParser::parseRegWithFPInputMods(OperandVector &Operands) {
1949   return parseRegOrImmWithFPInputMods(Operands, false);
1950 }
1951 
1952 OperandMatchResultTy
1953 AMDGPUAsmParser::parseRegWithIntInputMods(OperandVector &Operands) {
1954   return parseRegOrImmWithIntInputMods(Operands, false);
1955 }
1956 
1957 OperandMatchResultTy AMDGPUAsmParser::parseVReg32OrOff(OperandVector &Operands) {
1958   std::unique_ptr<AMDGPUOperand> Reg = parseRegister();
1959   if (Reg) {
1960     Operands.push_back(std::move(Reg));
1961     return MatchOperand_Success;
1962   }
1963 
1964   const AsmToken &Tok = Parser.getTok();
1965   if (Tok.getString() == "off") {
1966     Operands.push_back(AMDGPUOperand::CreateImm(this, 0, Tok.getLoc(),
1967                                                 AMDGPUOperand::ImmTyOff, false));
1968     Parser.Lex();
1969     return MatchOperand_Success;
1970   }
1971 
1972   return MatchOperand_NoMatch;
1973 }
1974 
1975 unsigned AMDGPUAsmParser::checkTargetMatchPredicate(MCInst &Inst) {
1976   uint64_t TSFlags = MII.get(Inst.getOpcode()).TSFlags;
1977 
1978   if ((getForcedEncodingSize() == 32 && (TSFlags & SIInstrFlags::VOP3)) ||
1979       (getForcedEncodingSize() == 64 && !(TSFlags & SIInstrFlags::VOP3)) ||
1980       (isForcedDPP() && !(TSFlags & SIInstrFlags::DPP)) ||
1981       (isForcedSDWA() && !(TSFlags & SIInstrFlags::SDWA)) )
1982     return Match_InvalidOperand;
1983 
1984   if ((TSFlags & SIInstrFlags::VOP3) &&
1985       (TSFlags & SIInstrFlags::VOPAsmPrefer32Bit) &&
1986       getForcedEncodingSize() != 64)
1987     return Match_PreferE32;
1988 
1989   if (Inst.getOpcode() == AMDGPU::V_MAC_F32_sdwa_vi ||
1990       Inst.getOpcode() == AMDGPU::V_MAC_F16_sdwa_vi) {
1991     // v_mac_f32/16 allow only dst_sel == DWORD;
1992     auto OpNum =
1993         AMDGPU::getNamedOperandIdx(Inst.getOpcode(), AMDGPU::OpName::dst_sel);
1994     const auto &Op = Inst.getOperand(OpNum);
1995     if (!Op.isImm() || Op.getImm() != AMDGPU::SDWA::SdwaSel::DWORD) {
1996       return Match_InvalidOperand;
1997     }
1998   }
1999 
2000   if ((TSFlags & SIInstrFlags::FLAT) && !hasFlatOffsets()) {
2001     // FIXME: Produces error without correct column reported.
2002     auto OpNum =
2003         AMDGPU::getNamedOperandIdx(Inst.getOpcode(), AMDGPU::OpName::offset);
2004     const auto &Op = Inst.getOperand(OpNum);
2005     if (Op.getImm() != 0)
2006       return Match_InvalidOperand;
2007   }
2008 
2009   return Match_Success;
2010 }
2011 
2012 // What asm variants we should check
2013 ArrayRef<unsigned> AMDGPUAsmParser::getMatchedVariants() const {
2014   if (getForcedEncodingSize() == 32) {
2015     static const unsigned Variants[] = {AMDGPUAsmVariants::DEFAULT};
2016     return makeArrayRef(Variants);
2017   }
2018 
2019   if (isForcedVOP3()) {
2020     static const unsigned Variants[] = {AMDGPUAsmVariants::VOP3};
2021     return makeArrayRef(Variants);
2022   }
2023 
2024   if (isForcedSDWA()) {
2025     static const unsigned Variants[] = {AMDGPUAsmVariants::SDWA,
2026                                         AMDGPUAsmVariants::SDWA9};
2027     return makeArrayRef(Variants);
2028   }
2029 
2030   if (isForcedDPP()) {
2031     static const unsigned Variants[] = {AMDGPUAsmVariants::DPP};
2032     return makeArrayRef(Variants);
2033   }
2034 
2035   static const unsigned Variants[] = {
2036     AMDGPUAsmVariants::DEFAULT, AMDGPUAsmVariants::VOP3,
2037     AMDGPUAsmVariants::SDWA, AMDGPUAsmVariants::SDWA9, AMDGPUAsmVariants::DPP
2038   };
2039 
2040   return makeArrayRef(Variants);
2041 }
2042 
2043 unsigned AMDGPUAsmParser::findImplicitSGPRReadInVOP(const MCInst &Inst) const {
2044   const MCInstrDesc &Desc = MII.get(Inst.getOpcode());
2045   const unsigned Num = Desc.getNumImplicitUses();
2046   for (unsigned i = 0; i < Num; ++i) {
2047     unsigned Reg = Desc.ImplicitUses[i];
2048     switch (Reg) {
2049     case AMDGPU::FLAT_SCR:
2050     case AMDGPU::VCC:
2051     case AMDGPU::M0:
2052       return Reg;
2053     default:
2054       break;
2055     }
2056   }
2057   return AMDGPU::NoRegister;
2058 }
2059 
2060 // NB: This code is correct only when used to check constant
2061 // bus limitations because GFX7 support no f16 inline constants.
2062 // Note that there are no cases when a GFX7 opcode violates
2063 // constant bus limitations due to the use of an f16 constant.
2064 bool AMDGPUAsmParser::isInlineConstant(const MCInst &Inst,
2065                                        unsigned OpIdx) const {
2066   const MCInstrDesc &Desc = MII.get(Inst.getOpcode());
2067 
2068   if (!AMDGPU::isSISrcOperand(Desc, OpIdx)) {
2069     return false;
2070   }
2071 
2072   const MCOperand &MO = Inst.getOperand(OpIdx);
2073 
2074   int64_t Val = MO.getImm();
2075   auto OpSize = AMDGPU::getOperandSize(Desc, OpIdx);
2076 
2077   switch (OpSize) { // expected operand size
2078   case 8:
2079     return AMDGPU::isInlinableLiteral64(Val, hasInv2PiInlineImm());
2080   case 4:
2081     return AMDGPU::isInlinableLiteral32(Val, hasInv2PiInlineImm());
2082   case 2: {
2083     const unsigned OperandType = Desc.OpInfo[OpIdx].OperandType;
2084     if (OperandType == AMDGPU::OPERAND_REG_INLINE_C_V2INT16 ||
2085         OperandType == AMDGPU::OPERAND_REG_INLINE_C_V2FP16) {
2086       return AMDGPU::isInlinableLiteralV216(Val, hasInv2PiInlineImm());
2087     } else {
2088       return AMDGPU::isInlinableLiteral16(Val, hasInv2PiInlineImm());
2089     }
2090   }
2091   default:
2092     llvm_unreachable("invalid operand size");
2093   }
2094 }
2095 
2096 bool AMDGPUAsmParser::usesConstantBus(const MCInst &Inst, unsigned OpIdx) {
2097   const MCOperand &MO = Inst.getOperand(OpIdx);
2098   if (MO.isImm()) {
2099     return !isInlineConstant(Inst, OpIdx);
2100   }
2101   return !MO.isReg() ||
2102          isSGPR(mc2PseudoReg(MO.getReg()), getContext().getRegisterInfo());
2103 }
2104 
2105 bool AMDGPUAsmParser::validateConstantBusLimitations(const MCInst &Inst) {
2106   const unsigned Opcode = Inst.getOpcode();
2107   const MCInstrDesc &Desc = MII.get(Opcode);
2108   unsigned ConstantBusUseCount = 0;
2109 
2110   if (Desc.TSFlags &
2111       (SIInstrFlags::VOPC |
2112        SIInstrFlags::VOP1 | SIInstrFlags::VOP2 |
2113        SIInstrFlags::VOP3 | SIInstrFlags::VOP3P |
2114        SIInstrFlags::SDWA)) {
2115 
2116     // Check special imm operands (used by madmk, etc)
2117     if (AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::imm) != -1) {
2118       ++ConstantBusUseCount;
2119     }
2120 
2121     unsigned SGPRUsed = findImplicitSGPRReadInVOP(Inst);
2122     if (SGPRUsed != AMDGPU::NoRegister) {
2123       ++ConstantBusUseCount;
2124     }
2125 
2126     const int Src0Idx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::src0);
2127     const int Src1Idx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::src1);
2128     const int Src2Idx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::src2);
2129 
2130     const int OpIndices[] = { Src0Idx, Src1Idx, Src2Idx };
2131 
2132     for (int OpIdx : OpIndices) {
2133       if (OpIdx == -1) break;
2134 
2135       const MCOperand &MO = Inst.getOperand(OpIdx);
2136       if (usesConstantBus(Inst, OpIdx)) {
2137         if (MO.isReg()) {
2138           const unsigned Reg = mc2PseudoReg(MO.getReg());
2139           // Pairs of registers with a partial intersections like these
2140           //   s0, s[0:1]
2141           //   flat_scratch_lo, flat_scratch
2142           //   flat_scratch_lo, flat_scratch_hi
2143           // are theoretically valid but they are disabled anyway.
2144           // Note that this code mimics SIInstrInfo::verifyInstruction
2145           if (Reg != SGPRUsed) {
2146             ++ConstantBusUseCount;
2147           }
2148           SGPRUsed = Reg;
2149         } else { // Expression or a literal
2150           ++ConstantBusUseCount;
2151         }
2152       }
2153     }
2154   }
2155 
2156   return ConstantBusUseCount <= 1;
2157 }
2158 
2159 bool AMDGPUAsmParser::validateEarlyClobberLimitations(const MCInst &Inst) {
2160 
2161   const unsigned Opcode = Inst.getOpcode();
2162   const MCInstrDesc &Desc = MII.get(Opcode);
2163 
2164   const int DstIdx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::vdst);
2165   if (DstIdx == -1 ||
2166       Desc.getOperandConstraint(DstIdx, MCOI::EARLY_CLOBBER) == -1) {
2167     return true;
2168   }
2169 
2170   const MCRegisterInfo *TRI = getContext().getRegisterInfo();
2171 
2172   const int Src0Idx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::src0);
2173   const int Src1Idx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::src1);
2174   const int Src2Idx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::src2);
2175 
2176   assert(DstIdx != -1);
2177   const MCOperand &Dst = Inst.getOperand(DstIdx);
2178   assert(Dst.isReg());
2179   const unsigned DstReg = mc2PseudoReg(Dst.getReg());
2180 
2181   const int SrcIndices[] = { Src0Idx, Src1Idx, Src2Idx };
2182 
2183   for (int SrcIdx : SrcIndices) {
2184     if (SrcIdx == -1) break;
2185     const MCOperand &Src = Inst.getOperand(SrcIdx);
2186     if (Src.isReg()) {
2187       const unsigned SrcReg = mc2PseudoReg(Src.getReg());
2188       if (isRegIntersect(DstReg, SrcReg, TRI)) {
2189         return false;
2190       }
2191     }
2192   }
2193 
2194   return true;
2195 }
2196 
2197 bool AMDGPUAsmParser::validateInstruction(const MCInst &Inst,
2198                                           const SMLoc &IDLoc) {
2199   if (!validateConstantBusLimitations(Inst)) {
2200     Error(IDLoc,
2201       "invalid operand (violates constant bus restrictions)");
2202     return false;
2203   }
2204   if (!validateEarlyClobberLimitations(Inst)) {
2205     Error(IDLoc,
2206       "destination must be different than all sources");
2207     return false;
2208   }
2209 
2210   return true;
2211 }
2212 
2213 bool AMDGPUAsmParser::MatchAndEmitInstruction(SMLoc IDLoc, unsigned &Opcode,
2214                                               OperandVector &Operands,
2215                                               MCStreamer &Out,
2216                                               uint64_t &ErrorInfo,
2217                                               bool MatchingInlineAsm) {
2218   MCInst Inst;
2219   unsigned Result = Match_Success;
2220   for (auto Variant : getMatchedVariants()) {
2221     uint64_t EI;
2222     auto R = MatchInstructionImpl(Operands, Inst, EI, MatchingInlineAsm,
2223                                   Variant);
2224     // We order match statuses from least to most specific. We use most specific
2225     // status as resulting
2226     // Match_MnemonicFail < Match_InvalidOperand < Match_MissingFeature < Match_PreferE32
2227     if ((R == Match_Success) ||
2228         (R == Match_PreferE32) ||
2229         (R == Match_MissingFeature && Result != Match_PreferE32) ||
2230         (R == Match_InvalidOperand && Result != Match_MissingFeature
2231                                    && Result != Match_PreferE32) ||
2232         (R == Match_MnemonicFail   && Result != Match_InvalidOperand
2233                                    && Result != Match_MissingFeature
2234                                    && Result != Match_PreferE32)) {
2235       Result = R;
2236       ErrorInfo = EI;
2237     }
2238     if (R == Match_Success)
2239       break;
2240   }
2241 
2242   switch (Result) {
2243   default: break;
2244   case Match_Success:
2245     if (!validateInstruction(Inst, IDLoc)) {
2246       return true;
2247     }
2248     Inst.setLoc(IDLoc);
2249     Out.EmitInstruction(Inst, getSTI());
2250     return false;
2251 
2252   case Match_MissingFeature:
2253     return Error(IDLoc, "instruction not supported on this GPU");
2254 
2255   case Match_MnemonicFail:
2256     return Error(IDLoc, "unrecognized instruction mnemonic");
2257 
2258   case Match_InvalidOperand: {
2259     SMLoc ErrorLoc = IDLoc;
2260     if (ErrorInfo != ~0ULL) {
2261       if (ErrorInfo >= Operands.size()) {
2262         return Error(IDLoc, "too few operands for instruction");
2263       }
2264       ErrorLoc = ((AMDGPUOperand &)*Operands[ErrorInfo]).getStartLoc();
2265       if (ErrorLoc == SMLoc())
2266         ErrorLoc = IDLoc;
2267     }
2268     return Error(ErrorLoc, "invalid operand for instruction");
2269   }
2270 
2271   case Match_PreferE32:
2272     return Error(IDLoc, "internal error: instruction without _e64 suffix "
2273                         "should be encoded as e32");
2274   }
2275   llvm_unreachable("Implement any new match types added!");
2276 }
2277 
2278 bool AMDGPUAsmParser::ParseAsAbsoluteExpression(uint32_t &Ret) {
2279   int64_t Tmp = -1;
2280   if (getLexer().isNot(AsmToken::Integer) && getLexer().isNot(AsmToken::Identifier)) {
2281     return true;
2282   }
2283   if (getParser().parseAbsoluteExpression(Tmp)) {
2284     return true;
2285   }
2286   Ret = static_cast<uint32_t>(Tmp);
2287   return false;
2288 }
2289 
2290 bool AMDGPUAsmParser::ParseDirectiveMajorMinor(uint32_t &Major,
2291                                                uint32_t &Minor) {
2292   if (ParseAsAbsoluteExpression(Major))
2293     return TokError("invalid major version");
2294 
2295   if (getLexer().isNot(AsmToken::Comma))
2296     return TokError("minor version number required, comma expected");
2297   Lex();
2298 
2299   if (ParseAsAbsoluteExpression(Minor))
2300     return TokError("invalid minor version");
2301 
2302   return false;
2303 }
2304 
2305 bool AMDGPUAsmParser::ParseDirectiveHSACodeObjectVersion() {
2306   uint32_t Major;
2307   uint32_t Minor;
2308 
2309   if (ParseDirectiveMajorMinor(Major, Minor))
2310     return true;
2311 
2312   getTargetStreamer().EmitDirectiveHSACodeObjectVersion(Major, Minor);
2313   return false;
2314 }
2315 
2316 bool AMDGPUAsmParser::ParseDirectiveHSACodeObjectISA() {
2317   uint32_t Major;
2318   uint32_t Minor;
2319   uint32_t Stepping;
2320   StringRef VendorName;
2321   StringRef ArchName;
2322 
2323   // If this directive has no arguments, then use the ISA version for the
2324   // targeted GPU.
2325   if (getLexer().is(AsmToken::EndOfStatement)) {
2326     AMDGPU::IsaInfo::IsaVersion ISA =
2327         AMDGPU::IsaInfo::getIsaVersion(getFeatureBits());
2328     getTargetStreamer().EmitDirectiveHSACodeObjectISA(ISA.Major, ISA.Minor,
2329                                                       ISA.Stepping,
2330                                                       "AMD", "AMDGPU");
2331     return false;
2332   }
2333 
2334   if (ParseDirectiveMajorMinor(Major, Minor))
2335     return true;
2336 
2337   if (getLexer().isNot(AsmToken::Comma))
2338     return TokError("stepping version number required, comma expected");
2339   Lex();
2340 
2341   if (ParseAsAbsoluteExpression(Stepping))
2342     return TokError("invalid stepping version");
2343 
2344   if (getLexer().isNot(AsmToken::Comma))
2345     return TokError("vendor name required, comma expected");
2346   Lex();
2347 
2348   if (getLexer().isNot(AsmToken::String))
2349     return TokError("invalid vendor name");
2350 
2351   VendorName = getLexer().getTok().getStringContents();
2352   Lex();
2353 
2354   if (getLexer().isNot(AsmToken::Comma))
2355     return TokError("arch name required, comma expected");
2356   Lex();
2357 
2358   if (getLexer().isNot(AsmToken::String))
2359     return TokError("invalid arch name");
2360 
2361   ArchName = getLexer().getTok().getStringContents();
2362   Lex();
2363 
2364   getTargetStreamer().EmitDirectiveHSACodeObjectISA(Major, Minor, Stepping,
2365                                                     VendorName, ArchName);
2366   return false;
2367 }
2368 
2369 bool AMDGPUAsmParser::ParseDirectiveCodeObjectMetadata() {
2370   std::string YamlString;
2371   raw_string_ostream YamlStream(YamlString);
2372 
2373   getLexer().setSkipSpace(false);
2374 
2375   bool FoundEnd = false;
2376   while (!getLexer().is(AsmToken::Eof)) {
2377     while (getLexer().is(AsmToken::Space)) {
2378       YamlStream << getLexer().getTok().getString();
2379       Lex();
2380     }
2381 
2382     if (getLexer().is(AsmToken::Identifier)) {
2383       StringRef ID = getLexer().getTok().getIdentifier();
2384       if (ID == AMDGPU::CodeObject::MetadataAssemblerDirectiveEnd) {
2385         Lex();
2386         FoundEnd = true;
2387         break;
2388       }
2389     }
2390 
2391     YamlStream << Parser.parseStringToEndOfStatement()
2392                << getContext().getAsmInfo()->getSeparatorString();
2393 
2394     Parser.eatToEndOfStatement();
2395   }
2396 
2397   getLexer().setSkipSpace(true);
2398 
2399   if (getLexer().is(AsmToken::Eof) && !FoundEnd) {
2400     return TokError(
2401         "expected directive .end_amdgpu_code_object_metadata not found");
2402   }
2403 
2404   YamlStream.flush();
2405 
2406   if (!getTargetStreamer().EmitCodeObjectMetadata(YamlString))
2407     return Error(getParser().getTok().getLoc(), "invalid code object metadata");
2408 
2409   return false;
2410 }
2411 
2412 bool AMDGPUAsmParser::ParseAMDKernelCodeTValue(StringRef ID,
2413                                                amd_kernel_code_t &Header) {
2414   SmallString<40> ErrStr;
2415   raw_svector_ostream Err(ErrStr);
2416   if (!parseAmdKernelCodeField(ID, getParser(), Header, Err)) {
2417     return TokError(Err.str());
2418   }
2419   Lex();
2420   return false;
2421 }
2422 
2423 bool AMDGPUAsmParser::ParseDirectiveAMDKernelCodeT() {
2424   amd_kernel_code_t Header;
2425   AMDGPU::initDefaultAMDKernelCodeT(Header, getFeatureBits());
2426 
2427   while (true) {
2428     // Lex EndOfStatement.  This is in a while loop, because lexing a comment
2429     // will set the current token to EndOfStatement.
2430     while(getLexer().is(AsmToken::EndOfStatement))
2431       Lex();
2432 
2433     if (getLexer().isNot(AsmToken::Identifier))
2434       return TokError("expected value identifier or .end_amd_kernel_code_t");
2435 
2436     StringRef ID = getLexer().getTok().getIdentifier();
2437     Lex();
2438 
2439     if (ID == ".end_amd_kernel_code_t")
2440       break;
2441 
2442     if (ParseAMDKernelCodeTValue(ID, Header))
2443       return true;
2444   }
2445 
2446   getTargetStreamer().EmitAMDKernelCodeT(Header);
2447 
2448   return false;
2449 }
2450 
2451 bool AMDGPUAsmParser::ParseDirectiveAMDGPUHsaKernel() {
2452   if (getLexer().isNot(AsmToken::Identifier))
2453     return TokError("expected symbol name");
2454 
2455   StringRef KernelName = Parser.getTok().getString();
2456 
2457   getTargetStreamer().EmitAMDGPUSymbolType(KernelName,
2458                                            ELF::STT_AMDGPU_HSA_KERNEL);
2459   Lex();
2460   KernelScope.initialize(getContext());
2461   return false;
2462 }
2463 
2464 bool AMDGPUAsmParser::ParseDirective(AsmToken DirectiveID) {
2465   StringRef IDVal = DirectiveID.getString();
2466 
2467   if (IDVal == ".hsa_code_object_version")
2468     return ParseDirectiveHSACodeObjectVersion();
2469 
2470   if (IDVal == ".hsa_code_object_isa")
2471     return ParseDirectiveHSACodeObjectISA();
2472 
2473   if (IDVal == AMDGPU::CodeObject::MetadataAssemblerDirectiveBegin)
2474     return ParseDirectiveCodeObjectMetadata();
2475 
2476   if (IDVal == ".amd_kernel_code_t")
2477     return ParseDirectiveAMDKernelCodeT();
2478 
2479   if (IDVal == ".amdgpu_hsa_kernel")
2480     return ParseDirectiveAMDGPUHsaKernel();
2481 
2482   return true;
2483 }
2484 
2485 bool AMDGPUAsmParser::subtargetHasRegister(const MCRegisterInfo &MRI,
2486                                            unsigned RegNo) const {
2487   if (isCI())
2488     return true;
2489 
2490   if (isSI()) {
2491     // No flat_scr
2492     switch (RegNo) {
2493     case AMDGPU::FLAT_SCR:
2494     case AMDGPU::FLAT_SCR_LO:
2495     case AMDGPU::FLAT_SCR_HI:
2496       return false;
2497     default:
2498       return true;
2499     }
2500   }
2501 
2502   // VI only has 102 SGPRs, so make sure we aren't trying to use the 2 more that
2503   // SI/CI have.
2504   for (MCRegAliasIterator R(AMDGPU::SGPR102_SGPR103, &MRI, true);
2505        R.isValid(); ++R) {
2506     if (*R == RegNo)
2507       return false;
2508   }
2509 
2510   return true;
2511 }
2512 
2513 OperandMatchResultTy
2514 AMDGPUAsmParser::parseOperand(OperandVector &Operands, StringRef Mnemonic) {
2515   // Try to parse with a custom parser
2516   OperandMatchResultTy ResTy = MatchOperandParserImpl(Operands, Mnemonic);
2517 
2518   // If we successfully parsed the operand or if there as an error parsing,
2519   // we are done.
2520   //
2521   // If we are parsing after we reach EndOfStatement then this means we
2522   // are appending default values to the Operands list.  This is only done
2523   // by custom parser, so we shouldn't continue on to the generic parsing.
2524   if (ResTy == MatchOperand_Success || ResTy == MatchOperand_ParseFail ||
2525       getLexer().is(AsmToken::EndOfStatement))
2526     return ResTy;
2527 
2528   ResTy = parseRegOrImm(Operands);
2529 
2530   if (ResTy == MatchOperand_Success)
2531     return ResTy;
2532 
2533   if (getLexer().getKind() == AsmToken::Identifier) {
2534     // If this identifier is a symbol, we want to create an expression for it.
2535     // It is a little difficult to distinguish between a symbol name, and
2536     // an instruction flag like 'gds'.  In order to do this, we parse
2537     // all tokens as expressions and then treate the symbol name as the token
2538     // string when we want to interpret the operand as a token.
2539     const auto &Tok = Parser.getTok();
2540     SMLoc S = Tok.getLoc();
2541     const MCExpr *Expr = nullptr;
2542     if (!Parser.parseExpression(Expr)) {
2543       Operands.push_back(AMDGPUOperand::CreateExpr(this, Expr, S));
2544       return MatchOperand_Success;
2545     }
2546 
2547     Operands.push_back(AMDGPUOperand::CreateToken(this, Tok.getString(), Tok.getLoc()));
2548     Parser.Lex();
2549     return MatchOperand_Success;
2550   }
2551   return MatchOperand_NoMatch;
2552 }
2553 
2554 StringRef AMDGPUAsmParser::parseMnemonicSuffix(StringRef Name) {
2555   // Clear any forced encodings from the previous instruction.
2556   setForcedEncodingSize(0);
2557   setForcedDPP(false);
2558   setForcedSDWA(false);
2559 
2560   if (Name.endswith("_e64")) {
2561     setForcedEncodingSize(64);
2562     return Name.substr(0, Name.size() - 4);
2563   } else if (Name.endswith("_e32")) {
2564     setForcedEncodingSize(32);
2565     return Name.substr(0, Name.size() - 4);
2566   } else if (Name.endswith("_dpp")) {
2567     setForcedDPP(true);
2568     return Name.substr(0, Name.size() - 4);
2569   } else if (Name.endswith("_sdwa")) {
2570     setForcedSDWA(true);
2571     return Name.substr(0, Name.size() - 5);
2572   }
2573   return Name;
2574 }
2575 
2576 bool AMDGPUAsmParser::ParseInstruction(ParseInstructionInfo &Info,
2577                                        StringRef Name,
2578                                        SMLoc NameLoc, OperandVector &Operands) {
2579   // Add the instruction mnemonic
2580   Name = parseMnemonicSuffix(Name);
2581   Operands.push_back(AMDGPUOperand::CreateToken(this, Name, NameLoc));
2582 
2583   while (!getLexer().is(AsmToken::EndOfStatement)) {
2584     OperandMatchResultTy Res = parseOperand(Operands, Name);
2585 
2586     // Eat the comma or space if there is one.
2587     if (getLexer().is(AsmToken::Comma))
2588       Parser.Lex();
2589 
2590     switch (Res) {
2591       case MatchOperand_Success: break;
2592       case MatchOperand_ParseFail:
2593         Error(getLexer().getLoc(), "failed parsing operand.");
2594         while (!getLexer().is(AsmToken::EndOfStatement)) {
2595           Parser.Lex();
2596         }
2597         return true;
2598       case MatchOperand_NoMatch:
2599         Error(getLexer().getLoc(), "not a valid operand.");
2600         while (!getLexer().is(AsmToken::EndOfStatement)) {
2601           Parser.Lex();
2602         }
2603         return true;
2604     }
2605   }
2606 
2607   return false;
2608 }
2609 
2610 //===----------------------------------------------------------------------===//
2611 // Utility functions
2612 //===----------------------------------------------------------------------===//
2613 
2614 OperandMatchResultTy
2615 AMDGPUAsmParser::parseIntWithPrefix(const char *Prefix, int64_t &Int) {
2616   switch(getLexer().getKind()) {
2617     default: return MatchOperand_NoMatch;
2618     case AsmToken::Identifier: {
2619       StringRef Name = Parser.getTok().getString();
2620       if (!Name.equals(Prefix)) {
2621         return MatchOperand_NoMatch;
2622       }
2623 
2624       Parser.Lex();
2625       if (getLexer().isNot(AsmToken::Colon))
2626         return MatchOperand_ParseFail;
2627 
2628       Parser.Lex();
2629 
2630       bool IsMinus = false;
2631       if (getLexer().getKind() == AsmToken::Minus) {
2632         Parser.Lex();
2633         IsMinus = true;
2634       }
2635 
2636       if (getLexer().isNot(AsmToken::Integer))
2637         return MatchOperand_ParseFail;
2638 
2639       if (getParser().parseAbsoluteExpression(Int))
2640         return MatchOperand_ParseFail;
2641 
2642       if (IsMinus)
2643         Int = -Int;
2644       break;
2645     }
2646   }
2647   return MatchOperand_Success;
2648 }
2649 
2650 OperandMatchResultTy
2651 AMDGPUAsmParser::parseIntWithPrefix(const char *Prefix, OperandVector &Operands,
2652                                     AMDGPUOperand::ImmTy ImmTy,
2653                                     bool (*ConvertResult)(int64_t&)) {
2654   SMLoc S = Parser.getTok().getLoc();
2655   int64_t Value = 0;
2656 
2657   OperandMatchResultTy Res = parseIntWithPrefix(Prefix, Value);
2658   if (Res != MatchOperand_Success)
2659     return Res;
2660 
2661   if (ConvertResult && !ConvertResult(Value)) {
2662     return MatchOperand_ParseFail;
2663   }
2664 
2665   Operands.push_back(AMDGPUOperand::CreateImm(this, Value, S, ImmTy));
2666   return MatchOperand_Success;
2667 }
2668 
2669 OperandMatchResultTy AMDGPUAsmParser::parseOperandArrayWithPrefix(
2670   const char *Prefix,
2671   OperandVector &Operands,
2672   AMDGPUOperand::ImmTy ImmTy,
2673   bool (*ConvertResult)(int64_t&)) {
2674   StringRef Name = Parser.getTok().getString();
2675   if (!Name.equals(Prefix))
2676     return MatchOperand_NoMatch;
2677 
2678   Parser.Lex();
2679   if (getLexer().isNot(AsmToken::Colon))
2680     return MatchOperand_ParseFail;
2681 
2682   Parser.Lex();
2683   if (getLexer().isNot(AsmToken::LBrac))
2684     return MatchOperand_ParseFail;
2685   Parser.Lex();
2686 
2687   unsigned Val = 0;
2688   SMLoc S = Parser.getTok().getLoc();
2689 
2690   // FIXME: How to verify the number of elements matches the number of src
2691   // operands?
2692   for (int I = 0; I < 4; ++I) {
2693     if (I != 0) {
2694       if (getLexer().is(AsmToken::RBrac))
2695         break;
2696 
2697       if (getLexer().isNot(AsmToken::Comma))
2698         return MatchOperand_ParseFail;
2699       Parser.Lex();
2700     }
2701 
2702     if (getLexer().isNot(AsmToken::Integer))
2703       return MatchOperand_ParseFail;
2704 
2705     int64_t Op;
2706     if (getParser().parseAbsoluteExpression(Op))
2707       return MatchOperand_ParseFail;
2708 
2709     if (Op != 0 && Op != 1)
2710       return MatchOperand_ParseFail;
2711     Val |= (Op << I);
2712   }
2713 
2714   Parser.Lex();
2715   Operands.push_back(AMDGPUOperand::CreateImm(this, Val, S, ImmTy));
2716   return MatchOperand_Success;
2717 }
2718 
2719 OperandMatchResultTy
2720 AMDGPUAsmParser::parseNamedBit(const char *Name, OperandVector &Operands,
2721                                AMDGPUOperand::ImmTy ImmTy) {
2722   int64_t Bit = 0;
2723   SMLoc S = Parser.getTok().getLoc();
2724 
2725   // We are at the end of the statement, and this is a default argument, so
2726   // use a default value.
2727   if (getLexer().isNot(AsmToken::EndOfStatement)) {
2728     switch(getLexer().getKind()) {
2729       case AsmToken::Identifier: {
2730         StringRef Tok = Parser.getTok().getString();
2731         if (Tok == Name) {
2732           Bit = 1;
2733           Parser.Lex();
2734         } else if (Tok.startswith("no") && Tok.endswith(Name)) {
2735           Bit = 0;
2736           Parser.Lex();
2737         } else {
2738           return MatchOperand_NoMatch;
2739         }
2740         break;
2741       }
2742       default:
2743         return MatchOperand_NoMatch;
2744     }
2745   }
2746 
2747   Operands.push_back(AMDGPUOperand::CreateImm(this, Bit, S, ImmTy));
2748   return MatchOperand_Success;
2749 }
2750 
2751 static void addOptionalImmOperand(
2752   MCInst& Inst, const OperandVector& Operands,
2753   AMDGPUAsmParser::OptionalImmIndexMap& OptionalIdx,
2754   AMDGPUOperand::ImmTy ImmT,
2755   int64_t Default = 0) {
2756   auto i = OptionalIdx.find(ImmT);
2757   if (i != OptionalIdx.end()) {
2758     unsigned Idx = i->second;
2759     ((AMDGPUOperand &)*Operands[Idx]).addImmOperands(Inst, 1);
2760   } else {
2761     Inst.addOperand(MCOperand::createImm(Default));
2762   }
2763 }
2764 
2765 OperandMatchResultTy
2766 AMDGPUAsmParser::parseStringWithPrefix(StringRef Prefix, StringRef &Value) {
2767   if (getLexer().isNot(AsmToken::Identifier)) {
2768     return MatchOperand_NoMatch;
2769   }
2770   StringRef Tok = Parser.getTok().getString();
2771   if (Tok != Prefix) {
2772     return MatchOperand_NoMatch;
2773   }
2774 
2775   Parser.Lex();
2776   if (getLexer().isNot(AsmToken::Colon)) {
2777     return MatchOperand_ParseFail;
2778   }
2779 
2780   Parser.Lex();
2781   if (getLexer().isNot(AsmToken::Identifier)) {
2782     return MatchOperand_ParseFail;
2783   }
2784 
2785   Value = Parser.getTok().getString();
2786   return MatchOperand_Success;
2787 }
2788 
2789 //===----------------------------------------------------------------------===//
2790 // ds
2791 //===----------------------------------------------------------------------===//
2792 
2793 void AMDGPUAsmParser::cvtDSOffset01(MCInst &Inst,
2794                                     const OperandVector &Operands) {
2795   OptionalImmIndexMap OptionalIdx;
2796 
2797   for (unsigned i = 1, e = Operands.size(); i != e; ++i) {
2798     AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[i]);
2799 
2800     // Add the register arguments
2801     if (Op.isReg()) {
2802       Op.addRegOperands(Inst, 1);
2803       continue;
2804     }
2805 
2806     // Handle optional arguments
2807     OptionalIdx[Op.getImmTy()] = i;
2808   }
2809 
2810   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyOffset0);
2811   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyOffset1);
2812   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyGDS);
2813 
2814   Inst.addOperand(MCOperand::createReg(AMDGPU::M0)); // m0
2815 }
2816 
2817 void AMDGPUAsmParser::cvtDSImpl(MCInst &Inst, const OperandVector &Operands,
2818                                 bool IsGdsHardcoded) {
2819   OptionalImmIndexMap OptionalIdx;
2820 
2821   for (unsigned i = 1, e = Operands.size(); i != e; ++i) {
2822     AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[i]);
2823 
2824     // Add the register arguments
2825     if (Op.isReg()) {
2826       Op.addRegOperands(Inst, 1);
2827       continue;
2828     }
2829 
2830     if (Op.isToken() && Op.getToken() == "gds") {
2831       IsGdsHardcoded = true;
2832       continue;
2833     }
2834 
2835     // Handle optional arguments
2836     OptionalIdx[Op.getImmTy()] = i;
2837   }
2838 
2839   AMDGPUOperand::ImmTy OffsetType =
2840     (Inst.getOpcode() == AMDGPU::DS_SWIZZLE_B32_si ||
2841      Inst.getOpcode() == AMDGPU::DS_SWIZZLE_B32_vi) ? AMDGPUOperand::ImmTySwizzle :
2842                                                       AMDGPUOperand::ImmTyOffset;
2843 
2844   addOptionalImmOperand(Inst, Operands, OptionalIdx, OffsetType);
2845 
2846   if (!IsGdsHardcoded) {
2847     addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyGDS);
2848   }
2849   Inst.addOperand(MCOperand::createReg(AMDGPU::M0)); // m0
2850 }
2851 
2852 void AMDGPUAsmParser::cvtExp(MCInst &Inst, const OperandVector &Operands) {
2853   OptionalImmIndexMap OptionalIdx;
2854 
2855   unsigned OperandIdx[4];
2856   unsigned EnMask = 0;
2857   int SrcIdx = 0;
2858 
2859   for (unsigned i = 1, e = Operands.size(); i != e; ++i) {
2860     AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[i]);
2861 
2862     // Add the register arguments
2863     if (Op.isReg()) {
2864       assert(SrcIdx < 4);
2865       OperandIdx[SrcIdx] = Inst.size();
2866       Op.addRegOperands(Inst, 1);
2867       ++SrcIdx;
2868       continue;
2869     }
2870 
2871     if (Op.isOff()) {
2872       assert(SrcIdx < 4);
2873       OperandIdx[SrcIdx] = Inst.size();
2874       Inst.addOperand(MCOperand::createReg(AMDGPU::NoRegister));
2875       ++SrcIdx;
2876       continue;
2877     }
2878 
2879     if (Op.isImm() && Op.getImmTy() == AMDGPUOperand::ImmTyExpTgt) {
2880       Op.addImmOperands(Inst, 1);
2881       continue;
2882     }
2883 
2884     if (Op.isToken() && Op.getToken() == "done")
2885       continue;
2886 
2887     // Handle optional arguments
2888     OptionalIdx[Op.getImmTy()] = i;
2889   }
2890 
2891   assert(SrcIdx == 4);
2892 
2893   bool Compr = false;
2894   if (OptionalIdx.find(AMDGPUOperand::ImmTyExpCompr) != OptionalIdx.end()) {
2895     Compr = true;
2896     Inst.getOperand(OperandIdx[1]) = Inst.getOperand(OperandIdx[2]);
2897     Inst.getOperand(OperandIdx[2]).setReg(AMDGPU::NoRegister);
2898     Inst.getOperand(OperandIdx[3]).setReg(AMDGPU::NoRegister);
2899   }
2900 
2901   for (auto i = 0; i < SrcIdx; ++i) {
2902     if (Inst.getOperand(OperandIdx[i]).getReg() != AMDGPU::NoRegister) {
2903       EnMask |= Compr? (0x3 << i * 2) : (0x1 << i);
2904     }
2905   }
2906 
2907   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyExpVM);
2908   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyExpCompr);
2909 
2910   Inst.addOperand(MCOperand::createImm(EnMask));
2911 }
2912 
2913 //===----------------------------------------------------------------------===//
2914 // s_waitcnt
2915 //===----------------------------------------------------------------------===//
2916 
2917 static bool
2918 encodeCnt(
2919   const AMDGPU::IsaInfo::IsaVersion ISA,
2920   int64_t &IntVal,
2921   int64_t CntVal,
2922   bool Saturate,
2923   unsigned (*encode)(const IsaInfo::IsaVersion &Version, unsigned, unsigned),
2924   unsigned (*decode)(const IsaInfo::IsaVersion &Version, unsigned))
2925 {
2926   bool Failed = false;
2927 
2928   IntVal = encode(ISA, IntVal, CntVal);
2929   if (CntVal != decode(ISA, IntVal)) {
2930     if (Saturate) {
2931       IntVal = encode(ISA, IntVal, -1);
2932     } else {
2933       Failed = true;
2934     }
2935   }
2936   return Failed;
2937 }
2938 
2939 bool AMDGPUAsmParser::parseCnt(int64_t &IntVal) {
2940   StringRef CntName = Parser.getTok().getString();
2941   int64_t CntVal;
2942 
2943   Parser.Lex();
2944   if (getLexer().isNot(AsmToken::LParen))
2945     return true;
2946 
2947   Parser.Lex();
2948   if (getLexer().isNot(AsmToken::Integer))
2949     return true;
2950 
2951   SMLoc ValLoc = Parser.getTok().getLoc();
2952   if (getParser().parseAbsoluteExpression(CntVal))
2953     return true;
2954 
2955   AMDGPU::IsaInfo::IsaVersion ISA =
2956       AMDGPU::IsaInfo::getIsaVersion(getFeatureBits());
2957 
2958   bool Failed = true;
2959   bool Sat = CntName.endswith("_sat");
2960 
2961   if (CntName == "vmcnt" || CntName == "vmcnt_sat") {
2962     Failed = encodeCnt(ISA, IntVal, CntVal, Sat, encodeVmcnt, decodeVmcnt);
2963   } else if (CntName == "expcnt" || CntName == "expcnt_sat") {
2964     Failed = encodeCnt(ISA, IntVal, CntVal, Sat, encodeExpcnt, decodeExpcnt);
2965   } else if (CntName == "lgkmcnt" || CntName == "lgkmcnt_sat") {
2966     Failed = encodeCnt(ISA, IntVal, CntVal, Sat, encodeLgkmcnt, decodeLgkmcnt);
2967   }
2968 
2969   if (Failed) {
2970     Error(ValLoc, "too large value for " + CntName);
2971     return true;
2972   }
2973 
2974   if (getLexer().isNot(AsmToken::RParen)) {
2975     return true;
2976   }
2977 
2978   Parser.Lex();
2979   if (getLexer().is(AsmToken::Amp) || getLexer().is(AsmToken::Comma)) {
2980     const AsmToken NextToken = getLexer().peekTok();
2981     if (NextToken.is(AsmToken::Identifier)) {
2982       Parser.Lex();
2983     }
2984   }
2985 
2986   return false;
2987 }
2988 
2989 OperandMatchResultTy
2990 AMDGPUAsmParser::parseSWaitCntOps(OperandVector &Operands) {
2991   AMDGPU::IsaInfo::IsaVersion ISA =
2992       AMDGPU::IsaInfo::getIsaVersion(getFeatureBits());
2993   int64_t Waitcnt = getWaitcntBitMask(ISA);
2994   SMLoc S = Parser.getTok().getLoc();
2995 
2996   switch(getLexer().getKind()) {
2997     default: return MatchOperand_ParseFail;
2998     case AsmToken::Integer:
2999       // The operand can be an integer value.
3000       if (getParser().parseAbsoluteExpression(Waitcnt))
3001         return MatchOperand_ParseFail;
3002       break;
3003 
3004     case AsmToken::Identifier:
3005       do {
3006         if (parseCnt(Waitcnt))
3007           return MatchOperand_ParseFail;
3008       } while(getLexer().isNot(AsmToken::EndOfStatement));
3009       break;
3010   }
3011   Operands.push_back(AMDGPUOperand::CreateImm(this, Waitcnt, S));
3012   return MatchOperand_Success;
3013 }
3014 
3015 bool AMDGPUAsmParser::parseHwregConstruct(OperandInfoTy &HwReg, int64_t &Offset,
3016                                           int64_t &Width) {
3017   using namespace llvm::AMDGPU::Hwreg;
3018 
3019   if (Parser.getTok().getString() != "hwreg")
3020     return true;
3021   Parser.Lex();
3022 
3023   if (getLexer().isNot(AsmToken::LParen))
3024     return true;
3025   Parser.Lex();
3026 
3027   if (getLexer().is(AsmToken::Identifier)) {
3028     HwReg.IsSymbolic = true;
3029     HwReg.Id = ID_UNKNOWN_;
3030     const StringRef tok = Parser.getTok().getString();
3031     for (int i = ID_SYMBOLIC_FIRST_; i < ID_SYMBOLIC_LAST_; ++i) {
3032       if (tok == IdSymbolic[i]) {
3033         HwReg.Id = i;
3034         break;
3035       }
3036     }
3037     Parser.Lex();
3038   } else {
3039     HwReg.IsSymbolic = false;
3040     if (getLexer().isNot(AsmToken::Integer))
3041       return true;
3042     if (getParser().parseAbsoluteExpression(HwReg.Id))
3043       return true;
3044   }
3045 
3046   if (getLexer().is(AsmToken::RParen)) {
3047     Parser.Lex();
3048     return false;
3049   }
3050 
3051   // optional params
3052   if (getLexer().isNot(AsmToken::Comma))
3053     return true;
3054   Parser.Lex();
3055 
3056   if (getLexer().isNot(AsmToken::Integer))
3057     return true;
3058   if (getParser().parseAbsoluteExpression(Offset))
3059     return true;
3060 
3061   if (getLexer().isNot(AsmToken::Comma))
3062     return true;
3063   Parser.Lex();
3064 
3065   if (getLexer().isNot(AsmToken::Integer))
3066     return true;
3067   if (getParser().parseAbsoluteExpression(Width))
3068     return true;
3069 
3070   if (getLexer().isNot(AsmToken::RParen))
3071     return true;
3072   Parser.Lex();
3073 
3074   return false;
3075 }
3076 
3077 OperandMatchResultTy AMDGPUAsmParser::parseHwreg(OperandVector &Operands) {
3078   using namespace llvm::AMDGPU::Hwreg;
3079 
3080   int64_t Imm16Val = 0;
3081   SMLoc S = Parser.getTok().getLoc();
3082 
3083   switch(getLexer().getKind()) {
3084     default: return MatchOperand_NoMatch;
3085     case AsmToken::Integer:
3086       // The operand can be an integer value.
3087       if (getParser().parseAbsoluteExpression(Imm16Val))
3088         return MatchOperand_NoMatch;
3089       if (Imm16Val < 0 || !isUInt<16>(Imm16Val)) {
3090         Error(S, "invalid immediate: only 16-bit values are legal");
3091         // Do not return error code, but create an imm operand anyway and proceed
3092         // to the next operand, if any. That avoids unneccessary error messages.
3093       }
3094       break;
3095 
3096     case AsmToken::Identifier: {
3097         OperandInfoTy HwReg(ID_UNKNOWN_);
3098         int64_t Offset = OFFSET_DEFAULT_;
3099         int64_t Width = WIDTH_M1_DEFAULT_ + 1;
3100         if (parseHwregConstruct(HwReg, Offset, Width))
3101           return MatchOperand_ParseFail;
3102         if (HwReg.Id < 0 || !isUInt<ID_WIDTH_>(HwReg.Id)) {
3103           if (HwReg.IsSymbolic)
3104             Error(S, "invalid symbolic name of hardware register");
3105           else
3106             Error(S, "invalid code of hardware register: only 6-bit values are legal");
3107         }
3108         if (Offset < 0 || !isUInt<OFFSET_WIDTH_>(Offset))
3109           Error(S, "invalid bit offset: only 5-bit values are legal");
3110         if ((Width-1) < 0 || !isUInt<WIDTH_M1_WIDTH_>(Width-1))
3111           Error(S, "invalid bitfield width: only values from 1 to 32 are legal");
3112         Imm16Val = (HwReg.Id << ID_SHIFT_) | (Offset << OFFSET_SHIFT_) | ((Width-1) << WIDTH_M1_SHIFT_);
3113       }
3114       break;
3115   }
3116   Operands.push_back(AMDGPUOperand::CreateImm(this, Imm16Val, S, AMDGPUOperand::ImmTyHwreg));
3117   return MatchOperand_Success;
3118 }
3119 
3120 bool AMDGPUOperand::isSWaitCnt() const {
3121   return isImm();
3122 }
3123 
3124 bool AMDGPUOperand::isHwreg() const {
3125   return isImmTy(ImmTyHwreg);
3126 }
3127 
3128 bool AMDGPUAsmParser::parseSendMsgConstruct(OperandInfoTy &Msg, OperandInfoTy &Operation, int64_t &StreamId) {
3129   using namespace llvm::AMDGPU::SendMsg;
3130 
3131   if (Parser.getTok().getString() != "sendmsg")
3132     return true;
3133   Parser.Lex();
3134 
3135   if (getLexer().isNot(AsmToken::LParen))
3136     return true;
3137   Parser.Lex();
3138 
3139   if (getLexer().is(AsmToken::Identifier)) {
3140     Msg.IsSymbolic = true;
3141     Msg.Id = ID_UNKNOWN_;
3142     const std::string tok = Parser.getTok().getString();
3143     for (int i = ID_GAPS_FIRST_; i < ID_GAPS_LAST_; ++i) {
3144       switch(i) {
3145         default: continue; // Omit gaps.
3146         case ID_INTERRUPT: case ID_GS: case ID_GS_DONE:  case ID_SYSMSG: break;
3147       }
3148       if (tok == IdSymbolic[i]) {
3149         Msg.Id = i;
3150         break;
3151       }
3152     }
3153     Parser.Lex();
3154   } else {
3155     Msg.IsSymbolic = false;
3156     if (getLexer().isNot(AsmToken::Integer))
3157       return true;
3158     if (getParser().parseAbsoluteExpression(Msg.Id))
3159       return true;
3160     if (getLexer().is(AsmToken::Integer))
3161       if (getParser().parseAbsoluteExpression(Msg.Id))
3162         Msg.Id = ID_UNKNOWN_;
3163   }
3164   if (Msg.Id == ID_UNKNOWN_) // Don't know how to parse the rest.
3165     return false;
3166 
3167   if (!(Msg.Id == ID_GS || Msg.Id == ID_GS_DONE || Msg.Id == ID_SYSMSG)) {
3168     if (getLexer().isNot(AsmToken::RParen))
3169       return true;
3170     Parser.Lex();
3171     return false;
3172   }
3173 
3174   if (getLexer().isNot(AsmToken::Comma))
3175     return true;
3176   Parser.Lex();
3177 
3178   assert(Msg.Id == ID_GS || Msg.Id == ID_GS_DONE || Msg.Id == ID_SYSMSG);
3179   Operation.Id = ID_UNKNOWN_;
3180   if (getLexer().is(AsmToken::Identifier)) {
3181     Operation.IsSymbolic = true;
3182     const char* const *S = (Msg.Id == ID_SYSMSG) ? OpSysSymbolic : OpGsSymbolic;
3183     const int F = (Msg.Id == ID_SYSMSG) ? OP_SYS_FIRST_ : OP_GS_FIRST_;
3184     const int L = (Msg.Id == ID_SYSMSG) ? OP_SYS_LAST_ : OP_GS_LAST_;
3185     const StringRef Tok = Parser.getTok().getString();
3186     for (int i = F; i < L; ++i) {
3187       if (Tok == S[i]) {
3188         Operation.Id = i;
3189         break;
3190       }
3191     }
3192     Parser.Lex();
3193   } else {
3194     Operation.IsSymbolic = false;
3195     if (getLexer().isNot(AsmToken::Integer))
3196       return true;
3197     if (getParser().parseAbsoluteExpression(Operation.Id))
3198       return true;
3199   }
3200 
3201   if ((Msg.Id == ID_GS || Msg.Id == ID_GS_DONE) && Operation.Id != OP_GS_NOP) {
3202     // Stream id is optional.
3203     if (getLexer().is(AsmToken::RParen)) {
3204       Parser.Lex();
3205       return false;
3206     }
3207 
3208     if (getLexer().isNot(AsmToken::Comma))
3209       return true;
3210     Parser.Lex();
3211 
3212     if (getLexer().isNot(AsmToken::Integer))
3213       return true;
3214     if (getParser().parseAbsoluteExpression(StreamId))
3215       return true;
3216   }
3217 
3218   if (getLexer().isNot(AsmToken::RParen))
3219     return true;
3220   Parser.Lex();
3221   return false;
3222 }
3223 
3224 OperandMatchResultTy AMDGPUAsmParser::parseInterpSlot(OperandVector &Operands) {
3225   if (getLexer().getKind() != AsmToken::Identifier)
3226     return MatchOperand_NoMatch;
3227 
3228   StringRef Str = Parser.getTok().getString();
3229   int Slot = StringSwitch<int>(Str)
3230     .Case("p10", 0)
3231     .Case("p20", 1)
3232     .Case("p0", 2)
3233     .Default(-1);
3234 
3235   SMLoc S = Parser.getTok().getLoc();
3236   if (Slot == -1)
3237     return MatchOperand_ParseFail;
3238 
3239   Parser.Lex();
3240   Operands.push_back(AMDGPUOperand::CreateImm(this, Slot, S,
3241                                               AMDGPUOperand::ImmTyInterpSlot));
3242   return MatchOperand_Success;
3243 }
3244 
3245 OperandMatchResultTy AMDGPUAsmParser::parseInterpAttr(OperandVector &Operands) {
3246   if (getLexer().getKind() != AsmToken::Identifier)
3247     return MatchOperand_NoMatch;
3248 
3249   StringRef Str = Parser.getTok().getString();
3250   if (!Str.startswith("attr"))
3251     return MatchOperand_NoMatch;
3252 
3253   StringRef Chan = Str.take_back(2);
3254   int AttrChan = StringSwitch<int>(Chan)
3255     .Case(".x", 0)
3256     .Case(".y", 1)
3257     .Case(".z", 2)
3258     .Case(".w", 3)
3259     .Default(-1);
3260   if (AttrChan == -1)
3261     return MatchOperand_ParseFail;
3262 
3263   Str = Str.drop_back(2).drop_front(4);
3264 
3265   uint8_t Attr;
3266   if (Str.getAsInteger(10, Attr))
3267     return MatchOperand_ParseFail;
3268 
3269   SMLoc S = Parser.getTok().getLoc();
3270   Parser.Lex();
3271   if (Attr > 63) {
3272     Error(S, "out of bounds attr");
3273     return MatchOperand_Success;
3274   }
3275 
3276   SMLoc SChan = SMLoc::getFromPointer(Chan.data());
3277 
3278   Operands.push_back(AMDGPUOperand::CreateImm(this, Attr, S,
3279                                               AMDGPUOperand::ImmTyInterpAttr));
3280   Operands.push_back(AMDGPUOperand::CreateImm(this, AttrChan, SChan,
3281                                               AMDGPUOperand::ImmTyAttrChan));
3282   return MatchOperand_Success;
3283 }
3284 
3285 void AMDGPUAsmParser::errorExpTgt() {
3286   Error(Parser.getTok().getLoc(), "invalid exp target");
3287 }
3288 
3289 OperandMatchResultTy AMDGPUAsmParser::parseExpTgtImpl(StringRef Str,
3290                                                       uint8_t &Val) {
3291   if (Str == "null") {
3292     Val = 9;
3293     return MatchOperand_Success;
3294   }
3295 
3296   if (Str.startswith("mrt")) {
3297     Str = Str.drop_front(3);
3298     if (Str == "z") { // == mrtz
3299       Val = 8;
3300       return MatchOperand_Success;
3301     }
3302 
3303     if (Str.getAsInteger(10, Val))
3304       return MatchOperand_ParseFail;
3305 
3306     if (Val > 7)
3307       errorExpTgt();
3308 
3309     return MatchOperand_Success;
3310   }
3311 
3312   if (Str.startswith("pos")) {
3313     Str = Str.drop_front(3);
3314     if (Str.getAsInteger(10, Val))
3315       return MatchOperand_ParseFail;
3316 
3317     if (Val > 3)
3318       errorExpTgt();
3319 
3320     Val += 12;
3321     return MatchOperand_Success;
3322   }
3323 
3324   if (Str.startswith("param")) {
3325     Str = Str.drop_front(5);
3326     if (Str.getAsInteger(10, Val))
3327       return MatchOperand_ParseFail;
3328 
3329     if (Val >= 32)
3330       errorExpTgt();
3331 
3332     Val += 32;
3333     return MatchOperand_Success;
3334   }
3335 
3336   if (Str.startswith("invalid_target_")) {
3337     Str = Str.drop_front(15);
3338     if (Str.getAsInteger(10, Val))
3339       return MatchOperand_ParseFail;
3340 
3341     errorExpTgt();
3342     return MatchOperand_Success;
3343   }
3344 
3345   return MatchOperand_NoMatch;
3346 }
3347 
3348 OperandMatchResultTy AMDGPUAsmParser::parseExpTgt(OperandVector &Operands) {
3349   uint8_t Val;
3350   StringRef Str = Parser.getTok().getString();
3351 
3352   auto Res = parseExpTgtImpl(Str, Val);
3353   if (Res != MatchOperand_Success)
3354     return Res;
3355 
3356   SMLoc S = Parser.getTok().getLoc();
3357   Parser.Lex();
3358 
3359   Operands.push_back(AMDGPUOperand::CreateImm(this, Val, S,
3360                                               AMDGPUOperand::ImmTyExpTgt));
3361   return MatchOperand_Success;
3362 }
3363 
3364 OperandMatchResultTy
3365 AMDGPUAsmParser::parseSendMsgOp(OperandVector &Operands) {
3366   using namespace llvm::AMDGPU::SendMsg;
3367 
3368   int64_t Imm16Val = 0;
3369   SMLoc S = Parser.getTok().getLoc();
3370 
3371   switch(getLexer().getKind()) {
3372   default:
3373     return MatchOperand_NoMatch;
3374   case AsmToken::Integer:
3375     // The operand can be an integer value.
3376     if (getParser().parseAbsoluteExpression(Imm16Val))
3377       return MatchOperand_NoMatch;
3378     if (Imm16Val < 0 || !isUInt<16>(Imm16Val)) {
3379       Error(S, "invalid immediate: only 16-bit values are legal");
3380       // Do not return error code, but create an imm operand anyway and proceed
3381       // to the next operand, if any. That avoids unneccessary error messages.
3382     }
3383     break;
3384   case AsmToken::Identifier: {
3385       OperandInfoTy Msg(ID_UNKNOWN_);
3386       OperandInfoTy Operation(OP_UNKNOWN_);
3387       int64_t StreamId = STREAM_ID_DEFAULT_;
3388       if (parseSendMsgConstruct(Msg, Operation, StreamId))
3389         return MatchOperand_ParseFail;
3390       do {
3391         // Validate and encode message ID.
3392         if (! ((ID_INTERRUPT <= Msg.Id && Msg.Id <= ID_GS_DONE)
3393                 || Msg.Id == ID_SYSMSG)) {
3394           if (Msg.IsSymbolic)
3395             Error(S, "invalid/unsupported symbolic name of message");
3396           else
3397             Error(S, "invalid/unsupported code of message");
3398           break;
3399         }
3400         Imm16Val = (Msg.Id << ID_SHIFT_);
3401         // Validate and encode operation ID.
3402         if (Msg.Id == ID_GS || Msg.Id == ID_GS_DONE) {
3403           if (! (OP_GS_FIRST_ <= Operation.Id && Operation.Id < OP_GS_LAST_)) {
3404             if (Operation.IsSymbolic)
3405               Error(S, "invalid symbolic name of GS_OP");
3406             else
3407               Error(S, "invalid code of GS_OP: only 2-bit values are legal");
3408             break;
3409           }
3410           if (Operation.Id == OP_GS_NOP
3411               && Msg.Id != ID_GS_DONE) {
3412             Error(S, "invalid GS_OP: NOP is for GS_DONE only");
3413             break;
3414           }
3415           Imm16Val |= (Operation.Id << OP_SHIFT_);
3416         }
3417         if (Msg.Id == ID_SYSMSG) {
3418           if (! (OP_SYS_FIRST_ <= Operation.Id && Operation.Id < OP_SYS_LAST_)) {
3419             if (Operation.IsSymbolic)
3420               Error(S, "invalid/unsupported symbolic name of SYSMSG_OP");
3421             else
3422               Error(S, "invalid/unsupported code of SYSMSG_OP");
3423             break;
3424           }
3425           Imm16Val |= (Operation.Id << OP_SHIFT_);
3426         }
3427         // Validate and encode stream ID.
3428         if ((Msg.Id == ID_GS || Msg.Id == ID_GS_DONE) && Operation.Id != OP_GS_NOP) {
3429           if (! (STREAM_ID_FIRST_ <= StreamId && StreamId < STREAM_ID_LAST_)) {
3430             Error(S, "invalid stream id: only 2-bit values are legal");
3431             break;
3432           }
3433           Imm16Val |= (StreamId << STREAM_ID_SHIFT_);
3434         }
3435       } while (false);
3436     }
3437     break;
3438   }
3439   Operands.push_back(AMDGPUOperand::CreateImm(this, Imm16Val, S, AMDGPUOperand::ImmTySendMsg));
3440   return MatchOperand_Success;
3441 }
3442 
3443 bool AMDGPUOperand::isSendMsg() const {
3444   return isImmTy(ImmTySendMsg);
3445 }
3446 
3447 //===----------------------------------------------------------------------===//
3448 // parser helpers
3449 //===----------------------------------------------------------------------===//
3450 
3451 bool
3452 AMDGPUAsmParser::trySkipId(const StringRef Id) {
3453   if (getLexer().getKind() == AsmToken::Identifier &&
3454       Parser.getTok().getString() == Id) {
3455     Parser.Lex();
3456     return true;
3457   }
3458   return false;
3459 }
3460 
3461 bool
3462 AMDGPUAsmParser::trySkipToken(const AsmToken::TokenKind Kind) {
3463   if (getLexer().getKind() == Kind) {
3464     Parser.Lex();
3465     return true;
3466   }
3467   return false;
3468 }
3469 
3470 bool
3471 AMDGPUAsmParser::skipToken(const AsmToken::TokenKind Kind,
3472                            const StringRef ErrMsg) {
3473   if (!trySkipToken(Kind)) {
3474     Error(Parser.getTok().getLoc(), ErrMsg);
3475     return false;
3476   }
3477   return true;
3478 }
3479 
3480 bool
3481 AMDGPUAsmParser::parseExpr(int64_t &Imm) {
3482   return !getParser().parseAbsoluteExpression(Imm);
3483 }
3484 
3485 bool
3486 AMDGPUAsmParser::parseString(StringRef &Val, const StringRef ErrMsg) {
3487   SMLoc S = Parser.getTok().getLoc();
3488   if (getLexer().getKind() == AsmToken::String) {
3489     Val = Parser.getTok().getStringContents();
3490     Parser.Lex();
3491     return true;
3492   } else {
3493     Error(S, ErrMsg);
3494     return false;
3495   }
3496 }
3497 
3498 //===----------------------------------------------------------------------===//
3499 // swizzle
3500 //===----------------------------------------------------------------------===//
3501 
3502 LLVM_READNONE
3503 static unsigned
3504 encodeBitmaskPerm(const unsigned AndMask,
3505                   const unsigned OrMask,
3506                   const unsigned XorMask) {
3507   using namespace llvm::AMDGPU::Swizzle;
3508 
3509   return BITMASK_PERM_ENC |
3510          (AndMask << BITMASK_AND_SHIFT) |
3511          (OrMask  << BITMASK_OR_SHIFT)  |
3512          (XorMask << BITMASK_XOR_SHIFT);
3513 }
3514 
3515 bool
3516 AMDGPUAsmParser::parseSwizzleOperands(const unsigned OpNum, int64_t* Op,
3517                                       const unsigned MinVal,
3518                                       const unsigned MaxVal,
3519                                       const StringRef ErrMsg) {
3520   for (unsigned i = 0; i < OpNum; ++i) {
3521     if (!skipToken(AsmToken::Comma, "expected a comma")){
3522       return false;
3523     }
3524     SMLoc ExprLoc = Parser.getTok().getLoc();
3525     if (!parseExpr(Op[i])) {
3526       return false;
3527     }
3528     if (Op[i] < MinVal || Op[i] > MaxVal) {
3529       Error(ExprLoc, ErrMsg);
3530       return false;
3531     }
3532   }
3533 
3534   return true;
3535 }
3536 
3537 bool
3538 AMDGPUAsmParser::parseSwizzleQuadPerm(int64_t &Imm) {
3539   using namespace llvm::AMDGPU::Swizzle;
3540 
3541   int64_t Lane[LANE_NUM];
3542   if (parseSwizzleOperands(LANE_NUM, Lane, 0, LANE_MAX,
3543                            "expected a 2-bit lane id")) {
3544     Imm = QUAD_PERM_ENC;
3545     for (auto i = 0; i < LANE_NUM; ++i) {
3546       Imm |= Lane[i] << (LANE_SHIFT * i);
3547     }
3548     return true;
3549   }
3550   return false;
3551 }
3552 
3553 bool
3554 AMDGPUAsmParser::parseSwizzleBroadcast(int64_t &Imm) {
3555   using namespace llvm::AMDGPU::Swizzle;
3556 
3557   SMLoc S = Parser.getTok().getLoc();
3558   int64_t GroupSize;
3559   int64_t LaneIdx;
3560 
3561   if (!parseSwizzleOperands(1, &GroupSize,
3562                             2, 32,
3563                             "group size must be in the interval [2,32]")) {
3564     return false;
3565   }
3566   if (!isPowerOf2_64(GroupSize)) {
3567     Error(S, "group size must be a power of two");
3568     return false;
3569   }
3570   if (parseSwizzleOperands(1, &LaneIdx,
3571                            0, GroupSize - 1,
3572                            "lane id must be in the interval [0,group size - 1]")) {
3573     Imm = encodeBitmaskPerm(BITMASK_MAX - GroupSize + 1, LaneIdx, 0);
3574     return true;
3575   }
3576   return false;
3577 }
3578 
3579 bool
3580 AMDGPUAsmParser::parseSwizzleReverse(int64_t &Imm) {
3581   using namespace llvm::AMDGPU::Swizzle;
3582 
3583   SMLoc S = Parser.getTok().getLoc();
3584   int64_t GroupSize;
3585 
3586   if (!parseSwizzleOperands(1, &GroupSize,
3587       2, 32, "group size must be in the interval [2,32]")) {
3588     return false;
3589   }
3590   if (!isPowerOf2_64(GroupSize)) {
3591     Error(S, "group size must be a power of two");
3592     return false;
3593   }
3594 
3595   Imm = encodeBitmaskPerm(BITMASK_MAX, 0, GroupSize - 1);
3596   return true;
3597 }
3598 
3599 bool
3600 AMDGPUAsmParser::parseSwizzleSwap(int64_t &Imm) {
3601   using namespace llvm::AMDGPU::Swizzle;
3602 
3603   SMLoc S = Parser.getTok().getLoc();
3604   int64_t GroupSize;
3605 
3606   if (!parseSwizzleOperands(1, &GroupSize,
3607       1, 16, "group size must be in the interval [1,16]")) {
3608     return false;
3609   }
3610   if (!isPowerOf2_64(GroupSize)) {
3611     Error(S, "group size must be a power of two");
3612     return false;
3613   }
3614 
3615   Imm = encodeBitmaskPerm(BITMASK_MAX, 0, GroupSize);
3616   return true;
3617 }
3618 
3619 bool
3620 AMDGPUAsmParser::parseSwizzleBitmaskPerm(int64_t &Imm) {
3621   using namespace llvm::AMDGPU::Swizzle;
3622 
3623   if (!skipToken(AsmToken::Comma, "expected a comma")) {
3624     return false;
3625   }
3626 
3627   StringRef Ctl;
3628   SMLoc StrLoc = Parser.getTok().getLoc();
3629   if (!parseString(Ctl)) {
3630     return false;
3631   }
3632   if (Ctl.size() != BITMASK_WIDTH) {
3633     Error(StrLoc, "expected a 5-character mask");
3634     return false;
3635   }
3636 
3637   unsigned AndMask = 0;
3638   unsigned OrMask = 0;
3639   unsigned XorMask = 0;
3640 
3641   for (size_t i = 0; i < Ctl.size(); ++i) {
3642     unsigned Mask = 1 << (BITMASK_WIDTH - 1 - i);
3643     switch(Ctl[i]) {
3644     default:
3645       Error(StrLoc, "invalid mask");
3646       return false;
3647     case '0':
3648       break;
3649     case '1':
3650       OrMask |= Mask;
3651       break;
3652     case 'p':
3653       AndMask |= Mask;
3654       break;
3655     case 'i':
3656       AndMask |= Mask;
3657       XorMask |= Mask;
3658       break;
3659     }
3660   }
3661 
3662   Imm = encodeBitmaskPerm(AndMask, OrMask, XorMask);
3663   return true;
3664 }
3665 
3666 bool
3667 AMDGPUAsmParser::parseSwizzleOffset(int64_t &Imm) {
3668 
3669   SMLoc OffsetLoc = Parser.getTok().getLoc();
3670 
3671   if (!parseExpr(Imm)) {
3672     return false;
3673   }
3674   if (!isUInt<16>(Imm)) {
3675     Error(OffsetLoc, "expected a 16-bit offset");
3676     return false;
3677   }
3678   return true;
3679 }
3680 
3681 bool
3682 AMDGPUAsmParser::parseSwizzleMacro(int64_t &Imm) {
3683   using namespace llvm::AMDGPU::Swizzle;
3684 
3685   if (skipToken(AsmToken::LParen, "expected a left parentheses")) {
3686 
3687     SMLoc ModeLoc = Parser.getTok().getLoc();
3688     bool Ok = false;
3689 
3690     if (trySkipId(IdSymbolic[ID_QUAD_PERM])) {
3691       Ok = parseSwizzleQuadPerm(Imm);
3692     } else if (trySkipId(IdSymbolic[ID_BITMASK_PERM])) {
3693       Ok = parseSwizzleBitmaskPerm(Imm);
3694     } else if (trySkipId(IdSymbolic[ID_BROADCAST])) {
3695       Ok = parseSwizzleBroadcast(Imm);
3696     } else if (trySkipId(IdSymbolic[ID_SWAP])) {
3697       Ok = parseSwizzleSwap(Imm);
3698     } else if (trySkipId(IdSymbolic[ID_REVERSE])) {
3699       Ok = parseSwizzleReverse(Imm);
3700     } else {
3701       Error(ModeLoc, "expected a swizzle mode");
3702     }
3703 
3704     return Ok && skipToken(AsmToken::RParen, "expected a closing parentheses");
3705   }
3706 
3707   return false;
3708 }
3709 
3710 OperandMatchResultTy
3711 AMDGPUAsmParser::parseSwizzleOp(OperandVector &Operands) {
3712   SMLoc S = Parser.getTok().getLoc();
3713   int64_t Imm = 0;
3714 
3715   if (trySkipId("offset")) {
3716 
3717     bool Ok = false;
3718     if (skipToken(AsmToken::Colon, "expected a colon")) {
3719       if (trySkipId("swizzle")) {
3720         Ok = parseSwizzleMacro(Imm);
3721       } else {
3722         Ok = parseSwizzleOffset(Imm);
3723       }
3724     }
3725 
3726     Operands.push_back(AMDGPUOperand::CreateImm(this, Imm, S, AMDGPUOperand::ImmTySwizzle));
3727 
3728     return Ok? MatchOperand_Success : MatchOperand_ParseFail;
3729   } else {
3730     return MatchOperand_NoMatch;
3731   }
3732 }
3733 
3734 bool
3735 AMDGPUOperand::isSwizzle() const {
3736   return isImmTy(ImmTySwizzle);
3737 }
3738 
3739 //===----------------------------------------------------------------------===//
3740 // sopp branch targets
3741 //===----------------------------------------------------------------------===//
3742 
3743 OperandMatchResultTy
3744 AMDGPUAsmParser::parseSOppBrTarget(OperandVector &Operands) {
3745   SMLoc S = Parser.getTok().getLoc();
3746 
3747   switch (getLexer().getKind()) {
3748     default: return MatchOperand_ParseFail;
3749     case AsmToken::Integer: {
3750       int64_t Imm;
3751       if (getParser().parseAbsoluteExpression(Imm))
3752         return MatchOperand_ParseFail;
3753       Operands.push_back(AMDGPUOperand::CreateImm(this, Imm, S));
3754       return MatchOperand_Success;
3755     }
3756 
3757     case AsmToken::Identifier:
3758       Operands.push_back(AMDGPUOperand::CreateExpr(this,
3759           MCSymbolRefExpr::create(getContext().getOrCreateSymbol(
3760                                   Parser.getTok().getString()), getContext()), S));
3761       Parser.Lex();
3762       return MatchOperand_Success;
3763   }
3764 }
3765 
3766 //===----------------------------------------------------------------------===//
3767 // mubuf
3768 //===----------------------------------------------------------------------===//
3769 
3770 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultGLC() const {
3771   return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTyGLC);
3772 }
3773 
3774 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultSLC() const {
3775   return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTySLC);
3776 }
3777 
3778 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultTFE() const {
3779   return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTyTFE);
3780 }
3781 
3782 void AMDGPUAsmParser::cvtMubufImpl(MCInst &Inst,
3783                                const OperandVector &Operands,
3784                                bool IsAtomic, bool IsAtomicReturn) {
3785   OptionalImmIndexMap OptionalIdx;
3786   assert(IsAtomicReturn ? IsAtomic : true);
3787 
3788   for (unsigned i = 1, e = Operands.size(); i != e; ++i) {
3789     AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[i]);
3790 
3791     // Add the register arguments
3792     if (Op.isReg()) {
3793       Op.addRegOperands(Inst, 1);
3794       continue;
3795     }
3796 
3797     // Handle the case where soffset is an immediate
3798     if (Op.isImm() && Op.getImmTy() == AMDGPUOperand::ImmTyNone) {
3799       Op.addImmOperands(Inst, 1);
3800       continue;
3801     }
3802 
3803     // Handle tokens like 'offen' which are sometimes hard-coded into the
3804     // asm string.  There are no MCInst operands for these.
3805     if (Op.isToken()) {
3806       continue;
3807     }
3808     assert(Op.isImm());
3809 
3810     // Handle optional arguments
3811     OptionalIdx[Op.getImmTy()] = i;
3812   }
3813 
3814   // Copy $vdata_in operand and insert as $vdata for MUBUF_Atomic RTN insns.
3815   if (IsAtomicReturn) {
3816     MCInst::iterator I = Inst.begin(); // $vdata_in is always at the beginning.
3817     Inst.insert(I, *I);
3818   }
3819 
3820   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyOffset);
3821   if (!IsAtomic) { // glc is hard-coded.
3822     addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyGLC);
3823   }
3824   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySLC);
3825   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyTFE);
3826 }
3827 
3828 void AMDGPUAsmParser::cvtMtbuf(MCInst &Inst, const OperandVector &Operands) {
3829   OptionalImmIndexMap OptionalIdx;
3830 
3831   for (unsigned i = 1, e = Operands.size(); i != e; ++i) {
3832     AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[i]);
3833 
3834     // Add the register arguments
3835     if (Op.isReg()) {
3836       Op.addRegOperands(Inst, 1);
3837       continue;
3838     }
3839 
3840     // Handle the case where soffset is an immediate
3841     if (Op.isImm() && Op.getImmTy() == AMDGPUOperand::ImmTyNone) {
3842       Op.addImmOperands(Inst, 1);
3843       continue;
3844     }
3845 
3846     // Handle tokens like 'offen' which are sometimes hard-coded into the
3847     // asm string.  There are no MCInst operands for these.
3848     if (Op.isToken()) {
3849       continue;
3850     }
3851     assert(Op.isImm());
3852 
3853     // Handle optional arguments
3854     OptionalIdx[Op.getImmTy()] = i;
3855   }
3856 
3857   addOptionalImmOperand(Inst, Operands, OptionalIdx,
3858                         AMDGPUOperand::ImmTyOffset);
3859   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyDFMT);
3860   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyNFMT);
3861   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyGLC);
3862   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySLC);
3863   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyTFE);
3864 }
3865 
3866 //===----------------------------------------------------------------------===//
3867 // mimg
3868 //===----------------------------------------------------------------------===//
3869 
3870 void AMDGPUAsmParser::cvtMIMG(MCInst &Inst, const OperandVector &Operands,
3871                               bool IsAtomic) {
3872   unsigned I = 1;
3873   const MCInstrDesc &Desc = MII.get(Inst.getOpcode());
3874   for (unsigned J = 0; J < Desc.getNumDefs(); ++J) {
3875     ((AMDGPUOperand &)*Operands[I++]).addRegOperands(Inst, 1);
3876   }
3877 
3878   if (IsAtomic) {
3879     // Add src, same as dst
3880     ((AMDGPUOperand &)*Operands[I]).addRegOperands(Inst, 1);
3881   }
3882 
3883   OptionalImmIndexMap OptionalIdx;
3884 
3885   for (unsigned E = Operands.size(); I != E; ++I) {
3886     AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[I]);
3887 
3888     // Add the register arguments
3889     if (Op.isRegOrImm()) {
3890       Op.addRegOrImmOperands(Inst, 1);
3891       continue;
3892     } else if (Op.isImmModifier()) {
3893       OptionalIdx[Op.getImmTy()] = I;
3894     } else {
3895       llvm_unreachable("unexpected operand type");
3896     }
3897   }
3898 
3899   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyDMask);
3900   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyUNorm);
3901   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyGLC);
3902   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyDA);
3903   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyR128);
3904   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyTFE);
3905   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyLWE);
3906   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySLC);
3907 }
3908 
3909 void AMDGPUAsmParser::cvtMIMGAtomic(MCInst &Inst, const OperandVector &Operands) {
3910   cvtMIMG(Inst, Operands, true);
3911 }
3912 
3913 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultDMask() const {
3914   return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTyDMask);
3915 }
3916 
3917 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultUNorm() const {
3918   return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTyUNorm);
3919 }
3920 
3921 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultDA() const {
3922   return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTyDA);
3923 }
3924 
3925 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultR128() const {
3926   return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTyR128);
3927 }
3928 
3929 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultLWE() const {
3930   return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTyLWE);
3931 }
3932 
3933 //===----------------------------------------------------------------------===//
3934 // smrd
3935 //===----------------------------------------------------------------------===//
3936 
3937 bool AMDGPUOperand::isSMRDOffset8() const {
3938   return isImm() && isUInt<8>(getImm());
3939 }
3940 
3941 bool AMDGPUOperand::isSMRDOffset20() const {
3942   return isImm() && isUInt<20>(getImm());
3943 }
3944 
3945 bool AMDGPUOperand::isSMRDLiteralOffset() const {
3946   // 32-bit literals are only supported on CI and we only want to use them
3947   // when the offset is > 8-bits.
3948   return isImm() && !isUInt<8>(getImm()) && isUInt<32>(getImm());
3949 }
3950 
3951 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultSMRDOffset8() const {
3952   return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTyOffset);
3953 }
3954 
3955 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultSMRDOffset20() const {
3956   return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTyOffset);
3957 }
3958 
3959 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultSMRDLiteralOffset() const {
3960   return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTyOffset);
3961 }
3962 
3963 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultOffsetU12() const {
3964   return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTyOffset);
3965 }
3966 
3967 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultOffsetS13() const {
3968   return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTyOffset);
3969 }
3970 
3971 //===----------------------------------------------------------------------===//
3972 // vop3
3973 //===----------------------------------------------------------------------===//
3974 
3975 static bool ConvertOmodMul(int64_t &Mul) {
3976   if (Mul != 1 && Mul != 2 && Mul != 4)
3977     return false;
3978 
3979   Mul >>= 1;
3980   return true;
3981 }
3982 
3983 static bool ConvertOmodDiv(int64_t &Div) {
3984   if (Div == 1) {
3985     Div = 0;
3986     return true;
3987   }
3988 
3989   if (Div == 2) {
3990     Div = 3;
3991     return true;
3992   }
3993 
3994   return false;
3995 }
3996 
3997 static bool ConvertBoundCtrl(int64_t &BoundCtrl) {
3998   if (BoundCtrl == 0) {
3999     BoundCtrl = 1;
4000     return true;
4001   }
4002 
4003   if (BoundCtrl == -1) {
4004     BoundCtrl = 0;
4005     return true;
4006   }
4007 
4008   return false;
4009 }
4010 
4011 // Note: the order in this table matches the order of operands in AsmString.
4012 static const OptionalOperand AMDGPUOptionalOperandTable[] = {
4013   {"offen",   AMDGPUOperand::ImmTyOffen, true, nullptr},
4014   {"idxen",   AMDGPUOperand::ImmTyIdxen, true, nullptr},
4015   {"addr64",  AMDGPUOperand::ImmTyAddr64, true, nullptr},
4016   {"offset0", AMDGPUOperand::ImmTyOffset0, false, nullptr},
4017   {"offset1", AMDGPUOperand::ImmTyOffset1, false, nullptr},
4018   {"gds",     AMDGPUOperand::ImmTyGDS, true, nullptr},
4019   {"offset",  AMDGPUOperand::ImmTyOffset, false, nullptr},
4020   {"dfmt",    AMDGPUOperand::ImmTyDFMT, false, nullptr},
4021   {"nfmt",    AMDGPUOperand::ImmTyNFMT, false, nullptr},
4022   {"glc",     AMDGPUOperand::ImmTyGLC, true, nullptr},
4023   {"slc",     AMDGPUOperand::ImmTySLC, true, nullptr},
4024   {"tfe",     AMDGPUOperand::ImmTyTFE, true, nullptr},
4025   {"clamp",   AMDGPUOperand::ImmTyClampSI, true, nullptr},
4026   {"omod",    AMDGPUOperand::ImmTyOModSI, false, ConvertOmodMul},
4027   {"unorm",   AMDGPUOperand::ImmTyUNorm, true, nullptr},
4028   {"da",      AMDGPUOperand::ImmTyDA,    true, nullptr},
4029   {"r128",    AMDGPUOperand::ImmTyR128,  true, nullptr},
4030   {"lwe",     AMDGPUOperand::ImmTyLWE,   true, nullptr},
4031   {"dmask",   AMDGPUOperand::ImmTyDMask, false, nullptr},
4032   {"row_mask",   AMDGPUOperand::ImmTyDppRowMask, false, nullptr},
4033   {"bank_mask",  AMDGPUOperand::ImmTyDppBankMask, false, nullptr},
4034   {"bound_ctrl", AMDGPUOperand::ImmTyDppBoundCtrl, false, ConvertBoundCtrl},
4035   {"dst_sel",    AMDGPUOperand::ImmTySdwaDstSel, false, nullptr},
4036   {"src0_sel",   AMDGPUOperand::ImmTySdwaSrc0Sel, false, nullptr},
4037   {"src1_sel",   AMDGPUOperand::ImmTySdwaSrc1Sel, false, nullptr},
4038   {"dst_unused", AMDGPUOperand::ImmTySdwaDstUnused, false, nullptr},
4039   {"compr", AMDGPUOperand::ImmTyExpCompr, true, nullptr },
4040   {"vm", AMDGPUOperand::ImmTyExpVM, true, nullptr},
4041   {"op_sel", AMDGPUOperand::ImmTyOpSel, false, nullptr},
4042   {"op_sel_hi", AMDGPUOperand::ImmTyOpSelHi, false, nullptr},
4043   {"neg_lo", AMDGPUOperand::ImmTyNegLo, false, nullptr},
4044   {"neg_hi", AMDGPUOperand::ImmTyNegHi, false, nullptr}
4045 };
4046 
4047 OperandMatchResultTy AMDGPUAsmParser::parseOptionalOperand(OperandVector &Operands) {
4048   OperandMatchResultTy res;
4049   for (const OptionalOperand &Op : AMDGPUOptionalOperandTable) {
4050     // try to parse any optional operand here
4051     if (Op.IsBit) {
4052       res = parseNamedBit(Op.Name, Operands, Op.Type);
4053     } else if (Op.Type == AMDGPUOperand::ImmTyOModSI) {
4054       res = parseOModOperand(Operands);
4055     } else if (Op.Type == AMDGPUOperand::ImmTySdwaDstSel ||
4056                Op.Type == AMDGPUOperand::ImmTySdwaSrc0Sel ||
4057                Op.Type == AMDGPUOperand::ImmTySdwaSrc1Sel) {
4058       res = parseSDWASel(Operands, Op.Name, Op.Type);
4059     } else if (Op.Type == AMDGPUOperand::ImmTySdwaDstUnused) {
4060       res = parseSDWADstUnused(Operands);
4061     } else if (Op.Type == AMDGPUOperand::ImmTyOpSel ||
4062                Op.Type == AMDGPUOperand::ImmTyOpSelHi ||
4063                Op.Type == AMDGPUOperand::ImmTyNegLo ||
4064                Op.Type == AMDGPUOperand::ImmTyNegHi) {
4065       res = parseOperandArrayWithPrefix(Op.Name, Operands, Op.Type,
4066                                         Op.ConvertResult);
4067     } else {
4068       res = parseIntWithPrefix(Op.Name, Operands, Op.Type, Op.ConvertResult);
4069     }
4070     if (res != MatchOperand_NoMatch) {
4071       return res;
4072     }
4073   }
4074   return MatchOperand_NoMatch;
4075 }
4076 
4077 OperandMatchResultTy AMDGPUAsmParser::parseOModOperand(OperandVector &Operands) {
4078   StringRef Name = Parser.getTok().getString();
4079   if (Name == "mul") {
4080     return parseIntWithPrefix("mul", Operands,
4081                               AMDGPUOperand::ImmTyOModSI, ConvertOmodMul);
4082   }
4083 
4084   if (Name == "div") {
4085     return parseIntWithPrefix("div", Operands,
4086                               AMDGPUOperand::ImmTyOModSI, ConvertOmodDiv);
4087   }
4088 
4089   return MatchOperand_NoMatch;
4090 }
4091 
4092 void AMDGPUAsmParser::cvtVOP3OpSel(MCInst &Inst, const OperandVector &Operands) {
4093   cvtVOP3P(Inst, Operands);
4094 
4095   int Opc = Inst.getOpcode();
4096 
4097   int SrcNum;
4098   const int Ops[] = { AMDGPU::OpName::src0,
4099                       AMDGPU::OpName::src1,
4100                       AMDGPU::OpName::src2 };
4101   for (SrcNum = 0;
4102        SrcNum < 3 && AMDGPU::getNamedOperandIdx(Opc, Ops[SrcNum]) != -1;
4103        ++SrcNum);
4104   assert(SrcNum > 0);
4105 
4106   int OpSelIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::op_sel);
4107   unsigned OpSel = Inst.getOperand(OpSelIdx).getImm();
4108 
4109   if ((OpSel & (1 << SrcNum)) != 0) {
4110     int ModIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src0_modifiers);
4111     uint32_t ModVal = Inst.getOperand(ModIdx).getImm();
4112     Inst.getOperand(ModIdx).setImm(ModVal | SISrcMods::DST_OP_SEL);
4113   }
4114 }
4115 
4116 static bool isRegOrImmWithInputMods(const MCInstrDesc &Desc, unsigned OpNum) {
4117       // 1. This operand is input modifiers
4118   return Desc.OpInfo[OpNum].OperandType == AMDGPU::OPERAND_INPUT_MODS
4119       // 2. This is not last operand
4120       && Desc.NumOperands > (OpNum + 1)
4121       // 3. Next operand is register class
4122       && Desc.OpInfo[OpNum + 1].RegClass != -1
4123       // 4. Next register is not tied to any other operand
4124       && Desc.getOperandConstraint(OpNum + 1, MCOI::OperandConstraint::TIED_TO) == -1;
4125 }
4126 
4127 void AMDGPUAsmParser::cvtVOP3(MCInst &Inst, const OperandVector &Operands,
4128                               OptionalImmIndexMap &OptionalIdx) {
4129   unsigned Opc = Inst.getOpcode();
4130 
4131   unsigned I = 1;
4132   const MCInstrDesc &Desc = MII.get(Inst.getOpcode());
4133   for (unsigned J = 0; J < Desc.getNumDefs(); ++J) {
4134     ((AMDGPUOperand &)*Operands[I++]).addRegOperands(Inst, 1);
4135   }
4136 
4137   if (AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src0_modifiers) != -1) {
4138     // This instruction has src modifiers
4139     for (unsigned E = Operands.size(); I != E; ++I) {
4140       AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[I]);
4141       if (isRegOrImmWithInputMods(Desc, Inst.getNumOperands())) {
4142         Op.addRegOrImmWithFPInputModsOperands(Inst, 2);
4143       } else if (Op.isImmModifier()) {
4144         OptionalIdx[Op.getImmTy()] = I;
4145       } else if (Op.isRegOrImm()) {
4146         Op.addRegOrImmOperands(Inst, 1);
4147       } else {
4148         llvm_unreachable("unhandled operand type");
4149       }
4150     }
4151   } else {
4152     // No src modifiers
4153     for (unsigned E = Operands.size(); I != E; ++I) {
4154       AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[I]);
4155       if (Op.isMod()) {
4156         OptionalIdx[Op.getImmTy()] = I;
4157       } else {
4158         Op.addRegOrImmOperands(Inst, 1);
4159       }
4160     }
4161   }
4162 
4163   if (AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::clamp) != -1) {
4164     addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyClampSI);
4165   }
4166 
4167   if (AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::omod) != -1) {
4168     addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyOModSI);
4169   }
4170 
4171   // special case v_mac_{f16, f32}:
4172   // it has src2 register operand that is tied to dst operand
4173   // we don't allow modifiers for this operand in assembler so src2_modifiers
4174   // should be 0
4175   if (Opc == AMDGPU::V_MAC_F32_e64_si || Opc == AMDGPU::V_MAC_F32_e64_vi ||
4176       Opc == AMDGPU::V_MAC_F16_e64_vi) {
4177     auto it = Inst.begin();
4178     std::advance(it, AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src2_modifiers));
4179     it = Inst.insert(it, MCOperand::createImm(0)); // no modifiers for src2
4180     ++it;
4181     Inst.insert(it, Inst.getOperand(0)); // src2 = dst
4182   }
4183 }
4184 
4185 void AMDGPUAsmParser::cvtVOP3(MCInst &Inst, const OperandVector &Operands) {
4186   OptionalImmIndexMap OptionalIdx;
4187   cvtVOP3(Inst, Operands, OptionalIdx);
4188 }
4189 
4190 void AMDGPUAsmParser::cvtVOP3P(MCInst &Inst, const OperandVector &Operands) {
4191   OptionalImmIndexMap OptIdx;
4192 
4193   cvtVOP3(Inst, Operands, OptIdx);
4194 
4195   // FIXME: This is messy. Parse the modifiers as if it was a normal VOP3
4196   // instruction, and then figure out where to actually put the modifiers
4197   int Opc = Inst.getOpcode();
4198 
4199   addOptionalImmOperand(Inst, Operands, OptIdx, AMDGPUOperand::ImmTyOpSel);
4200 
4201   int OpSelHiIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::op_sel_hi);
4202   if (OpSelHiIdx != -1) {
4203     addOptionalImmOperand(Inst, Operands, OptIdx, AMDGPUOperand::ImmTyOpSelHi, -1);
4204   }
4205 
4206   int NegLoIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::neg_lo);
4207   if (NegLoIdx != -1) {
4208     addOptionalImmOperand(Inst, Operands, OptIdx, AMDGPUOperand::ImmTyNegLo);
4209     addOptionalImmOperand(Inst, Operands, OptIdx, AMDGPUOperand::ImmTyNegHi);
4210   }
4211 
4212   const int Ops[] = { AMDGPU::OpName::src0,
4213                       AMDGPU::OpName::src1,
4214                       AMDGPU::OpName::src2 };
4215   const int ModOps[] = { AMDGPU::OpName::src0_modifiers,
4216                          AMDGPU::OpName::src1_modifiers,
4217                          AMDGPU::OpName::src2_modifiers };
4218 
4219   int OpSelIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::op_sel);
4220 
4221   unsigned OpSel = Inst.getOperand(OpSelIdx).getImm();
4222   unsigned OpSelHi = 0;
4223   unsigned NegLo = 0;
4224   unsigned NegHi = 0;
4225 
4226   if (OpSelHiIdx != -1) {
4227     OpSelHi = Inst.getOperand(OpSelHiIdx).getImm();
4228   }
4229 
4230   if (NegLoIdx != -1) {
4231     int NegHiIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::neg_hi);
4232     NegLo = Inst.getOperand(NegLoIdx).getImm();
4233     NegHi = Inst.getOperand(NegHiIdx).getImm();
4234   }
4235 
4236   for (int J = 0; J < 3; ++J) {
4237     int OpIdx = AMDGPU::getNamedOperandIdx(Opc, Ops[J]);
4238     if (OpIdx == -1)
4239       break;
4240 
4241     uint32_t ModVal = 0;
4242 
4243     if ((OpSel & (1 << J)) != 0)
4244       ModVal |= SISrcMods::OP_SEL_0;
4245 
4246     if ((OpSelHi & (1 << J)) != 0)
4247       ModVal |= SISrcMods::OP_SEL_1;
4248 
4249     if ((NegLo & (1 << J)) != 0)
4250       ModVal |= SISrcMods::NEG;
4251 
4252     if ((NegHi & (1 << J)) != 0)
4253       ModVal |= SISrcMods::NEG_HI;
4254 
4255     int ModIdx = AMDGPU::getNamedOperandIdx(Opc, ModOps[J]);
4256 
4257     Inst.getOperand(ModIdx).setImm(Inst.getOperand(ModIdx).getImm() | ModVal);
4258   }
4259 }
4260 
4261 //===----------------------------------------------------------------------===//
4262 // dpp
4263 //===----------------------------------------------------------------------===//
4264 
4265 bool AMDGPUOperand::isDPPCtrl() const {
4266   bool result = isImm() && getImmTy() == ImmTyDppCtrl && isUInt<9>(getImm());
4267   if (result) {
4268     int64_t Imm = getImm();
4269     return ((Imm >= 0x000) && (Imm <= 0x0ff)) ||
4270            ((Imm >= 0x101) && (Imm <= 0x10f)) ||
4271            ((Imm >= 0x111) && (Imm <= 0x11f)) ||
4272            ((Imm >= 0x121) && (Imm <= 0x12f)) ||
4273            (Imm == 0x130) ||
4274            (Imm == 0x134) ||
4275            (Imm == 0x138) ||
4276            (Imm == 0x13c) ||
4277            (Imm == 0x140) ||
4278            (Imm == 0x141) ||
4279            (Imm == 0x142) ||
4280            (Imm == 0x143);
4281   }
4282   return false;
4283 }
4284 
4285 bool AMDGPUOperand::isGPRIdxMode() const {
4286   return isImm() && isUInt<4>(getImm());
4287 }
4288 
4289 bool AMDGPUOperand::isS16Imm() const {
4290   return isImm() && (isInt<16>(getImm()) || isUInt<16>(getImm()));
4291 }
4292 
4293 bool AMDGPUOperand::isU16Imm() const {
4294   return isImm() && isUInt<16>(getImm());
4295 }
4296 
4297 OperandMatchResultTy
4298 AMDGPUAsmParser::parseDPPCtrl(OperandVector &Operands) {
4299   SMLoc S = Parser.getTok().getLoc();
4300   StringRef Prefix;
4301   int64_t Int;
4302 
4303   if (getLexer().getKind() == AsmToken::Identifier) {
4304     Prefix = Parser.getTok().getString();
4305   } else {
4306     return MatchOperand_NoMatch;
4307   }
4308 
4309   if (Prefix == "row_mirror") {
4310     Int = 0x140;
4311     Parser.Lex();
4312   } else if (Prefix == "row_half_mirror") {
4313     Int = 0x141;
4314     Parser.Lex();
4315   } else {
4316     // Check to prevent parseDPPCtrlOps from eating invalid tokens
4317     if (Prefix != "quad_perm"
4318         && Prefix != "row_shl"
4319         && Prefix != "row_shr"
4320         && Prefix != "row_ror"
4321         && Prefix != "wave_shl"
4322         && Prefix != "wave_rol"
4323         && Prefix != "wave_shr"
4324         && Prefix != "wave_ror"
4325         && Prefix != "row_bcast") {
4326       return MatchOperand_NoMatch;
4327     }
4328 
4329     Parser.Lex();
4330     if (getLexer().isNot(AsmToken::Colon))
4331       return MatchOperand_ParseFail;
4332 
4333     if (Prefix == "quad_perm") {
4334       // quad_perm:[%d,%d,%d,%d]
4335       Parser.Lex();
4336       if (getLexer().isNot(AsmToken::LBrac))
4337         return MatchOperand_ParseFail;
4338       Parser.Lex();
4339 
4340       if (getParser().parseAbsoluteExpression(Int) || !(0 <= Int && Int <=3))
4341         return MatchOperand_ParseFail;
4342 
4343       for (int i = 0; i < 3; ++i) {
4344         if (getLexer().isNot(AsmToken::Comma))
4345           return MatchOperand_ParseFail;
4346         Parser.Lex();
4347 
4348         int64_t Temp;
4349         if (getParser().parseAbsoluteExpression(Temp) || !(0 <= Temp && Temp <=3))
4350           return MatchOperand_ParseFail;
4351         const int shift = i*2 + 2;
4352         Int += (Temp << shift);
4353       }
4354 
4355       if (getLexer().isNot(AsmToken::RBrac))
4356         return MatchOperand_ParseFail;
4357       Parser.Lex();
4358 
4359     } else {
4360       // sel:%d
4361       Parser.Lex();
4362       if (getParser().parseAbsoluteExpression(Int))
4363         return MatchOperand_ParseFail;
4364 
4365       if (Prefix == "row_shl" && 1 <= Int && Int <= 15) {
4366         Int |= 0x100;
4367       } else if (Prefix == "row_shr" && 1 <= Int && Int <= 15) {
4368         Int |= 0x110;
4369       } else if (Prefix == "row_ror" && 1 <= Int && Int <= 15) {
4370         Int |= 0x120;
4371       } else if (Prefix == "wave_shl" && 1 == Int) {
4372         Int = 0x130;
4373       } else if (Prefix == "wave_rol" && 1 == Int) {
4374         Int = 0x134;
4375       } else if (Prefix == "wave_shr" && 1 == Int) {
4376         Int = 0x138;
4377       } else if (Prefix == "wave_ror" && 1 == Int) {
4378         Int = 0x13C;
4379       } else if (Prefix == "row_bcast") {
4380         if (Int == 15) {
4381           Int = 0x142;
4382         } else if (Int == 31) {
4383           Int = 0x143;
4384         } else {
4385           return MatchOperand_ParseFail;
4386         }
4387       } else {
4388         return MatchOperand_ParseFail;
4389       }
4390     }
4391   }
4392 
4393   Operands.push_back(AMDGPUOperand::CreateImm(this, Int, S, AMDGPUOperand::ImmTyDppCtrl));
4394   return MatchOperand_Success;
4395 }
4396 
4397 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultRowMask() const {
4398   return AMDGPUOperand::CreateImm(this, 0xf, SMLoc(), AMDGPUOperand::ImmTyDppRowMask);
4399 }
4400 
4401 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultBankMask() const {
4402   return AMDGPUOperand::CreateImm(this, 0xf, SMLoc(), AMDGPUOperand::ImmTyDppBankMask);
4403 }
4404 
4405 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultBoundCtrl() const {
4406   return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTyDppBoundCtrl);
4407 }
4408 
4409 void AMDGPUAsmParser::cvtDPP(MCInst &Inst, const OperandVector &Operands) {
4410   OptionalImmIndexMap OptionalIdx;
4411 
4412   unsigned I = 1;
4413   const MCInstrDesc &Desc = MII.get(Inst.getOpcode());
4414   for (unsigned J = 0; J < Desc.getNumDefs(); ++J) {
4415     ((AMDGPUOperand &)*Operands[I++]).addRegOperands(Inst, 1);
4416   }
4417 
4418   for (unsigned E = Operands.size(); I != E; ++I) {
4419     AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[I]);
4420     // Add the register arguments
4421     if (Op.isReg() && Op.Reg.RegNo == AMDGPU::VCC) {
4422       // VOP2b (v_add_u32, v_sub_u32 ...) dpp use "vcc" token.
4423       // Skip it.
4424       continue;
4425     } if (isRegOrImmWithInputMods(Desc, Inst.getNumOperands())) {
4426       Op.addRegWithFPInputModsOperands(Inst, 2);
4427     } else if (Op.isDPPCtrl()) {
4428       Op.addImmOperands(Inst, 1);
4429     } else if (Op.isImm()) {
4430       // Handle optional arguments
4431       OptionalIdx[Op.getImmTy()] = I;
4432     } else {
4433       llvm_unreachable("Invalid operand type");
4434     }
4435   }
4436 
4437   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyDppRowMask, 0xf);
4438   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyDppBankMask, 0xf);
4439   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyDppBoundCtrl);
4440 
4441   // special case v_mac_{f16, f32}:
4442   // it has src2 register operand that is tied to dst operand
4443   if (Inst.getOpcode() == AMDGPU::V_MAC_F32_dpp ||
4444       Inst.getOpcode() == AMDGPU::V_MAC_F16_dpp) {
4445     auto it = Inst.begin();
4446     std::advance(
4447         it, AMDGPU::getNamedOperandIdx(Inst.getOpcode(), AMDGPU::OpName::src2));
4448     Inst.insert(it, Inst.getOperand(0)); // src2 = dst
4449   }
4450 }
4451 
4452 //===----------------------------------------------------------------------===//
4453 // sdwa
4454 //===----------------------------------------------------------------------===//
4455 
4456 OperandMatchResultTy
4457 AMDGPUAsmParser::parseSDWASel(OperandVector &Operands, StringRef Prefix,
4458                               AMDGPUOperand::ImmTy Type) {
4459   using namespace llvm::AMDGPU::SDWA;
4460 
4461   SMLoc S = Parser.getTok().getLoc();
4462   StringRef Value;
4463   OperandMatchResultTy res;
4464 
4465   res = parseStringWithPrefix(Prefix, Value);
4466   if (res != MatchOperand_Success) {
4467     return res;
4468   }
4469 
4470   int64_t Int;
4471   Int = StringSwitch<int64_t>(Value)
4472         .Case("BYTE_0", SdwaSel::BYTE_0)
4473         .Case("BYTE_1", SdwaSel::BYTE_1)
4474         .Case("BYTE_2", SdwaSel::BYTE_2)
4475         .Case("BYTE_3", SdwaSel::BYTE_3)
4476         .Case("WORD_0", SdwaSel::WORD_0)
4477         .Case("WORD_1", SdwaSel::WORD_1)
4478         .Case("DWORD", SdwaSel::DWORD)
4479         .Default(0xffffffff);
4480   Parser.Lex(); // eat last token
4481 
4482   if (Int == 0xffffffff) {
4483     return MatchOperand_ParseFail;
4484   }
4485 
4486   Operands.push_back(AMDGPUOperand::CreateImm(this, Int, S, Type));
4487   return MatchOperand_Success;
4488 }
4489 
4490 OperandMatchResultTy
4491 AMDGPUAsmParser::parseSDWADstUnused(OperandVector &Operands) {
4492   using namespace llvm::AMDGPU::SDWA;
4493 
4494   SMLoc S = Parser.getTok().getLoc();
4495   StringRef Value;
4496   OperandMatchResultTy res;
4497 
4498   res = parseStringWithPrefix("dst_unused", Value);
4499   if (res != MatchOperand_Success) {
4500     return res;
4501   }
4502 
4503   int64_t Int;
4504   Int = StringSwitch<int64_t>(Value)
4505         .Case("UNUSED_PAD", DstUnused::UNUSED_PAD)
4506         .Case("UNUSED_SEXT", DstUnused::UNUSED_SEXT)
4507         .Case("UNUSED_PRESERVE", DstUnused::UNUSED_PRESERVE)
4508         .Default(0xffffffff);
4509   Parser.Lex(); // eat last token
4510 
4511   if (Int == 0xffffffff) {
4512     return MatchOperand_ParseFail;
4513   }
4514 
4515   Operands.push_back(AMDGPUOperand::CreateImm(this, Int, S, AMDGPUOperand::ImmTySdwaDstUnused));
4516   return MatchOperand_Success;
4517 }
4518 
4519 void AMDGPUAsmParser::cvtSdwaVOP1(MCInst &Inst, const OperandVector &Operands) {
4520   cvtSDWA(Inst, Operands, SIInstrFlags::VOP1);
4521 }
4522 
4523 void AMDGPUAsmParser::cvtSdwaVOP2(MCInst &Inst, const OperandVector &Operands) {
4524   cvtSDWA(Inst, Operands, SIInstrFlags::VOP2);
4525 }
4526 
4527 void AMDGPUAsmParser::cvtSdwaVOP2b(MCInst &Inst, const OperandVector &Operands) {
4528   cvtSDWA(Inst, Operands, SIInstrFlags::VOP2, true);
4529 }
4530 
4531 void AMDGPUAsmParser::cvtSdwaVOPC(MCInst &Inst, const OperandVector &Operands) {
4532   cvtSDWA(Inst, Operands, SIInstrFlags::VOPC, isVI());
4533 }
4534 
4535 void AMDGPUAsmParser::cvtSDWA(MCInst &Inst, const OperandVector &Operands,
4536                               uint64_t BasicInstType, bool skipVcc) {
4537   using namespace llvm::AMDGPU::SDWA;
4538   OptionalImmIndexMap OptionalIdx;
4539   bool skippedVcc = false;
4540 
4541   unsigned I = 1;
4542   const MCInstrDesc &Desc = MII.get(Inst.getOpcode());
4543   for (unsigned J = 0; J < Desc.getNumDefs(); ++J) {
4544     ((AMDGPUOperand &)*Operands[I++]).addRegOperands(Inst, 1);
4545   }
4546 
4547   for (unsigned E = Operands.size(); I != E; ++I) {
4548     AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[I]);
4549     if (skipVcc && !skippedVcc && Op.isReg() && Op.Reg.RegNo == AMDGPU::VCC) {
4550       // VOP2b (v_add_u32, v_sub_u32 ...) sdwa use "vcc" token as dst.
4551       // Skip it if it's 2nd (e.g. v_add_i32_sdwa v1, vcc, v2, v3)
4552       // or 4th (v_addc_u32_sdwa v1, vcc, v2, v3, vcc) operand.
4553       // Skip VCC only if we didn't skip it on previous iteration.
4554       if (BasicInstType == SIInstrFlags::VOP2 &&
4555           (Inst.getNumOperands() == 1 || Inst.getNumOperands() == 5)) {
4556         skippedVcc = true;
4557         continue;
4558       } else if (BasicInstType == SIInstrFlags::VOPC &&
4559                  Inst.getNumOperands() == 0) {
4560         skippedVcc = true;
4561         continue;
4562       }
4563     }
4564     if (isRegOrImmWithInputMods(Desc, Inst.getNumOperands())) {
4565       Op.addRegWithInputModsOperands(Inst, 2);
4566     } else if (Op.isImm()) {
4567       // Handle optional arguments
4568       OptionalIdx[Op.getImmTy()] = I;
4569     } else {
4570       llvm_unreachable("Invalid operand type");
4571     }
4572     skippedVcc = false;
4573   }
4574 
4575   if (Inst.getOpcode() != AMDGPU::V_NOP_sdwa_gfx9 &&
4576       Inst.getOpcode() != AMDGPU::V_NOP_sdwa_vi) {
4577     // v_nop_sdwa_sdwa_vi/gfx9 has no optional sdwa arguments
4578     switch (BasicInstType) {
4579     case SIInstrFlags::VOP1:
4580       addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyClampSI, 0);
4581       if (AMDGPU::getNamedOperandIdx(Inst.getOpcode(), AMDGPU::OpName::omod) != -1) {
4582         addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyOModSI, 0);
4583       }
4584       addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySdwaDstSel, SdwaSel::DWORD);
4585       addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySdwaDstUnused, DstUnused::UNUSED_PRESERVE);
4586       addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySdwaSrc0Sel, SdwaSel::DWORD);
4587       break;
4588 
4589     case SIInstrFlags::VOP2:
4590       addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyClampSI, 0);
4591       if (AMDGPU::getNamedOperandIdx(Inst.getOpcode(), AMDGPU::OpName::omod) != -1) {
4592         addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyOModSI, 0);
4593       }
4594       addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySdwaDstSel, SdwaSel::DWORD);
4595       addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySdwaDstUnused, DstUnused::UNUSED_PRESERVE);
4596       addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySdwaSrc0Sel, SdwaSel::DWORD);
4597       addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySdwaSrc1Sel, SdwaSel::DWORD);
4598       break;
4599 
4600     case SIInstrFlags::VOPC:
4601       addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyClampSI, 0);
4602       addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySdwaSrc0Sel, SdwaSel::DWORD);
4603       addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySdwaSrc1Sel, SdwaSel::DWORD);
4604       break;
4605 
4606     default:
4607       llvm_unreachable("Invalid instruction type. Only VOP1, VOP2 and VOPC allowed");
4608     }
4609   }
4610 
4611   // special case v_mac_{f16, f32}:
4612   // it has src2 register operand that is tied to dst operand
4613   if (Inst.getOpcode() == AMDGPU::V_MAC_F32_sdwa_vi ||
4614       Inst.getOpcode() == AMDGPU::V_MAC_F16_sdwa_vi)  {
4615     auto it = Inst.begin();
4616     std::advance(
4617       it, AMDGPU::getNamedOperandIdx(Inst.getOpcode(), AMDGPU::OpName::src2));
4618     Inst.insert(it, Inst.getOperand(0)); // src2 = dst
4619   }
4620 }
4621 
4622 /// Force static initialization.
4623 extern "C" void LLVMInitializeAMDGPUAsmParser() {
4624   RegisterMCAsmParser<AMDGPUAsmParser> A(getTheAMDGPUTarget());
4625   RegisterMCAsmParser<AMDGPUAsmParser> B(getTheGCNTarget());
4626 }
4627 
4628 #define GET_REGISTER_MATCHER
4629 #define GET_MATCHER_IMPLEMENTATION
4630 #include "AMDGPUGenAsmMatcher.inc"
4631 
4632 // This fuction should be defined after auto-generated include so that we have
4633 // MatchClassKind enum defined
4634 unsigned AMDGPUAsmParser::validateTargetOperandClass(MCParsedAsmOperand &Op,
4635                                                      unsigned Kind) {
4636   // Tokens like "glc" would be parsed as immediate operands in ParseOperand().
4637   // But MatchInstructionImpl() expects to meet token and fails to validate
4638   // operand. This method checks if we are given immediate operand but expect to
4639   // get corresponding token.
4640   AMDGPUOperand &Operand = (AMDGPUOperand&)Op;
4641   switch (Kind) {
4642   case MCK_addr64:
4643     return Operand.isAddr64() ? Match_Success : Match_InvalidOperand;
4644   case MCK_gds:
4645     return Operand.isGDS() ? Match_Success : Match_InvalidOperand;
4646   case MCK_glc:
4647     return Operand.isGLC() ? Match_Success : Match_InvalidOperand;
4648   case MCK_idxen:
4649     return Operand.isIdxen() ? Match_Success : Match_InvalidOperand;
4650   case MCK_offen:
4651     return Operand.isOffen() ? Match_Success : Match_InvalidOperand;
4652   case MCK_SSrcB32:
4653     // When operands have expression values, they will return true for isToken,
4654     // because it is not possible to distinguish between a token and an
4655     // expression at parse time. MatchInstructionImpl() will always try to
4656     // match an operand as a token, when isToken returns true, and when the
4657     // name of the expression is not a valid token, the match will fail,
4658     // so we need to handle it here.
4659     return Operand.isSSrcB32() ? Match_Success : Match_InvalidOperand;
4660   case MCK_SSrcF32:
4661     return Operand.isSSrcF32() ? Match_Success : Match_InvalidOperand;
4662   case MCK_SoppBrTarget:
4663     return Operand.isSoppBrTarget() ? Match_Success : Match_InvalidOperand;
4664   case MCK_VReg32OrOff:
4665     return Operand.isVReg32OrOff() ? Match_Success : Match_InvalidOperand;
4666   case MCK_InterpSlot:
4667     return Operand.isInterpSlot() ? Match_Success : Match_InvalidOperand;
4668   case MCK_Attr:
4669     return Operand.isInterpAttr() ? Match_Success : Match_InvalidOperand;
4670   case MCK_AttrChan:
4671     return Operand.isAttrChan() ? Match_Success : Match_InvalidOperand;
4672   default:
4673     return Match_InvalidOperand;
4674   }
4675 }
4676