1 //===- AMDGPUAsmParser.cpp - Parse SI asm to MCInst instructions ----------===//
2 //
3 //                     The LLVM Compiler Infrastructure
4 //
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
7 //
8 //===----------------------------------------------------------------------===//
9 
10 #include "AMDGPU.h"
11 #include "AMDKernelCodeT.h"
12 #include "MCTargetDesc/AMDGPUMCTargetDesc.h"
13 #include "MCTargetDesc/AMDGPUTargetStreamer.h"
14 #include "SIDefines.h"
15 #include "Utils/AMDGPUAsmUtils.h"
16 #include "Utils/AMDGPUBaseInfo.h"
17 #include "Utils/AMDKernelCodeTUtils.h"
18 #include "llvm/ADT/APFloat.h"
19 #include "llvm/ADT/APInt.h"
20 #include "llvm/ADT/ArrayRef.h"
21 #include "llvm/ADT/STLExtras.h"
22 #include "llvm/ADT/SmallBitVector.h"
23 #include "llvm/ADT/SmallString.h"
24 #include "llvm/ADT/StringRef.h"
25 #include "llvm/ADT/StringSwitch.h"
26 #include "llvm/ADT/Twine.h"
27 #include "llvm/BinaryFormat/ELF.h"
28 #include "llvm/CodeGen/MachineValueType.h"
29 #include "llvm/MC/MCAsmInfo.h"
30 #include "llvm/MC/MCContext.h"
31 #include "llvm/MC/MCExpr.h"
32 #include "llvm/MC/MCInst.h"
33 #include "llvm/MC/MCInstrDesc.h"
34 #include "llvm/MC/MCInstrInfo.h"
35 #include "llvm/MC/MCParser/MCAsmLexer.h"
36 #include "llvm/MC/MCParser/MCAsmParser.h"
37 #include "llvm/MC/MCParser/MCAsmParserExtension.h"
38 #include "llvm/MC/MCParser/MCParsedAsmOperand.h"
39 #include "llvm/MC/MCParser/MCTargetAsmParser.h"
40 #include "llvm/MC/MCRegisterInfo.h"
41 #include "llvm/MC/MCStreamer.h"
42 #include "llvm/MC/MCSubtargetInfo.h"
43 #include "llvm/MC/MCSymbol.h"
44 #include "llvm/Support/AMDGPUMetadata.h"
45 #include "llvm/Support/Casting.h"
46 #include "llvm/Support/Compiler.h"
47 #include "llvm/Support/ErrorHandling.h"
48 #include "llvm/Support/MathExtras.h"
49 #include "llvm/Support/SMLoc.h"
50 #include "llvm/Support/TargetRegistry.h"
51 #include "llvm/Support/raw_ostream.h"
52 #include <algorithm>
53 #include <cassert>
54 #include <cstdint>
55 #include <cstring>
56 #include <iterator>
57 #include <map>
58 #include <memory>
59 #include <string>
60 
61 using namespace llvm;
62 using namespace llvm::AMDGPU;
63 
64 namespace {
65 
66 class AMDGPUAsmParser;
67 
68 enum RegisterKind { IS_UNKNOWN, IS_VGPR, IS_SGPR, IS_TTMP, IS_SPECIAL };
69 
70 //===----------------------------------------------------------------------===//
71 // Operand
72 //===----------------------------------------------------------------------===//
73 
74 class AMDGPUOperand : public MCParsedAsmOperand {
75   enum KindTy {
76     Token,
77     Immediate,
78     Register,
79     Expression
80   } Kind;
81 
82   SMLoc StartLoc, EndLoc;
83   const AMDGPUAsmParser *AsmParser;
84 
85 public:
86   AMDGPUOperand(KindTy Kind_, const AMDGPUAsmParser *AsmParser_)
87     : MCParsedAsmOperand(), Kind(Kind_), AsmParser(AsmParser_) {}
88 
89   using Ptr = std::unique_ptr<AMDGPUOperand>;
90 
91   struct Modifiers {
92     bool Abs = false;
93     bool Neg = false;
94     bool Sext = false;
95 
96     bool hasFPModifiers() const { return Abs || Neg; }
97     bool hasIntModifiers() const { return Sext; }
98     bool hasModifiers() const { return hasFPModifiers() || hasIntModifiers(); }
99 
100     int64_t getFPModifiersOperand() const {
101       int64_t Operand = 0;
102       Operand |= Abs ? SISrcMods::ABS : 0;
103       Operand |= Neg ? SISrcMods::NEG : 0;
104       return Operand;
105     }
106 
107     int64_t getIntModifiersOperand() const {
108       int64_t Operand = 0;
109       Operand |= Sext ? SISrcMods::SEXT : 0;
110       return Operand;
111     }
112 
113     int64_t getModifiersOperand() const {
114       assert(!(hasFPModifiers() && hasIntModifiers())
115            && "fp and int modifiers should not be used simultaneously");
116       if (hasFPModifiers()) {
117         return getFPModifiersOperand();
118       } else if (hasIntModifiers()) {
119         return getIntModifiersOperand();
120       } else {
121         return 0;
122       }
123     }
124 
125     friend raw_ostream &operator <<(raw_ostream &OS, AMDGPUOperand::Modifiers Mods);
126   };
127 
128   enum ImmTy {
129     ImmTyNone,
130     ImmTyGDS,
131     ImmTyOffen,
132     ImmTyIdxen,
133     ImmTyAddr64,
134     ImmTyOffset,
135     ImmTyInstOffset,
136     ImmTyOffset0,
137     ImmTyOffset1,
138     ImmTyGLC,
139     ImmTySLC,
140     ImmTyTFE,
141     ImmTyD16,
142     ImmTyClampSI,
143     ImmTyOModSI,
144     ImmTyDppCtrl,
145     ImmTyDppRowMask,
146     ImmTyDppBankMask,
147     ImmTyDppBoundCtrl,
148     ImmTySdwaDstSel,
149     ImmTySdwaSrc0Sel,
150     ImmTySdwaSrc1Sel,
151     ImmTySdwaDstUnused,
152     ImmTyDMask,
153     ImmTyUNorm,
154     ImmTyDA,
155     ImmTyR128,
156     ImmTyLWE,
157     ImmTyExpTgt,
158     ImmTyExpCompr,
159     ImmTyExpVM,
160     ImmTyDFMT,
161     ImmTyNFMT,
162     ImmTyHwreg,
163     ImmTyOff,
164     ImmTySendMsg,
165     ImmTyInterpSlot,
166     ImmTyInterpAttr,
167     ImmTyAttrChan,
168     ImmTyOpSel,
169     ImmTyOpSelHi,
170     ImmTyNegLo,
171     ImmTyNegHi,
172     ImmTySwizzle,
173     ImmTyHigh
174   };
175 
176   struct TokOp {
177     const char *Data;
178     unsigned Length;
179   };
180 
181   struct ImmOp {
182     int64_t Val;
183     ImmTy Type;
184     bool IsFPImm;
185     Modifiers Mods;
186   };
187 
188   struct RegOp {
189     unsigned RegNo;
190     bool IsForcedVOP3;
191     Modifiers Mods;
192   };
193 
194   union {
195     TokOp Tok;
196     ImmOp Imm;
197     RegOp Reg;
198     const MCExpr *Expr;
199   };
200 
201   bool isToken() const override {
202     if (Kind == Token)
203       return true;
204 
205     if (Kind != Expression || !Expr)
206       return false;
207 
208     // When parsing operands, we can't always tell if something was meant to be
209     // a token, like 'gds', or an expression that references a global variable.
210     // In this case, we assume the string is an expression, and if we need to
211     // interpret is a token, then we treat the symbol name as the token.
212     return isa<MCSymbolRefExpr>(Expr);
213   }
214 
215   bool isImm() const override {
216     return Kind == Immediate;
217   }
218 
219   bool isInlinableImm(MVT type) const;
220   bool isLiteralImm(MVT type) const;
221 
222   bool isRegKind() const {
223     return Kind == Register;
224   }
225 
226   bool isReg() const override {
227     return isRegKind() && !hasModifiers();
228   }
229 
230   bool isRegOrImmWithInputMods(MVT type) const {
231     return isRegKind() || isInlinableImm(type);
232   }
233 
234   bool isRegOrImmWithInt16InputMods() const {
235     return isRegOrImmWithInputMods(MVT::i16);
236   }
237 
238   bool isRegOrImmWithInt32InputMods() const {
239     return isRegOrImmWithInputMods(MVT::i32);
240   }
241 
242   bool isRegOrImmWithInt64InputMods() const {
243     return isRegOrImmWithInputMods(MVT::i64);
244   }
245 
246   bool isRegOrImmWithFP16InputMods() const {
247     return isRegOrImmWithInputMods(MVT::f16);
248   }
249 
250   bool isRegOrImmWithFP32InputMods() const {
251     return isRegOrImmWithInputMods(MVT::f32);
252   }
253 
254   bool isRegOrImmWithFP64InputMods() const {
255     return isRegOrImmWithInputMods(MVT::f64);
256   }
257 
258   bool isVReg() const {
259     return isRegClass(AMDGPU::VGPR_32RegClassID) ||
260            isRegClass(AMDGPU::VReg_64RegClassID) ||
261            isRegClass(AMDGPU::VReg_96RegClassID) ||
262            isRegClass(AMDGPU::VReg_128RegClassID) ||
263            isRegClass(AMDGPU::VReg_256RegClassID) ||
264            isRegClass(AMDGPU::VReg_512RegClassID);
265   }
266 
267   bool isVReg32OrOff() const {
268     return isOff() || isRegClass(AMDGPU::VGPR_32RegClassID);
269   }
270 
271   bool isSDWAOperand(MVT type) const;
272   bool isSDWAFP16Operand() const;
273   bool isSDWAFP32Operand() const;
274   bool isSDWAInt16Operand() const;
275   bool isSDWAInt32Operand() const;
276 
277   bool isImmTy(ImmTy ImmT) const {
278     return isImm() && Imm.Type == ImmT;
279   }
280 
281   bool isImmModifier() const {
282     return isImm() && Imm.Type != ImmTyNone;
283   }
284 
285   bool isClampSI() const { return isImmTy(ImmTyClampSI); }
286   bool isOModSI() const { return isImmTy(ImmTyOModSI); }
287   bool isDMask() const { return isImmTy(ImmTyDMask); }
288   bool isUNorm() const { return isImmTy(ImmTyUNorm); }
289   bool isDA() const { return isImmTy(ImmTyDA); }
290   bool isR128() const { return isImmTy(ImmTyR128); }
291   bool isLWE() const { return isImmTy(ImmTyLWE); }
292   bool isOff() const { return isImmTy(ImmTyOff); }
293   bool isExpTgt() const { return isImmTy(ImmTyExpTgt); }
294   bool isExpVM() const { return isImmTy(ImmTyExpVM); }
295   bool isExpCompr() const { return isImmTy(ImmTyExpCompr); }
296   bool isOffen() const { return isImmTy(ImmTyOffen); }
297   bool isIdxen() const { return isImmTy(ImmTyIdxen); }
298   bool isAddr64() const { return isImmTy(ImmTyAddr64); }
299   bool isOffset() const { return isImmTy(ImmTyOffset) && isUInt<16>(getImm()); }
300   bool isOffset0() const { return isImmTy(ImmTyOffset0) && isUInt<16>(getImm()); }
301   bool isOffset1() const { return isImmTy(ImmTyOffset1) && isUInt<8>(getImm()); }
302 
303   bool isOffsetU12() const { return (isImmTy(ImmTyOffset) || isImmTy(ImmTyInstOffset)) && isUInt<12>(getImm()); }
304   bool isOffsetS13() const { return (isImmTy(ImmTyOffset) || isImmTy(ImmTyInstOffset)) && isInt<13>(getImm()); }
305   bool isGDS() const { return isImmTy(ImmTyGDS); }
306   bool isGLC() const { return isImmTy(ImmTyGLC); }
307   bool isSLC() const { return isImmTy(ImmTySLC); }
308   bool isTFE() const { return isImmTy(ImmTyTFE); }
309   bool isD16() const { return isImmTy(ImmTyD16); }
310   bool isDFMT() const { return isImmTy(ImmTyDFMT) && isUInt<8>(getImm()); }
311   bool isNFMT() const { return isImmTy(ImmTyNFMT) && isUInt<8>(getImm()); }
312   bool isBankMask() const { return isImmTy(ImmTyDppBankMask); }
313   bool isRowMask() const { return isImmTy(ImmTyDppRowMask); }
314   bool isBoundCtrl() const { return isImmTy(ImmTyDppBoundCtrl); }
315   bool isSDWADstSel() const { return isImmTy(ImmTySdwaDstSel); }
316   bool isSDWASrc0Sel() const { return isImmTy(ImmTySdwaSrc0Sel); }
317   bool isSDWASrc1Sel() const { return isImmTy(ImmTySdwaSrc1Sel); }
318   bool isSDWADstUnused() const { return isImmTy(ImmTySdwaDstUnused); }
319   bool isInterpSlot() const { return isImmTy(ImmTyInterpSlot); }
320   bool isInterpAttr() const { return isImmTy(ImmTyInterpAttr); }
321   bool isAttrChan() const { return isImmTy(ImmTyAttrChan); }
322   bool isOpSel() const { return isImmTy(ImmTyOpSel); }
323   bool isOpSelHi() const { return isImmTy(ImmTyOpSelHi); }
324   bool isNegLo() const { return isImmTy(ImmTyNegLo); }
325   bool isNegHi() const { return isImmTy(ImmTyNegHi); }
326   bool isHigh() const { return isImmTy(ImmTyHigh); }
327 
328   bool isMod() const {
329     return isClampSI() || isOModSI();
330   }
331 
332   bool isRegOrImm() const {
333     return isReg() || isImm();
334   }
335 
336   bool isRegClass(unsigned RCID) const;
337 
338   bool isRegOrInlineNoMods(unsigned RCID, MVT type) const {
339     return (isRegClass(RCID) || isInlinableImm(type)) && !hasModifiers();
340   }
341 
342   bool isSCSrcB16() const {
343     return isRegOrInlineNoMods(AMDGPU::SReg_32RegClassID, MVT::i16);
344   }
345 
346   bool isSCSrcV2B16() const {
347     return isSCSrcB16();
348   }
349 
350   bool isSCSrcB32() const {
351     return isRegOrInlineNoMods(AMDGPU::SReg_32RegClassID, MVT::i32);
352   }
353 
354   bool isSCSrcB64() const {
355     return isRegOrInlineNoMods(AMDGPU::SReg_64RegClassID, MVT::i64);
356   }
357 
358   bool isSCSrcF16() const {
359     return isRegOrInlineNoMods(AMDGPU::SReg_32RegClassID, MVT::f16);
360   }
361 
362   bool isSCSrcV2F16() const {
363     return isSCSrcF16();
364   }
365 
366   bool isSCSrcF32() const {
367     return isRegOrInlineNoMods(AMDGPU::SReg_32RegClassID, MVT::f32);
368   }
369 
370   bool isSCSrcF64() const {
371     return isRegOrInlineNoMods(AMDGPU::SReg_64RegClassID, MVT::f64);
372   }
373 
374   bool isSSrcB32() const {
375     return isSCSrcB32() || isLiteralImm(MVT::i32) || isExpr();
376   }
377 
378   bool isSSrcB16() const {
379     return isSCSrcB16() || isLiteralImm(MVT::i16);
380   }
381 
382   bool isSSrcV2B16() const {
383     llvm_unreachable("cannot happen");
384     return isSSrcB16();
385   }
386 
387   bool isSSrcB64() const {
388     // TODO: Find out how SALU supports extension of 32-bit literals to 64 bits.
389     // See isVSrc64().
390     return isSCSrcB64() || isLiteralImm(MVT::i64);
391   }
392 
393   bool isSSrcF32() const {
394     return isSCSrcB32() || isLiteralImm(MVT::f32) || isExpr();
395   }
396 
397   bool isSSrcF64() const {
398     return isSCSrcB64() || isLiteralImm(MVT::f64);
399   }
400 
401   bool isSSrcF16() const {
402     return isSCSrcB16() || isLiteralImm(MVT::f16);
403   }
404 
405   bool isSSrcV2F16() const {
406     llvm_unreachable("cannot happen");
407     return isSSrcF16();
408   }
409 
410   bool isVCSrcB32() const {
411     return isRegOrInlineNoMods(AMDGPU::VS_32RegClassID, MVT::i32);
412   }
413 
414   bool isVCSrcB64() const {
415     return isRegOrInlineNoMods(AMDGPU::VS_64RegClassID, MVT::i64);
416   }
417 
418   bool isVCSrcB16() const {
419     return isRegOrInlineNoMods(AMDGPU::VS_32RegClassID, MVT::i16);
420   }
421 
422   bool isVCSrcV2B16() const {
423     return isVCSrcB16();
424   }
425 
426   bool isVCSrcF32() const {
427     return isRegOrInlineNoMods(AMDGPU::VS_32RegClassID, MVT::f32);
428   }
429 
430   bool isVCSrcF64() const {
431     return isRegOrInlineNoMods(AMDGPU::VS_64RegClassID, MVT::f64);
432   }
433 
434   bool isVCSrcF16() const {
435     return isRegOrInlineNoMods(AMDGPU::VS_32RegClassID, MVT::f16);
436   }
437 
438   bool isVCSrcV2F16() const {
439     return isVCSrcF16();
440   }
441 
442   bool isVSrcB32() const {
443     return isVCSrcF32() || isLiteralImm(MVT::i32);
444   }
445 
446   bool isVSrcB64() const {
447     return isVCSrcF64() || isLiteralImm(MVT::i64);
448   }
449 
450   bool isVSrcB16() const {
451     return isVCSrcF16() || isLiteralImm(MVT::i16);
452   }
453 
454   bool isVSrcV2B16() const {
455     llvm_unreachable("cannot happen");
456     return isVSrcB16();
457   }
458 
459   bool isVSrcF32() const {
460     return isVCSrcF32() || isLiteralImm(MVT::f32);
461   }
462 
463   bool isVSrcF64() const {
464     return isVCSrcF64() || isLiteralImm(MVT::f64);
465   }
466 
467   bool isVSrcF16() const {
468     return isVCSrcF16() || isLiteralImm(MVT::f16);
469   }
470 
471   bool isVSrcV2F16() const {
472     llvm_unreachable("cannot happen");
473     return isVSrcF16();
474   }
475 
476   bool isKImmFP32() const {
477     return isLiteralImm(MVT::f32);
478   }
479 
480   bool isKImmFP16() const {
481     return isLiteralImm(MVT::f16);
482   }
483 
484   bool isMem() const override {
485     return false;
486   }
487 
488   bool isExpr() const {
489     return Kind == Expression;
490   }
491 
492   bool isSoppBrTarget() const {
493     return isExpr() || isImm();
494   }
495 
496   bool isSWaitCnt() const;
497   bool isHwreg() const;
498   bool isSendMsg() const;
499   bool isSwizzle() const;
500   bool isSMRDOffset8() const;
501   bool isSMRDOffset20() const;
502   bool isSMRDLiteralOffset() const;
503   bool isDPPCtrl() const;
504   bool isGPRIdxMode() const;
505   bool isS16Imm() const;
506   bool isU16Imm() const;
507 
508   StringRef getExpressionAsToken() const {
509     assert(isExpr());
510     const MCSymbolRefExpr *S = cast<MCSymbolRefExpr>(Expr);
511     return S->getSymbol().getName();
512   }
513 
514   StringRef getToken() const {
515     assert(isToken());
516 
517     if (Kind == Expression)
518       return getExpressionAsToken();
519 
520     return StringRef(Tok.Data, Tok.Length);
521   }
522 
523   int64_t getImm() const {
524     assert(isImm());
525     return Imm.Val;
526   }
527 
528   ImmTy getImmTy() const {
529     assert(isImm());
530     return Imm.Type;
531   }
532 
533   unsigned getReg() const override {
534     return Reg.RegNo;
535   }
536 
537   SMLoc getStartLoc() const override {
538     return StartLoc;
539   }
540 
541   SMLoc getEndLoc() const override {
542     return EndLoc;
543   }
544 
545   SMRange getLocRange() const {
546     return SMRange(StartLoc, EndLoc);
547   }
548 
549   Modifiers getModifiers() const {
550     assert(isRegKind() || isImmTy(ImmTyNone));
551     return isRegKind() ? Reg.Mods : Imm.Mods;
552   }
553 
554   void setModifiers(Modifiers Mods) {
555     assert(isRegKind() || isImmTy(ImmTyNone));
556     if (isRegKind())
557       Reg.Mods = Mods;
558     else
559       Imm.Mods = Mods;
560   }
561 
562   bool hasModifiers() const {
563     return getModifiers().hasModifiers();
564   }
565 
566   bool hasFPModifiers() const {
567     return getModifiers().hasFPModifiers();
568   }
569 
570   bool hasIntModifiers() const {
571     return getModifiers().hasIntModifiers();
572   }
573 
574   uint64_t applyInputFPModifiers(uint64_t Val, unsigned Size) const;
575 
576   void addImmOperands(MCInst &Inst, unsigned N, bool ApplyModifiers = true) const;
577 
578   void addLiteralImmOperand(MCInst &Inst, int64_t Val, bool ApplyModifiers) const;
579 
580   template <unsigned Bitwidth>
581   void addKImmFPOperands(MCInst &Inst, unsigned N) const;
582 
583   void addKImmFP16Operands(MCInst &Inst, unsigned N) const {
584     addKImmFPOperands<16>(Inst, N);
585   }
586 
587   void addKImmFP32Operands(MCInst &Inst, unsigned N) const {
588     addKImmFPOperands<32>(Inst, N);
589   }
590 
591   void addRegOperands(MCInst &Inst, unsigned N) const;
592 
593   void addRegOrImmOperands(MCInst &Inst, unsigned N) const {
594     if (isRegKind())
595       addRegOperands(Inst, N);
596     else if (isExpr())
597       Inst.addOperand(MCOperand::createExpr(Expr));
598     else
599       addImmOperands(Inst, N);
600   }
601 
602   void addRegOrImmWithInputModsOperands(MCInst &Inst, unsigned N) const {
603     Modifiers Mods = getModifiers();
604     Inst.addOperand(MCOperand::createImm(Mods.getModifiersOperand()));
605     if (isRegKind()) {
606       addRegOperands(Inst, N);
607     } else {
608       addImmOperands(Inst, N, false);
609     }
610   }
611 
612   void addRegOrImmWithFPInputModsOperands(MCInst &Inst, unsigned N) const {
613     assert(!hasIntModifiers());
614     addRegOrImmWithInputModsOperands(Inst, N);
615   }
616 
617   void addRegOrImmWithIntInputModsOperands(MCInst &Inst, unsigned N) const {
618     assert(!hasFPModifiers());
619     addRegOrImmWithInputModsOperands(Inst, N);
620   }
621 
622   void addRegWithInputModsOperands(MCInst &Inst, unsigned N) const {
623     Modifiers Mods = getModifiers();
624     Inst.addOperand(MCOperand::createImm(Mods.getModifiersOperand()));
625     assert(isRegKind());
626     addRegOperands(Inst, N);
627   }
628 
629   void addRegWithFPInputModsOperands(MCInst &Inst, unsigned N) const {
630     assert(!hasIntModifiers());
631     addRegWithInputModsOperands(Inst, N);
632   }
633 
634   void addRegWithIntInputModsOperands(MCInst &Inst, unsigned N) const {
635     assert(!hasFPModifiers());
636     addRegWithInputModsOperands(Inst, N);
637   }
638 
639   void addSoppBrTargetOperands(MCInst &Inst, unsigned N) const {
640     if (isImm())
641       addImmOperands(Inst, N);
642     else {
643       assert(isExpr());
644       Inst.addOperand(MCOperand::createExpr(Expr));
645     }
646   }
647 
648   static void printImmTy(raw_ostream& OS, ImmTy Type) {
649     switch (Type) {
650     case ImmTyNone: OS << "None"; break;
651     case ImmTyGDS: OS << "GDS"; break;
652     case ImmTyOffen: OS << "Offen"; break;
653     case ImmTyIdxen: OS << "Idxen"; break;
654     case ImmTyAddr64: OS << "Addr64"; break;
655     case ImmTyOffset: OS << "Offset"; break;
656     case ImmTyInstOffset: OS << "InstOffset"; break;
657     case ImmTyOffset0: OS << "Offset0"; break;
658     case ImmTyOffset1: OS << "Offset1"; break;
659     case ImmTyGLC: OS << "GLC"; break;
660     case ImmTySLC: OS << "SLC"; break;
661     case ImmTyTFE: OS << "TFE"; break;
662     case ImmTyD16: OS << "D16"; break;
663     case ImmTyDFMT: OS << "DFMT"; break;
664     case ImmTyNFMT: OS << "NFMT"; break;
665     case ImmTyClampSI: OS << "ClampSI"; break;
666     case ImmTyOModSI: OS << "OModSI"; break;
667     case ImmTyDppCtrl: OS << "DppCtrl"; break;
668     case ImmTyDppRowMask: OS << "DppRowMask"; break;
669     case ImmTyDppBankMask: OS << "DppBankMask"; break;
670     case ImmTyDppBoundCtrl: OS << "DppBoundCtrl"; break;
671     case ImmTySdwaDstSel: OS << "SdwaDstSel"; break;
672     case ImmTySdwaSrc0Sel: OS << "SdwaSrc0Sel"; break;
673     case ImmTySdwaSrc1Sel: OS << "SdwaSrc1Sel"; break;
674     case ImmTySdwaDstUnused: OS << "SdwaDstUnused"; break;
675     case ImmTyDMask: OS << "DMask"; break;
676     case ImmTyUNorm: OS << "UNorm"; break;
677     case ImmTyDA: OS << "DA"; break;
678     case ImmTyR128: OS << "R128"; break;
679     case ImmTyLWE: OS << "LWE"; break;
680     case ImmTyOff: OS << "Off"; break;
681     case ImmTyExpTgt: OS << "ExpTgt"; break;
682     case ImmTyExpCompr: OS << "ExpCompr"; break;
683     case ImmTyExpVM: OS << "ExpVM"; break;
684     case ImmTyHwreg: OS << "Hwreg"; break;
685     case ImmTySendMsg: OS << "SendMsg"; break;
686     case ImmTyInterpSlot: OS << "InterpSlot"; break;
687     case ImmTyInterpAttr: OS << "InterpAttr"; break;
688     case ImmTyAttrChan: OS << "AttrChan"; break;
689     case ImmTyOpSel: OS << "OpSel"; break;
690     case ImmTyOpSelHi: OS << "OpSelHi"; break;
691     case ImmTyNegLo: OS << "NegLo"; break;
692     case ImmTyNegHi: OS << "NegHi"; break;
693     case ImmTySwizzle: OS << "Swizzle"; break;
694     case ImmTyHigh: OS << "High"; break;
695     }
696   }
697 
698   void print(raw_ostream &OS) const override {
699     switch (Kind) {
700     case Register:
701       OS << "<register " << getReg() << " mods: " << Reg.Mods << '>';
702       break;
703     case Immediate:
704       OS << '<' << getImm();
705       if (getImmTy() != ImmTyNone) {
706         OS << " type: "; printImmTy(OS, getImmTy());
707       }
708       OS << " mods: " << Imm.Mods << '>';
709       break;
710     case Token:
711       OS << '\'' << getToken() << '\'';
712       break;
713     case Expression:
714       OS << "<expr " << *Expr << '>';
715       break;
716     }
717   }
718 
719   static AMDGPUOperand::Ptr CreateImm(const AMDGPUAsmParser *AsmParser,
720                                       int64_t Val, SMLoc Loc,
721                                       ImmTy Type = ImmTyNone,
722                                       bool IsFPImm = false) {
723     auto Op = llvm::make_unique<AMDGPUOperand>(Immediate, AsmParser);
724     Op->Imm.Val = Val;
725     Op->Imm.IsFPImm = IsFPImm;
726     Op->Imm.Type = Type;
727     Op->Imm.Mods = Modifiers();
728     Op->StartLoc = Loc;
729     Op->EndLoc = Loc;
730     return Op;
731   }
732 
733   static AMDGPUOperand::Ptr CreateToken(const AMDGPUAsmParser *AsmParser,
734                                         StringRef Str, SMLoc Loc,
735                                         bool HasExplicitEncodingSize = true) {
736     auto Res = llvm::make_unique<AMDGPUOperand>(Token, AsmParser);
737     Res->Tok.Data = Str.data();
738     Res->Tok.Length = Str.size();
739     Res->StartLoc = Loc;
740     Res->EndLoc = Loc;
741     return Res;
742   }
743 
744   static AMDGPUOperand::Ptr CreateReg(const AMDGPUAsmParser *AsmParser,
745                                       unsigned RegNo, SMLoc S,
746                                       SMLoc E,
747                                       bool ForceVOP3) {
748     auto Op = llvm::make_unique<AMDGPUOperand>(Register, AsmParser);
749     Op->Reg.RegNo = RegNo;
750     Op->Reg.Mods = Modifiers();
751     Op->Reg.IsForcedVOP3 = ForceVOP3;
752     Op->StartLoc = S;
753     Op->EndLoc = E;
754     return Op;
755   }
756 
757   static AMDGPUOperand::Ptr CreateExpr(const AMDGPUAsmParser *AsmParser,
758                                        const class MCExpr *Expr, SMLoc S) {
759     auto Op = llvm::make_unique<AMDGPUOperand>(Expression, AsmParser);
760     Op->Expr = Expr;
761     Op->StartLoc = S;
762     Op->EndLoc = S;
763     return Op;
764   }
765 };
766 
767 raw_ostream &operator <<(raw_ostream &OS, AMDGPUOperand::Modifiers Mods) {
768   OS << "abs:" << Mods.Abs << " neg: " << Mods.Neg << " sext:" << Mods.Sext;
769   return OS;
770 }
771 
772 //===----------------------------------------------------------------------===//
773 // AsmParser
774 //===----------------------------------------------------------------------===//
775 
776 // Holds info related to the current kernel, e.g. count of SGPRs used.
777 // Kernel scope begins at .amdgpu_hsa_kernel directive, ends at next
778 // .amdgpu_hsa_kernel or at EOF.
779 class KernelScopeInfo {
780   int SgprIndexUnusedMin = -1;
781   int VgprIndexUnusedMin = -1;
782   MCContext *Ctx = nullptr;
783 
784   void usesSgprAt(int i) {
785     if (i >= SgprIndexUnusedMin) {
786       SgprIndexUnusedMin = ++i;
787       if (Ctx) {
788         MCSymbol * const Sym = Ctx->getOrCreateSymbol(Twine(".kernel.sgpr_count"));
789         Sym->setVariableValue(MCConstantExpr::create(SgprIndexUnusedMin, *Ctx));
790       }
791     }
792   }
793 
794   void usesVgprAt(int i) {
795     if (i >= VgprIndexUnusedMin) {
796       VgprIndexUnusedMin = ++i;
797       if (Ctx) {
798         MCSymbol * const Sym = Ctx->getOrCreateSymbol(Twine(".kernel.vgpr_count"));
799         Sym->setVariableValue(MCConstantExpr::create(VgprIndexUnusedMin, *Ctx));
800       }
801     }
802   }
803 
804 public:
805   KernelScopeInfo() = default;
806 
807   void initialize(MCContext &Context) {
808     Ctx = &Context;
809     usesSgprAt(SgprIndexUnusedMin = -1);
810     usesVgprAt(VgprIndexUnusedMin = -1);
811   }
812 
813   void usesRegister(RegisterKind RegKind, unsigned DwordRegIndex, unsigned RegWidth) {
814     switch (RegKind) {
815       case IS_SGPR: usesSgprAt(DwordRegIndex + RegWidth - 1); break;
816       case IS_VGPR: usesVgprAt(DwordRegIndex + RegWidth - 1); break;
817       default: break;
818     }
819   }
820 };
821 
822 class AMDGPUAsmParser : public MCTargetAsmParser {
823   MCAsmParser &Parser;
824 
825   // Number of extra operands parsed after the first optional operand.
826   // This may be necessary to skip hardcoded mandatory operands.
827   static const unsigned MAX_OPR_LOOKAHEAD = 8;
828 
829   unsigned ForcedEncodingSize = 0;
830   bool ForcedDPP = false;
831   bool ForcedSDWA = false;
832   KernelScopeInfo KernelScope;
833 
834   /// @name Auto-generated Match Functions
835   /// {
836 
837 #define GET_ASSEMBLER_HEADER
838 #include "AMDGPUGenAsmMatcher.inc"
839 
840   /// }
841 
842 private:
843   bool ParseAsAbsoluteExpression(uint32_t &Ret);
844   bool ParseDirectiveMajorMinor(uint32_t &Major, uint32_t &Minor);
845   bool ParseDirectiveHSACodeObjectVersion();
846   bool ParseDirectiveHSACodeObjectISA();
847   bool ParseAMDKernelCodeTValue(StringRef ID, amd_kernel_code_t &Header);
848   bool ParseDirectiveAMDKernelCodeT();
849   bool subtargetHasRegister(const MCRegisterInfo &MRI, unsigned RegNo) const;
850   bool ParseDirectiveAMDGPUHsaKernel();
851 
852   bool ParseDirectiveISAVersion();
853   bool ParseDirectiveHSAMetadata();
854   bool ParseDirectivePALMetadata();
855 
856   bool AddNextRegisterToList(unsigned& Reg, unsigned& RegWidth,
857                              RegisterKind RegKind, unsigned Reg1,
858                              unsigned RegNum);
859   bool ParseAMDGPURegister(RegisterKind& RegKind, unsigned& Reg,
860                            unsigned& RegNum, unsigned& RegWidth,
861                            unsigned *DwordRegIndex);
862   void cvtMubufImpl(MCInst &Inst, const OperandVector &Operands,
863                     bool IsAtomic, bool IsAtomicReturn);
864   void cvtDSImpl(MCInst &Inst, const OperandVector &Operands,
865                  bool IsGdsHardcoded);
866 
867 public:
868   enum AMDGPUMatchResultTy {
869     Match_PreferE32 = FIRST_TARGET_MATCH_RESULT_TY
870   };
871 
872   using OptionalImmIndexMap = std::map<AMDGPUOperand::ImmTy, unsigned>;
873 
874   AMDGPUAsmParser(const MCSubtargetInfo &STI, MCAsmParser &_Parser,
875                const MCInstrInfo &MII,
876                const MCTargetOptions &Options)
877       : MCTargetAsmParser(Options, STI, MII), Parser(_Parser) {
878     MCAsmParserExtension::Initialize(Parser);
879 
880     if (getFeatureBits().none()) {
881       // Set default features.
882       copySTI().ToggleFeature("SOUTHERN_ISLANDS");
883     }
884 
885     setAvailableFeatures(ComputeAvailableFeatures(getFeatureBits()));
886 
887     {
888       // TODO: make those pre-defined variables read-only.
889       // Currently there is none suitable machinery in the core llvm-mc for this.
890       // MCSymbol::isRedefinable is intended for another purpose, and
891       // AsmParser::parseDirectiveSet() cannot be specialized for specific target.
892       AMDGPU::IsaInfo::IsaVersion ISA =
893           AMDGPU::IsaInfo::getIsaVersion(getFeatureBits());
894       MCContext &Ctx = getContext();
895       MCSymbol *Sym =
896           Ctx.getOrCreateSymbol(Twine(".option.machine_version_major"));
897       Sym->setVariableValue(MCConstantExpr::create(ISA.Major, Ctx));
898       Sym = Ctx.getOrCreateSymbol(Twine(".option.machine_version_minor"));
899       Sym->setVariableValue(MCConstantExpr::create(ISA.Minor, Ctx));
900       Sym = Ctx.getOrCreateSymbol(Twine(".option.machine_version_stepping"));
901       Sym->setVariableValue(MCConstantExpr::create(ISA.Stepping, Ctx));
902     }
903     KernelScope.initialize(getContext());
904   }
905 
906   bool hasXNACK() const {
907     return AMDGPU::hasXNACK(getSTI());
908   }
909 
910   bool isSI() const {
911     return AMDGPU::isSI(getSTI());
912   }
913 
914   bool isCI() const {
915     return AMDGPU::isCI(getSTI());
916   }
917 
918   bool isVI() const {
919     return AMDGPU::isVI(getSTI());
920   }
921 
922   bool isGFX9() const {
923     return AMDGPU::isGFX9(getSTI());
924   }
925 
926   bool hasInv2PiInlineImm() const {
927     return getFeatureBits()[AMDGPU::FeatureInv2PiInlineImm];
928   }
929 
930   bool hasFlatOffsets() const {
931     return getFeatureBits()[AMDGPU::FeatureFlatInstOffsets];
932   }
933 
934   bool hasSGPR102_SGPR103() const {
935     return !isVI();
936   }
937 
938   bool hasIntClamp() const {
939     return getFeatureBits()[AMDGPU::FeatureIntClamp];
940   }
941 
942   AMDGPUTargetStreamer &getTargetStreamer() {
943     MCTargetStreamer &TS = *getParser().getStreamer().getTargetStreamer();
944     return static_cast<AMDGPUTargetStreamer &>(TS);
945   }
946 
947   const MCRegisterInfo *getMRI() const {
948     // We need this const_cast because for some reason getContext() is not const
949     // in MCAsmParser.
950     return const_cast<AMDGPUAsmParser*>(this)->getContext().getRegisterInfo();
951   }
952 
953   const MCInstrInfo *getMII() const {
954     return &MII;
955   }
956 
957   const FeatureBitset &getFeatureBits() const {
958     return getSTI().getFeatureBits();
959   }
960 
961   void setForcedEncodingSize(unsigned Size) { ForcedEncodingSize = Size; }
962   void setForcedDPP(bool ForceDPP_) { ForcedDPP = ForceDPP_; }
963   void setForcedSDWA(bool ForceSDWA_) { ForcedSDWA = ForceSDWA_; }
964 
965   unsigned getForcedEncodingSize() const { return ForcedEncodingSize; }
966   bool isForcedVOP3() const { return ForcedEncodingSize == 64; }
967   bool isForcedDPP() const { return ForcedDPP; }
968   bool isForcedSDWA() const { return ForcedSDWA; }
969   ArrayRef<unsigned> getMatchedVariants() const;
970 
971   std::unique_ptr<AMDGPUOperand> parseRegister();
972   bool ParseRegister(unsigned &RegNo, SMLoc &StartLoc, SMLoc &EndLoc) override;
973   unsigned checkTargetMatchPredicate(MCInst &Inst) override;
974   unsigned validateTargetOperandClass(MCParsedAsmOperand &Op,
975                                       unsigned Kind) override;
976   bool MatchAndEmitInstruction(SMLoc IDLoc, unsigned &Opcode,
977                                OperandVector &Operands, MCStreamer &Out,
978                                uint64_t &ErrorInfo,
979                                bool MatchingInlineAsm) override;
980   bool ParseDirective(AsmToken DirectiveID) override;
981   OperandMatchResultTy parseOperand(OperandVector &Operands, StringRef Mnemonic);
982   StringRef parseMnemonicSuffix(StringRef Name);
983   bool ParseInstruction(ParseInstructionInfo &Info, StringRef Name,
984                         SMLoc NameLoc, OperandVector &Operands) override;
985   //bool ProcessInstruction(MCInst &Inst);
986 
987   OperandMatchResultTy parseIntWithPrefix(const char *Prefix, int64_t &Int);
988 
989   OperandMatchResultTy
990   parseIntWithPrefix(const char *Prefix, OperandVector &Operands,
991                      AMDGPUOperand::ImmTy ImmTy = AMDGPUOperand::ImmTyNone,
992                      bool (*ConvertResult)(int64_t &) = nullptr);
993 
994   OperandMatchResultTy parseOperandArrayWithPrefix(
995     const char *Prefix,
996     OperandVector &Operands,
997     AMDGPUOperand::ImmTy ImmTy = AMDGPUOperand::ImmTyNone,
998     bool (*ConvertResult)(int64_t&) = nullptr);
999 
1000   OperandMatchResultTy
1001   parseNamedBit(const char *Name, OperandVector &Operands,
1002                 AMDGPUOperand::ImmTy ImmTy = AMDGPUOperand::ImmTyNone);
1003   OperandMatchResultTy parseStringWithPrefix(StringRef Prefix,
1004                                              StringRef &Value);
1005 
1006   bool parseAbsoluteExpr(int64_t &Val, bool AbsMod = false);
1007   OperandMatchResultTy parseImm(OperandVector &Operands, bool AbsMod = false);
1008   OperandMatchResultTy parseReg(OperandVector &Operands);
1009   OperandMatchResultTy parseRegOrImm(OperandVector &Operands, bool AbsMod = false);
1010   OperandMatchResultTy parseRegOrImmWithFPInputMods(OperandVector &Operands, bool AllowImm = true);
1011   OperandMatchResultTy parseRegOrImmWithIntInputMods(OperandVector &Operands, bool AllowImm = true);
1012   OperandMatchResultTy parseRegWithFPInputMods(OperandVector &Operands);
1013   OperandMatchResultTy parseRegWithIntInputMods(OperandVector &Operands);
1014   OperandMatchResultTy parseVReg32OrOff(OperandVector &Operands);
1015 
1016   void cvtDSOffset01(MCInst &Inst, const OperandVector &Operands);
1017   void cvtDS(MCInst &Inst, const OperandVector &Operands) { cvtDSImpl(Inst, Operands, false); }
1018   void cvtDSGds(MCInst &Inst, const OperandVector &Operands) { cvtDSImpl(Inst, Operands, true); }
1019   void cvtExp(MCInst &Inst, const OperandVector &Operands);
1020 
1021   bool parseCnt(int64_t &IntVal);
1022   OperandMatchResultTy parseSWaitCntOps(OperandVector &Operands);
1023   OperandMatchResultTy parseHwreg(OperandVector &Operands);
1024 
1025 private:
1026   struct OperandInfoTy {
1027     int64_t Id;
1028     bool IsSymbolic = false;
1029 
1030     OperandInfoTy(int64_t Id_) : Id(Id_) {}
1031   };
1032 
1033   bool parseSendMsgConstruct(OperandInfoTy &Msg, OperandInfoTy &Operation, int64_t &StreamId);
1034   bool parseHwregConstruct(OperandInfoTy &HwReg, int64_t &Offset, int64_t &Width);
1035 
1036   void errorExpTgt();
1037   OperandMatchResultTy parseExpTgtImpl(StringRef Str, uint8_t &Val);
1038 
1039   bool validateInstruction(const MCInst &Inst, const SMLoc &IDLoc);
1040   bool validateConstantBusLimitations(const MCInst &Inst);
1041   bool validateEarlyClobberLimitations(const MCInst &Inst);
1042   bool validateIntClampSupported(const MCInst &Inst);
1043   bool validateMIMGAtomicDMask(const MCInst &Inst);
1044   bool validateMIMGDataSize(const MCInst &Inst);
1045   bool usesConstantBus(const MCInst &Inst, unsigned OpIdx);
1046   bool isInlineConstant(const MCInst &Inst, unsigned OpIdx) const;
1047   unsigned findImplicitSGPRReadInVOP(const MCInst &Inst) const;
1048 
1049   bool trySkipId(const StringRef Id);
1050   bool trySkipToken(const AsmToken::TokenKind Kind);
1051   bool skipToken(const AsmToken::TokenKind Kind, const StringRef ErrMsg);
1052   bool parseString(StringRef &Val, const StringRef ErrMsg = "expected a string");
1053   bool parseExpr(int64_t &Imm);
1054 
1055 public:
1056   OperandMatchResultTy parseOptionalOperand(OperandVector &Operands);
1057   OperandMatchResultTy parseOptionalOpr(OperandVector &Operands);
1058 
1059   OperandMatchResultTy parseExpTgt(OperandVector &Operands);
1060   OperandMatchResultTy parseSendMsgOp(OperandVector &Operands);
1061   OperandMatchResultTy parseInterpSlot(OperandVector &Operands);
1062   OperandMatchResultTy parseInterpAttr(OperandVector &Operands);
1063   OperandMatchResultTy parseSOppBrTarget(OperandVector &Operands);
1064 
1065   bool parseSwizzleOperands(const unsigned OpNum, int64_t* Op,
1066                             const unsigned MinVal,
1067                             const unsigned MaxVal,
1068                             const StringRef ErrMsg);
1069   OperandMatchResultTy parseSwizzleOp(OperandVector &Operands);
1070   bool parseSwizzleOffset(int64_t &Imm);
1071   bool parseSwizzleMacro(int64_t &Imm);
1072   bool parseSwizzleQuadPerm(int64_t &Imm);
1073   bool parseSwizzleBitmaskPerm(int64_t &Imm);
1074   bool parseSwizzleBroadcast(int64_t &Imm);
1075   bool parseSwizzleSwap(int64_t &Imm);
1076   bool parseSwizzleReverse(int64_t &Imm);
1077 
1078   void cvtMubuf(MCInst &Inst, const OperandVector &Operands) { cvtMubufImpl(Inst, Operands, false, false); }
1079   void cvtMubufAtomic(MCInst &Inst, const OperandVector &Operands) { cvtMubufImpl(Inst, Operands, true, false); }
1080   void cvtMubufAtomicReturn(MCInst &Inst, const OperandVector &Operands) { cvtMubufImpl(Inst, Operands, true, true); }
1081   void cvtMtbuf(MCInst &Inst, const OperandVector &Operands);
1082 
1083   AMDGPUOperand::Ptr defaultGLC() const;
1084   AMDGPUOperand::Ptr defaultSLC() const;
1085   AMDGPUOperand::Ptr defaultTFE() const;
1086 
1087   AMDGPUOperand::Ptr defaultD16() const;
1088   AMDGPUOperand::Ptr defaultDMask() const;
1089   AMDGPUOperand::Ptr defaultUNorm() const;
1090   AMDGPUOperand::Ptr defaultDA() const;
1091   AMDGPUOperand::Ptr defaultR128() const;
1092   AMDGPUOperand::Ptr defaultLWE() const;
1093   AMDGPUOperand::Ptr defaultSMRDOffset8() const;
1094   AMDGPUOperand::Ptr defaultSMRDOffset20() const;
1095   AMDGPUOperand::Ptr defaultSMRDLiteralOffset() const;
1096   AMDGPUOperand::Ptr defaultOffsetU12() const;
1097   AMDGPUOperand::Ptr defaultOffsetS13() const;
1098 
1099   OperandMatchResultTy parseOModOperand(OperandVector &Operands);
1100 
1101   void cvtVOP3(MCInst &Inst, const OperandVector &Operands,
1102                OptionalImmIndexMap &OptionalIdx);
1103   void cvtVOP3OpSel(MCInst &Inst, const OperandVector &Operands);
1104   void cvtVOP3(MCInst &Inst, const OperandVector &Operands);
1105   void cvtVOP3P(MCInst &Inst, const OperandVector &Operands);
1106 
1107   void cvtVOP3Interp(MCInst &Inst, const OperandVector &Operands);
1108 
1109   void cvtMIMG(MCInst &Inst, const OperandVector &Operands,
1110                bool IsAtomic = false);
1111   void cvtMIMGAtomic(MCInst &Inst, const OperandVector &Operands);
1112 
1113   OperandMatchResultTy parseDPPCtrl(OperandVector &Operands);
1114   AMDGPUOperand::Ptr defaultRowMask() const;
1115   AMDGPUOperand::Ptr defaultBankMask() const;
1116   AMDGPUOperand::Ptr defaultBoundCtrl() const;
1117   void cvtDPP(MCInst &Inst, const OperandVector &Operands);
1118 
1119   OperandMatchResultTy parseSDWASel(OperandVector &Operands, StringRef Prefix,
1120                                     AMDGPUOperand::ImmTy Type);
1121   OperandMatchResultTy parseSDWADstUnused(OperandVector &Operands);
1122   void cvtSdwaVOP1(MCInst &Inst, const OperandVector &Operands);
1123   void cvtSdwaVOP2(MCInst &Inst, const OperandVector &Operands);
1124   void cvtSdwaVOP2b(MCInst &Inst, const OperandVector &Operands);
1125   void cvtSdwaVOPC(MCInst &Inst, const OperandVector &Operands);
1126   void cvtSDWA(MCInst &Inst, const OperandVector &Operands,
1127                 uint64_t BasicInstType, bool skipVcc = false);
1128 };
1129 
1130 struct OptionalOperand {
1131   const char *Name;
1132   AMDGPUOperand::ImmTy Type;
1133   bool IsBit;
1134   bool (*ConvertResult)(int64_t&);
1135 };
1136 
1137 } // end anonymous namespace
1138 
1139 // May be called with integer type with equivalent bitwidth.
1140 static const fltSemantics *getFltSemantics(unsigned Size) {
1141   switch (Size) {
1142   case 4:
1143     return &APFloat::IEEEsingle();
1144   case 8:
1145     return &APFloat::IEEEdouble();
1146   case 2:
1147     return &APFloat::IEEEhalf();
1148   default:
1149     llvm_unreachable("unsupported fp type");
1150   }
1151 }
1152 
1153 static const fltSemantics *getFltSemantics(MVT VT) {
1154   return getFltSemantics(VT.getSizeInBits() / 8);
1155 }
1156 
1157 static const fltSemantics *getOpFltSemantics(uint8_t OperandType) {
1158   switch (OperandType) {
1159   case AMDGPU::OPERAND_REG_IMM_INT32:
1160   case AMDGPU::OPERAND_REG_IMM_FP32:
1161   case AMDGPU::OPERAND_REG_INLINE_C_INT32:
1162   case AMDGPU::OPERAND_REG_INLINE_C_FP32:
1163     return &APFloat::IEEEsingle();
1164   case AMDGPU::OPERAND_REG_IMM_INT64:
1165   case AMDGPU::OPERAND_REG_IMM_FP64:
1166   case AMDGPU::OPERAND_REG_INLINE_C_INT64:
1167   case AMDGPU::OPERAND_REG_INLINE_C_FP64:
1168     return &APFloat::IEEEdouble();
1169   case AMDGPU::OPERAND_REG_IMM_INT16:
1170   case AMDGPU::OPERAND_REG_IMM_FP16:
1171   case AMDGPU::OPERAND_REG_INLINE_C_INT16:
1172   case AMDGPU::OPERAND_REG_INLINE_C_FP16:
1173   case AMDGPU::OPERAND_REG_INLINE_C_V2INT16:
1174   case AMDGPU::OPERAND_REG_INLINE_C_V2FP16:
1175     return &APFloat::IEEEhalf();
1176   default:
1177     llvm_unreachable("unsupported fp type");
1178   }
1179 }
1180 
1181 //===----------------------------------------------------------------------===//
1182 // Operand
1183 //===----------------------------------------------------------------------===//
1184 
1185 static bool canLosslesslyConvertToFPType(APFloat &FPLiteral, MVT VT) {
1186   bool Lost;
1187 
1188   // Convert literal to single precision
1189   APFloat::opStatus Status = FPLiteral.convert(*getFltSemantics(VT),
1190                                                APFloat::rmNearestTiesToEven,
1191                                                &Lost);
1192   // We allow precision lost but not overflow or underflow
1193   if (Status != APFloat::opOK &&
1194       Lost &&
1195       ((Status & APFloat::opOverflow)  != 0 ||
1196        (Status & APFloat::opUnderflow) != 0)) {
1197     return false;
1198   }
1199 
1200   return true;
1201 }
1202 
1203 bool AMDGPUOperand::isInlinableImm(MVT type) const {
1204   if (!isImmTy(ImmTyNone)) {
1205     // Only plain immediates are inlinable (e.g. "clamp" attribute is not)
1206     return false;
1207   }
1208   // TODO: We should avoid using host float here. It would be better to
1209   // check the float bit values which is what a few other places do.
1210   // We've had bot failures before due to weird NaN support on mips hosts.
1211 
1212   APInt Literal(64, Imm.Val);
1213 
1214   if (Imm.IsFPImm) { // We got fp literal token
1215     if (type == MVT::f64 || type == MVT::i64) { // Expected 64-bit operand
1216       return AMDGPU::isInlinableLiteral64(Imm.Val,
1217                                           AsmParser->hasInv2PiInlineImm());
1218     }
1219 
1220     APFloat FPLiteral(APFloat::IEEEdouble(), APInt(64, Imm.Val));
1221     if (!canLosslesslyConvertToFPType(FPLiteral, type))
1222       return false;
1223 
1224     if (type.getScalarSizeInBits() == 16) {
1225       return AMDGPU::isInlinableLiteral16(
1226         static_cast<int16_t>(FPLiteral.bitcastToAPInt().getZExtValue()),
1227         AsmParser->hasInv2PiInlineImm());
1228     }
1229 
1230     // Check if single precision literal is inlinable
1231     return AMDGPU::isInlinableLiteral32(
1232       static_cast<int32_t>(FPLiteral.bitcastToAPInt().getZExtValue()),
1233       AsmParser->hasInv2PiInlineImm());
1234   }
1235 
1236   // We got int literal token.
1237   if (type == MVT::f64 || type == MVT::i64) { // Expected 64-bit operand
1238     return AMDGPU::isInlinableLiteral64(Imm.Val,
1239                                         AsmParser->hasInv2PiInlineImm());
1240   }
1241 
1242   if (type.getScalarSizeInBits() == 16) {
1243     return AMDGPU::isInlinableLiteral16(
1244       static_cast<int16_t>(Literal.getLoBits(16).getSExtValue()),
1245       AsmParser->hasInv2PiInlineImm());
1246   }
1247 
1248   return AMDGPU::isInlinableLiteral32(
1249     static_cast<int32_t>(Literal.getLoBits(32).getZExtValue()),
1250     AsmParser->hasInv2PiInlineImm());
1251 }
1252 
1253 bool AMDGPUOperand::isLiteralImm(MVT type) const {
1254   // Check that this immediate can be added as literal
1255   if (!isImmTy(ImmTyNone)) {
1256     return false;
1257   }
1258 
1259   if (!Imm.IsFPImm) {
1260     // We got int literal token.
1261 
1262     if (type == MVT::f64 && hasFPModifiers()) {
1263       // Cannot apply fp modifiers to int literals preserving the same semantics
1264       // for VOP1/2/C and VOP3 because of integer truncation. To avoid ambiguity,
1265       // disable these cases.
1266       return false;
1267     }
1268 
1269     unsigned Size = type.getSizeInBits();
1270     if (Size == 64)
1271       Size = 32;
1272 
1273     // FIXME: 64-bit operands can zero extend, sign extend, or pad zeroes for FP
1274     // types.
1275     return isUIntN(Size, Imm.Val) || isIntN(Size, Imm.Val);
1276   }
1277 
1278   // We got fp literal token
1279   if (type == MVT::f64) { // Expected 64-bit fp operand
1280     // We would set low 64-bits of literal to zeroes but we accept this literals
1281     return true;
1282   }
1283 
1284   if (type == MVT::i64) { // Expected 64-bit int operand
1285     // We don't allow fp literals in 64-bit integer instructions. It is
1286     // unclear how we should encode them.
1287     return false;
1288   }
1289 
1290   APFloat FPLiteral(APFloat::IEEEdouble(), APInt(64, Imm.Val));
1291   return canLosslesslyConvertToFPType(FPLiteral, type);
1292 }
1293 
1294 bool AMDGPUOperand::isRegClass(unsigned RCID) const {
1295   return isRegKind() && AsmParser->getMRI()->getRegClass(RCID).contains(getReg());
1296 }
1297 
1298 bool AMDGPUOperand::isSDWAOperand(MVT type) const {
1299   if (AsmParser->isVI())
1300     return isVReg();
1301   else if (AsmParser->isGFX9())
1302     return isRegKind() || isInlinableImm(type);
1303   else
1304     return false;
1305 }
1306 
1307 bool AMDGPUOperand::isSDWAFP16Operand() const {
1308   return isSDWAOperand(MVT::f16);
1309 }
1310 
1311 bool AMDGPUOperand::isSDWAFP32Operand() const {
1312   return isSDWAOperand(MVT::f32);
1313 }
1314 
1315 bool AMDGPUOperand::isSDWAInt16Operand() const {
1316   return isSDWAOperand(MVT::i16);
1317 }
1318 
1319 bool AMDGPUOperand::isSDWAInt32Operand() const {
1320   return isSDWAOperand(MVT::i32);
1321 }
1322 
1323 uint64_t AMDGPUOperand::applyInputFPModifiers(uint64_t Val, unsigned Size) const
1324 {
1325   assert(isImmTy(ImmTyNone) && Imm.Mods.hasFPModifiers());
1326   assert(Size == 2 || Size == 4 || Size == 8);
1327 
1328   const uint64_t FpSignMask = (1ULL << (Size * 8 - 1));
1329 
1330   if (Imm.Mods.Abs) {
1331     Val &= ~FpSignMask;
1332   }
1333   if (Imm.Mods.Neg) {
1334     Val ^= FpSignMask;
1335   }
1336 
1337   return Val;
1338 }
1339 
1340 void AMDGPUOperand::addImmOperands(MCInst &Inst, unsigned N, bool ApplyModifiers) const {
1341   if (AMDGPU::isSISrcOperand(AsmParser->getMII()->get(Inst.getOpcode()),
1342                              Inst.getNumOperands())) {
1343     addLiteralImmOperand(Inst, Imm.Val,
1344                          ApplyModifiers &
1345                          isImmTy(ImmTyNone) && Imm.Mods.hasFPModifiers());
1346   } else {
1347     assert(!isImmTy(ImmTyNone) || !hasModifiers());
1348     Inst.addOperand(MCOperand::createImm(Imm.Val));
1349   }
1350 }
1351 
1352 void AMDGPUOperand::addLiteralImmOperand(MCInst &Inst, int64_t Val, bool ApplyModifiers) const {
1353   const auto& InstDesc = AsmParser->getMII()->get(Inst.getOpcode());
1354   auto OpNum = Inst.getNumOperands();
1355   // Check that this operand accepts literals
1356   assert(AMDGPU::isSISrcOperand(InstDesc, OpNum));
1357 
1358   if (ApplyModifiers) {
1359     assert(AMDGPU::isSISrcFPOperand(InstDesc, OpNum));
1360     const unsigned Size = Imm.IsFPImm ? sizeof(double) : getOperandSize(InstDesc, OpNum);
1361     Val = applyInputFPModifiers(Val, Size);
1362   }
1363 
1364   APInt Literal(64, Val);
1365   uint8_t OpTy = InstDesc.OpInfo[OpNum].OperandType;
1366 
1367   if (Imm.IsFPImm) { // We got fp literal token
1368     switch (OpTy) {
1369     case AMDGPU::OPERAND_REG_IMM_INT64:
1370     case AMDGPU::OPERAND_REG_IMM_FP64:
1371     case AMDGPU::OPERAND_REG_INLINE_C_INT64:
1372     case AMDGPU::OPERAND_REG_INLINE_C_FP64:
1373       if (AMDGPU::isInlinableLiteral64(Literal.getZExtValue(),
1374                                        AsmParser->hasInv2PiInlineImm())) {
1375         Inst.addOperand(MCOperand::createImm(Literal.getZExtValue()));
1376         return;
1377       }
1378 
1379       // Non-inlineable
1380       if (AMDGPU::isSISrcFPOperand(InstDesc, OpNum)) { // Expected 64-bit fp operand
1381         // For fp operands we check if low 32 bits are zeros
1382         if (Literal.getLoBits(32) != 0) {
1383           const_cast<AMDGPUAsmParser *>(AsmParser)->Warning(Inst.getLoc(),
1384           "Can't encode literal as exact 64-bit floating-point operand. "
1385           "Low 32-bits will be set to zero");
1386         }
1387 
1388         Inst.addOperand(MCOperand::createImm(Literal.lshr(32).getZExtValue()));
1389         return;
1390       }
1391 
1392       // We don't allow fp literals in 64-bit integer instructions. It is
1393       // unclear how we should encode them. This case should be checked earlier
1394       // in predicate methods (isLiteralImm())
1395       llvm_unreachable("fp literal in 64-bit integer instruction.");
1396 
1397     case AMDGPU::OPERAND_REG_IMM_INT32:
1398     case AMDGPU::OPERAND_REG_IMM_FP32:
1399     case AMDGPU::OPERAND_REG_INLINE_C_INT32:
1400     case AMDGPU::OPERAND_REG_INLINE_C_FP32:
1401     case AMDGPU::OPERAND_REG_IMM_INT16:
1402     case AMDGPU::OPERAND_REG_IMM_FP16:
1403     case AMDGPU::OPERAND_REG_INLINE_C_INT16:
1404     case AMDGPU::OPERAND_REG_INLINE_C_FP16:
1405     case AMDGPU::OPERAND_REG_INLINE_C_V2INT16:
1406     case AMDGPU::OPERAND_REG_INLINE_C_V2FP16: {
1407       bool lost;
1408       APFloat FPLiteral(APFloat::IEEEdouble(), Literal);
1409       // Convert literal to single precision
1410       FPLiteral.convert(*getOpFltSemantics(OpTy),
1411                         APFloat::rmNearestTiesToEven, &lost);
1412       // We allow precision lost but not overflow or underflow. This should be
1413       // checked earlier in isLiteralImm()
1414 
1415       uint64_t ImmVal = FPLiteral.bitcastToAPInt().getZExtValue();
1416       if (OpTy == AMDGPU::OPERAND_REG_INLINE_C_V2INT16 ||
1417           OpTy == AMDGPU::OPERAND_REG_INLINE_C_V2FP16) {
1418         ImmVal |= (ImmVal << 16);
1419       }
1420 
1421       Inst.addOperand(MCOperand::createImm(ImmVal));
1422       return;
1423     }
1424     default:
1425       llvm_unreachable("invalid operand size");
1426     }
1427 
1428     return;
1429   }
1430 
1431    // We got int literal token.
1432   // Only sign extend inline immediates.
1433   // FIXME: No errors on truncation
1434   switch (OpTy) {
1435   case AMDGPU::OPERAND_REG_IMM_INT32:
1436   case AMDGPU::OPERAND_REG_IMM_FP32:
1437   case AMDGPU::OPERAND_REG_INLINE_C_INT32:
1438   case AMDGPU::OPERAND_REG_INLINE_C_FP32:
1439     if (isInt<32>(Val) &&
1440         AMDGPU::isInlinableLiteral32(static_cast<int32_t>(Val),
1441                                      AsmParser->hasInv2PiInlineImm())) {
1442       Inst.addOperand(MCOperand::createImm(Val));
1443       return;
1444     }
1445 
1446     Inst.addOperand(MCOperand::createImm(Val & 0xffffffff));
1447     return;
1448 
1449   case AMDGPU::OPERAND_REG_IMM_INT64:
1450   case AMDGPU::OPERAND_REG_IMM_FP64:
1451   case AMDGPU::OPERAND_REG_INLINE_C_INT64:
1452   case AMDGPU::OPERAND_REG_INLINE_C_FP64:
1453     if (AMDGPU::isInlinableLiteral64(Val, AsmParser->hasInv2PiInlineImm())) {
1454       Inst.addOperand(MCOperand::createImm(Val));
1455       return;
1456     }
1457 
1458     Inst.addOperand(MCOperand::createImm(Lo_32(Val)));
1459     return;
1460 
1461   case AMDGPU::OPERAND_REG_IMM_INT16:
1462   case AMDGPU::OPERAND_REG_IMM_FP16:
1463   case AMDGPU::OPERAND_REG_INLINE_C_INT16:
1464   case AMDGPU::OPERAND_REG_INLINE_C_FP16:
1465     if (isInt<16>(Val) &&
1466         AMDGPU::isInlinableLiteral16(static_cast<int16_t>(Val),
1467                                      AsmParser->hasInv2PiInlineImm())) {
1468       Inst.addOperand(MCOperand::createImm(Val));
1469       return;
1470     }
1471 
1472     Inst.addOperand(MCOperand::createImm(Val & 0xffff));
1473     return;
1474 
1475   case AMDGPU::OPERAND_REG_INLINE_C_V2INT16:
1476   case AMDGPU::OPERAND_REG_INLINE_C_V2FP16: {
1477     auto LiteralVal = static_cast<uint16_t>(Literal.getLoBits(16).getZExtValue());
1478     assert(AMDGPU::isInlinableLiteral16(LiteralVal,
1479                                         AsmParser->hasInv2PiInlineImm()));
1480 
1481     uint32_t ImmVal = static_cast<uint32_t>(LiteralVal) << 16 |
1482                       static_cast<uint32_t>(LiteralVal);
1483     Inst.addOperand(MCOperand::createImm(ImmVal));
1484     return;
1485   }
1486   default:
1487     llvm_unreachable("invalid operand size");
1488   }
1489 }
1490 
1491 template <unsigned Bitwidth>
1492 void AMDGPUOperand::addKImmFPOperands(MCInst &Inst, unsigned N) const {
1493   APInt Literal(64, Imm.Val);
1494 
1495   if (!Imm.IsFPImm) {
1496     // We got int literal token.
1497     Inst.addOperand(MCOperand::createImm(Literal.getLoBits(Bitwidth).getZExtValue()));
1498     return;
1499   }
1500 
1501   bool Lost;
1502   APFloat FPLiteral(APFloat::IEEEdouble(), Literal);
1503   FPLiteral.convert(*getFltSemantics(Bitwidth / 8),
1504                     APFloat::rmNearestTiesToEven, &Lost);
1505   Inst.addOperand(MCOperand::createImm(FPLiteral.bitcastToAPInt().getZExtValue()));
1506 }
1507 
1508 void AMDGPUOperand::addRegOperands(MCInst &Inst, unsigned N) const {
1509   Inst.addOperand(MCOperand::createReg(AMDGPU::getMCReg(getReg(), AsmParser->getSTI())));
1510 }
1511 
1512 //===----------------------------------------------------------------------===//
1513 // AsmParser
1514 //===----------------------------------------------------------------------===//
1515 
1516 static int getRegClass(RegisterKind Is, unsigned RegWidth) {
1517   if (Is == IS_VGPR) {
1518     switch (RegWidth) {
1519       default: return -1;
1520       case 1: return AMDGPU::VGPR_32RegClassID;
1521       case 2: return AMDGPU::VReg_64RegClassID;
1522       case 3: return AMDGPU::VReg_96RegClassID;
1523       case 4: return AMDGPU::VReg_128RegClassID;
1524       case 8: return AMDGPU::VReg_256RegClassID;
1525       case 16: return AMDGPU::VReg_512RegClassID;
1526     }
1527   } else if (Is == IS_TTMP) {
1528     switch (RegWidth) {
1529       default: return -1;
1530       case 1: return AMDGPU::TTMP_32RegClassID;
1531       case 2: return AMDGPU::TTMP_64RegClassID;
1532       case 4: return AMDGPU::TTMP_128RegClassID;
1533       case 8: return AMDGPU::TTMP_256RegClassID;
1534       case 16: return AMDGPU::TTMP_512RegClassID;
1535     }
1536   } else if (Is == IS_SGPR) {
1537     switch (RegWidth) {
1538       default: return -1;
1539       case 1: return AMDGPU::SGPR_32RegClassID;
1540       case 2: return AMDGPU::SGPR_64RegClassID;
1541       case 4: return AMDGPU::SGPR_128RegClassID;
1542       case 8: return AMDGPU::SGPR_256RegClassID;
1543       case 16: return AMDGPU::SGPR_512RegClassID;
1544     }
1545   }
1546   return -1;
1547 }
1548 
1549 static unsigned getSpecialRegForName(StringRef RegName) {
1550   return StringSwitch<unsigned>(RegName)
1551     .Case("exec", AMDGPU::EXEC)
1552     .Case("vcc", AMDGPU::VCC)
1553     .Case("flat_scratch", AMDGPU::FLAT_SCR)
1554     .Case("xnack_mask", AMDGPU::XNACK_MASK)
1555     .Case("m0", AMDGPU::M0)
1556     .Case("scc", AMDGPU::SCC)
1557     .Case("tba", AMDGPU::TBA)
1558     .Case("tma", AMDGPU::TMA)
1559     .Case("flat_scratch_lo", AMDGPU::FLAT_SCR_LO)
1560     .Case("flat_scratch_hi", AMDGPU::FLAT_SCR_HI)
1561     .Case("xnack_mask_lo", AMDGPU::XNACK_MASK_LO)
1562     .Case("xnack_mask_hi", AMDGPU::XNACK_MASK_HI)
1563     .Case("vcc_lo", AMDGPU::VCC_LO)
1564     .Case("vcc_hi", AMDGPU::VCC_HI)
1565     .Case("exec_lo", AMDGPU::EXEC_LO)
1566     .Case("exec_hi", AMDGPU::EXEC_HI)
1567     .Case("tma_lo", AMDGPU::TMA_LO)
1568     .Case("tma_hi", AMDGPU::TMA_HI)
1569     .Case("tba_lo", AMDGPU::TBA_LO)
1570     .Case("tba_hi", AMDGPU::TBA_HI)
1571     .Default(0);
1572 }
1573 
1574 bool AMDGPUAsmParser::ParseRegister(unsigned &RegNo, SMLoc &StartLoc,
1575                                     SMLoc &EndLoc) {
1576   auto R = parseRegister();
1577   if (!R) return true;
1578   assert(R->isReg());
1579   RegNo = R->getReg();
1580   StartLoc = R->getStartLoc();
1581   EndLoc = R->getEndLoc();
1582   return false;
1583 }
1584 
1585 bool AMDGPUAsmParser::AddNextRegisterToList(unsigned &Reg, unsigned &RegWidth,
1586                                             RegisterKind RegKind, unsigned Reg1,
1587                                             unsigned RegNum) {
1588   switch (RegKind) {
1589   case IS_SPECIAL:
1590     if (Reg == AMDGPU::EXEC_LO && Reg1 == AMDGPU::EXEC_HI) {
1591       Reg = AMDGPU::EXEC;
1592       RegWidth = 2;
1593       return true;
1594     }
1595     if (Reg == AMDGPU::FLAT_SCR_LO && Reg1 == AMDGPU::FLAT_SCR_HI) {
1596       Reg = AMDGPU::FLAT_SCR;
1597       RegWidth = 2;
1598       return true;
1599     }
1600     if (Reg == AMDGPU::XNACK_MASK_LO && Reg1 == AMDGPU::XNACK_MASK_HI) {
1601       Reg = AMDGPU::XNACK_MASK;
1602       RegWidth = 2;
1603       return true;
1604     }
1605     if (Reg == AMDGPU::VCC_LO && Reg1 == AMDGPU::VCC_HI) {
1606       Reg = AMDGPU::VCC;
1607       RegWidth = 2;
1608       return true;
1609     }
1610     if (Reg == AMDGPU::TBA_LO && Reg1 == AMDGPU::TBA_HI) {
1611       Reg = AMDGPU::TBA;
1612       RegWidth = 2;
1613       return true;
1614     }
1615     if (Reg == AMDGPU::TMA_LO && Reg1 == AMDGPU::TMA_HI) {
1616       Reg = AMDGPU::TMA;
1617       RegWidth = 2;
1618       return true;
1619     }
1620     return false;
1621   case IS_VGPR:
1622   case IS_SGPR:
1623   case IS_TTMP:
1624     if (Reg1 != Reg + RegWidth) {
1625       return false;
1626     }
1627     RegWidth++;
1628     return true;
1629   default:
1630     llvm_unreachable("unexpected register kind");
1631   }
1632 }
1633 
1634 bool AMDGPUAsmParser::ParseAMDGPURegister(RegisterKind &RegKind, unsigned &Reg,
1635                                           unsigned &RegNum, unsigned &RegWidth,
1636                                           unsigned *DwordRegIndex) {
1637   if (DwordRegIndex) { *DwordRegIndex = 0; }
1638   const MCRegisterInfo *TRI = getContext().getRegisterInfo();
1639   if (getLexer().is(AsmToken::Identifier)) {
1640     StringRef RegName = Parser.getTok().getString();
1641     if ((Reg = getSpecialRegForName(RegName))) {
1642       Parser.Lex();
1643       RegKind = IS_SPECIAL;
1644     } else {
1645       unsigned RegNumIndex = 0;
1646       if (RegName[0] == 'v') {
1647         RegNumIndex = 1;
1648         RegKind = IS_VGPR;
1649       } else if (RegName[0] == 's') {
1650         RegNumIndex = 1;
1651         RegKind = IS_SGPR;
1652       } else if (RegName.startswith("ttmp")) {
1653         RegNumIndex = strlen("ttmp");
1654         RegKind = IS_TTMP;
1655       } else {
1656         return false;
1657       }
1658       if (RegName.size() > RegNumIndex) {
1659         // Single 32-bit register: vXX.
1660         if (RegName.substr(RegNumIndex).getAsInteger(10, RegNum))
1661           return false;
1662         Parser.Lex();
1663         RegWidth = 1;
1664       } else {
1665         // Range of registers: v[XX:YY]. ":YY" is optional.
1666         Parser.Lex();
1667         int64_t RegLo, RegHi;
1668         if (getLexer().isNot(AsmToken::LBrac))
1669           return false;
1670         Parser.Lex();
1671 
1672         if (getParser().parseAbsoluteExpression(RegLo))
1673           return false;
1674 
1675         const bool isRBrace = getLexer().is(AsmToken::RBrac);
1676         if (!isRBrace && getLexer().isNot(AsmToken::Colon))
1677           return false;
1678         Parser.Lex();
1679 
1680         if (isRBrace) {
1681           RegHi = RegLo;
1682         } else {
1683           if (getParser().parseAbsoluteExpression(RegHi))
1684             return false;
1685 
1686           if (getLexer().isNot(AsmToken::RBrac))
1687             return false;
1688           Parser.Lex();
1689         }
1690         RegNum = (unsigned) RegLo;
1691         RegWidth = (RegHi - RegLo) + 1;
1692       }
1693     }
1694   } else if (getLexer().is(AsmToken::LBrac)) {
1695     // List of consecutive registers: [s0,s1,s2,s3]
1696     Parser.Lex();
1697     if (!ParseAMDGPURegister(RegKind, Reg, RegNum, RegWidth, nullptr))
1698       return false;
1699     if (RegWidth != 1)
1700       return false;
1701     RegisterKind RegKind1;
1702     unsigned Reg1, RegNum1, RegWidth1;
1703     do {
1704       if (getLexer().is(AsmToken::Comma)) {
1705         Parser.Lex();
1706       } else if (getLexer().is(AsmToken::RBrac)) {
1707         Parser.Lex();
1708         break;
1709       } else if (ParseAMDGPURegister(RegKind1, Reg1, RegNum1, RegWidth1, nullptr)) {
1710         if (RegWidth1 != 1) {
1711           return false;
1712         }
1713         if (RegKind1 != RegKind) {
1714           return false;
1715         }
1716         if (!AddNextRegisterToList(Reg, RegWidth, RegKind1, Reg1, RegNum1)) {
1717           return false;
1718         }
1719       } else {
1720         return false;
1721       }
1722     } while (true);
1723   } else {
1724     return false;
1725   }
1726   switch (RegKind) {
1727   case IS_SPECIAL:
1728     RegNum = 0;
1729     RegWidth = 1;
1730     break;
1731   case IS_VGPR:
1732   case IS_SGPR:
1733   case IS_TTMP:
1734   {
1735     unsigned Size = 1;
1736     if (RegKind == IS_SGPR || RegKind == IS_TTMP) {
1737       // SGPR and TTMP registers must be aligned. Max required alignment is 4 dwords.
1738       Size = std::min(RegWidth, 4u);
1739     }
1740     if (RegNum % Size != 0)
1741       return false;
1742     if (DwordRegIndex) { *DwordRegIndex = RegNum; }
1743     RegNum = RegNum / Size;
1744     int RCID = getRegClass(RegKind, RegWidth);
1745     if (RCID == -1)
1746       return false;
1747     const MCRegisterClass RC = TRI->getRegClass(RCID);
1748     if (RegNum >= RC.getNumRegs())
1749       return false;
1750     Reg = RC.getRegister(RegNum);
1751     break;
1752   }
1753 
1754   default:
1755     llvm_unreachable("unexpected register kind");
1756   }
1757 
1758   if (!subtargetHasRegister(*TRI, Reg))
1759     return false;
1760   return true;
1761 }
1762 
1763 std::unique_ptr<AMDGPUOperand> AMDGPUAsmParser::parseRegister() {
1764   const auto &Tok = Parser.getTok();
1765   SMLoc StartLoc = Tok.getLoc();
1766   SMLoc EndLoc = Tok.getEndLoc();
1767   RegisterKind RegKind;
1768   unsigned Reg, RegNum, RegWidth, DwordRegIndex;
1769 
1770   if (!ParseAMDGPURegister(RegKind, Reg, RegNum, RegWidth, &DwordRegIndex)) {
1771     return nullptr;
1772   }
1773   KernelScope.usesRegister(RegKind, DwordRegIndex, RegWidth);
1774   return AMDGPUOperand::CreateReg(this, Reg, StartLoc, EndLoc, false);
1775 }
1776 
1777 bool
1778 AMDGPUAsmParser::parseAbsoluteExpr(int64_t &Val, bool AbsMod) {
1779   if (AbsMod && getLexer().peekTok().is(AsmToken::Pipe) &&
1780       (getLexer().getKind() == AsmToken::Integer ||
1781        getLexer().getKind() == AsmToken::Real)) {
1782     // This is a workaround for handling operands like these:
1783     //     |1.0|
1784     //     |-1|
1785     // This syntax is not compatible with syntax of standard
1786     // MC expressions (due to the trailing '|').
1787 
1788     SMLoc EndLoc;
1789     const MCExpr *Expr;
1790 
1791     if (getParser().parsePrimaryExpr(Expr, EndLoc)) {
1792       return true;
1793     }
1794 
1795     return !Expr->evaluateAsAbsolute(Val);
1796   }
1797 
1798   return getParser().parseAbsoluteExpression(Val);
1799 }
1800 
1801 OperandMatchResultTy
1802 AMDGPUAsmParser::parseImm(OperandVector &Operands, bool AbsMod) {
1803   // TODO: add syntactic sugar for 1/(2*PI)
1804   bool Minus = false;
1805   if (getLexer().getKind() == AsmToken::Minus) {
1806     const AsmToken NextToken = getLexer().peekTok();
1807     if (!NextToken.is(AsmToken::Integer) &&
1808         !NextToken.is(AsmToken::Real)) {
1809         return MatchOperand_NoMatch;
1810     }
1811     Minus = true;
1812     Parser.Lex();
1813   }
1814 
1815   SMLoc S = Parser.getTok().getLoc();
1816   switch(getLexer().getKind()) {
1817   case AsmToken::Integer: {
1818     int64_t IntVal;
1819     if (parseAbsoluteExpr(IntVal, AbsMod))
1820       return MatchOperand_ParseFail;
1821     if (Minus)
1822       IntVal *= -1;
1823     Operands.push_back(AMDGPUOperand::CreateImm(this, IntVal, S));
1824     return MatchOperand_Success;
1825   }
1826   case AsmToken::Real: {
1827     int64_t IntVal;
1828     if (parseAbsoluteExpr(IntVal, AbsMod))
1829       return MatchOperand_ParseFail;
1830 
1831     APFloat F(BitsToDouble(IntVal));
1832     if (Minus)
1833       F.changeSign();
1834     Operands.push_back(
1835         AMDGPUOperand::CreateImm(this, F.bitcastToAPInt().getZExtValue(), S,
1836                                  AMDGPUOperand::ImmTyNone, true));
1837     return MatchOperand_Success;
1838   }
1839   default:
1840     return MatchOperand_NoMatch;
1841   }
1842 }
1843 
1844 OperandMatchResultTy
1845 AMDGPUAsmParser::parseReg(OperandVector &Operands) {
1846   if (auto R = parseRegister()) {
1847     assert(R->isReg());
1848     R->Reg.IsForcedVOP3 = isForcedVOP3();
1849     Operands.push_back(std::move(R));
1850     return MatchOperand_Success;
1851   }
1852   return MatchOperand_NoMatch;
1853 }
1854 
1855 OperandMatchResultTy
1856 AMDGPUAsmParser::parseRegOrImm(OperandVector &Operands, bool AbsMod) {
1857   auto res = parseImm(Operands, AbsMod);
1858   if (res != MatchOperand_NoMatch) {
1859     return res;
1860   }
1861 
1862   return parseReg(Operands);
1863 }
1864 
1865 OperandMatchResultTy
1866 AMDGPUAsmParser::parseRegOrImmWithFPInputMods(OperandVector &Operands,
1867                                               bool AllowImm) {
1868   bool Negate = false, Negate2 = false, Abs = false, Abs2 = false;
1869 
1870   if (getLexer().getKind()== AsmToken::Minus) {
1871     const AsmToken NextToken = getLexer().peekTok();
1872 
1873     // Disable ambiguous constructs like '--1' etc. Should use neg(-1) instead.
1874     if (NextToken.is(AsmToken::Minus)) {
1875       Error(Parser.getTok().getLoc(), "invalid syntax, expected 'neg' modifier");
1876       return MatchOperand_ParseFail;
1877     }
1878 
1879     // '-' followed by an integer literal N should be interpreted as integer
1880     // negation rather than a floating-point NEG modifier applied to N.
1881     // Beside being contr-intuitive, such use of floating-point NEG modifier
1882     // results in different meaning of integer literals used with VOP1/2/C
1883     // and VOP3, for example:
1884     //    v_exp_f32_e32 v5, -1 // VOP1: src0 = 0xFFFFFFFF
1885     //    v_exp_f32_e64 v5, -1 // VOP3: src0 = 0x80000001
1886     // Negative fp literals should be handled likewise for unifomtity
1887     if (!NextToken.is(AsmToken::Integer) && !NextToken.is(AsmToken::Real)) {
1888       Parser.Lex();
1889       Negate = true;
1890     }
1891   }
1892 
1893   if (getLexer().getKind() == AsmToken::Identifier &&
1894       Parser.getTok().getString() == "neg") {
1895     if (Negate) {
1896       Error(Parser.getTok().getLoc(), "expected register or immediate");
1897       return MatchOperand_ParseFail;
1898     }
1899     Parser.Lex();
1900     Negate2 = true;
1901     if (getLexer().isNot(AsmToken::LParen)) {
1902       Error(Parser.getTok().getLoc(), "expected left paren after neg");
1903       return MatchOperand_ParseFail;
1904     }
1905     Parser.Lex();
1906   }
1907 
1908   if (getLexer().getKind() == AsmToken::Identifier &&
1909       Parser.getTok().getString() == "abs") {
1910     Parser.Lex();
1911     Abs2 = true;
1912     if (getLexer().isNot(AsmToken::LParen)) {
1913       Error(Parser.getTok().getLoc(), "expected left paren after abs");
1914       return MatchOperand_ParseFail;
1915     }
1916     Parser.Lex();
1917   }
1918 
1919   if (getLexer().getKind() == AsmToken::Pipe) {
1920     if (Abs2) {
1921       Error(Parser.getTok().getLoc(), "expected register or immediate");
1922       return MatchOperand_ParseFail;
1923     }
1924     Parser.Lex();
1925     Abs = true;
1926   }
1927 
1928   OperandMatchResultTy Res;
1929   if (AllowImm) {
1930     Res = parseRegOrImm(Operands, Abs);
1931   } else {
1932     Res = parseReg(Operands);
1933   }
1934   if (Res != MatchOperand_Success) {
1935     return Res;
1936   }
1937 
1938   AMDGPUOperand::Modifiers Mods;
1939   if (Abs) {
1940     if (getLexer().getKind() != AsmToken::Pipe) {
1941       Error(Parser.getTok().getLoc(), "expected vertical bar");
1942       return MatchOperand_ParseFail;
1943     }
1944     Parser.Lex();
1945     Mods.Abs = true;
1946   }
1947   if (Abs2) {
1948     if (getLexer().isNot(AsmToken::RParen)) {
1949       Error(Parser.getTok().getLoc(), "expected closing parentheses");
1950       return MatchOperand_ParseFail;
1951     }
1952     Parser.Lex();
1953     Mods.Abs = true;
1954   }
1955 
1956   if (Negate) {
1957     Mods.Neg = true;
1958   } else if (Negate2) {
1959     if (getLexer().isNot(AsmToken::RParen)) {
1960       Error(Parser.getTok().getLoc(), "expected closing parentheses");
1961       return MatchOperand_ParseFail;
1962     }
1963     Parser.Lex();
1964     Mods.Neg = true;
1965   }
1966 
1967   if (Mods.hasFPModifiers()) {
1968     AMDGPUOperand &Op = static_cast<AMDGPUOperand &>(*Operands.back());
1969     Op.setModifiers(Mods);
1970   }
1971   return MatchOperand_Success;
1972 }
1973 
1974 OperandMatchResultTy
1975 AMDGPUAsmParser::parseRegOrImmWithIntInputMods(OperandVector &Operands,
1976                                                bool AllowImm) {
1977   bool Sext = false;
1978 
1979   if (getLexer().getKind() == AsmToken::Identifier &&
1980       Parser.getTok().getString() == "sext") {
1981     Parser.Lex();
1982     Sext = true;
1983     if (getLexer().isNot(AsmToken::LParen)) {
1984       Error(Parser.getTok().getLoc(), "expected left paren after sext");
1985       return MatchOperand_ParseFail;
1986     }
1987     Parser.Lex();
1988   }
1989 
1990   OperandMatchResultTy Res;
1991   if (AllowImm) {
1992     Res = parseRegOrImm(Operands);
1993   } else {
1994     Res = parseReg(Operands);
1995   }
1996   if (Res != MatchOperand_Success) {
1997     return Res;
1998   }
1999 
2000   AMDGPUOperand::Modifiers Mods;
2001   if (Sext) {
2002     if (getLexer().isNot(AsmToken::RParen)) {
2003       Error(Parser.getTok().getLoc(), "expected closing parentheses");
2004       return MatchOperand_ParseFail;
2005     }
2006     Parser.Lex();
2007     Mods.Sext = true;
2008   }
2009 
2010   if (Mods.hasIntModifiers()) {
2011     AMDGPUOperand &Op = static_cast<AMDGPUOperand &>(*Operands.back());
2012     Op.setModifiers(Mods);
2013   }
2014 
2015   return MatchOperand_Success;
2016 }
2017 
2018 OperandMatchResultTy
2019 AMDGPUAsmParser::parseRegWithFPInputMods(OperandVector &Operands) {
2020   return parseRegOrImmWithFPInputMods(Operands, false);
2021 }
2022 
2023 OperandMatchResultTy
2024 AMDGPUAsmParser::parseRegWithIntInputMods(OperandVector &Operands) {
2025   return parseRegOrImmWithIntInputMods(Operands, false);
2026 }
2027 
2028 OperandMatchResultTy AMDGPUAsmParser::parseVReg32OrOff(OperandVector &Operands) {
2029   std::unique_ptr<AMDGPUOperand> Reg = parseRegister();
2030   if (Reg) {
2031     Operands.push_back(std::move(Reg));
2032     return MatchOperand_Success;
2033   }
2034 
2035   const AsmToken &Tok = Parser.getTok();
2036   if (Tok.getString() == "off") {
2037     Operands.push_back(AMDGPUOperand::CreateImm(this, 0, Tok.getLoc(),
2038                                                 AMDGPUOperand::ImmTyOff, false));
2039     Parser.Lex();
2040     return MatchOperand_Success;
2041   }
2042 
2043   return MatchOperand_NoMatch;
2044 }
2045 
2046 unsigned AMDGPUAsmParser::checkTargetMatchPredicate(MCInst &Inst) {
2047   uint64_t TSFlags = MII.get(Inst.getOpcode()).TSFlags;
2048 
2049   if ((getForcedEncodingSize() == 32 && (TSFlags & SIInstrFlags::VOP3)) ||
2050       (getForcedEncodingSize() == 64 && !(TSFlags & SIInstrFlags::VOP3)) ||
2051       (isForcedDPP() && !(TSFlags & SIInstrFlags::DPP)) ||
2052       (isForcedSDWA() && !(TSFlags & SIInstrFlags::SDWA)) )
2053     return Match_InvalidOperand;
2054 
2055   if ((TSFlags & SIInstrFlags::VOP3) &&
2056       (TSFlags & SIInstrFlags::VOPAsmPrefer32Bit) &&
2057       getForcedEncodingSize() != 64)
2058     return Match_PreferE32;
2059 
2060   if (Inst.getOpcode() == AMDGPU::V_MAC_F32_sdwa_vi ||
2061       Inst.getOpcode() == AMDGPU::V_MAC_F16_sdwa_vi) {
2062     // v_mac_f32/16 allow only dst_sel == DWORD;
2063     auto OpNum =
2064         AMDGPU::getNamedOperandIdx(Inst.getOpcode(), AMDGPU::OpName::dst_sel);
2065     const auto &Op = Inst.getOperand(OpNum);
2066     if (!Op.isImm() || Op.getImm() != AMDGPU::SDWA::SdwaSel::DWORD) {
2067       return Match_InvalidOperand;
2068     }
2069   }
2070 
2071   if ((TSFlags & SIInstrFlags::FLAT) && !hasFlatOffsets()) {
2072     // FIXME: Produces error without correct column reported.
2073     auto OpNum =
2074         AMDGPU::getNamedOperandIdx(Inst.getOpcode(), AMDGPU::OpName::offset);
2075     const auto &Op = Inst.getOperand(OpNum);
2076     if (Op.getImm() != 0)
2077       return Match_InvalidOperand;
2078   }
2079 
2080   return Match_Success;
2081 }
2082 
2083 // What asm variants we should check
2084 ArrayRef<unsigned> AMDGPUAsmParser::getMatchedVariants() const {
2085   if (getForcedEncodingSize() == 32) {
2086     static const unsigned Variants[] = {AMDGPUAsmVariants::DEFAULT};
2087     return makeArrayRef(Variants);
2088   }
2089 
2090   if (isForcedVOP3()) {
2091     static const unsigned Variants[] = {AMDGPUAsmVariants::VOP3};
2092     return makeArrayRef(Variants);
2093   }
2094 
2095   if (isForcedSDWA()) {
2096     static const unsigned Variants[] = {AMDGPUAsmVariants::SDWA,
2097                                         AMDGPUAsmVariants::SDWA9};
2098     return makeArrayRef(Variants);
2099   }
2100 
2101   if (isForcedDPP()) {
2102     static const unsigned Variants[] = {AMDGPUAsmVariants::DPP};
2103     return makeArrayRef(Variants);
2104   }
2105 
2106   static const unsigned Variants[] = {
2107     AMDGPUAsmVariants::DEFAULT, AMDGPUAsmVariants::VOP3,
2108     AMDGPUAsmVariants::SDWA, AMDGPUAsmVariants::SDWA9, AMDGPUAsmVariants::DPP
2109   };
2110 
2111   return makeArrayRef(Variants);
2112 }
2113 
2114 unsigned AMDGPUAsmParser::findImplicitSGPRReadInVOP(const MCInst &Inst) const {
2115   const MCInstrDesc &Desc = MII.get(Inst.getOpcode());
2116   const unsigned Num = Desc.getNumImplicitUses();
2117   for (unsigned i = 0; i < Num; ++i) {
2118     unsigned Reg = Desc.ImplicitUses[i];
2119     switch (Reg) {
2120     case AMDGPU::FLAT_SCR:
2121     case AMDGPU::VCC:
2122     case AMDGPU::M0:
2123       return Reg;
2124     default:
2125       break;
2126     }
2127   }
2128   return AMDGPU::NoRegister;
2129 }
2130 
2131 // NB: This code is correct only when used to check constant
2132 // bus limitations because GFX7 support no f16 inline constants.
2133 // Note that there are no cases when a GFX7 opcode violates
2134 // constant bus limitations due to the use of an f16 constant.
2135 bool AMDGPUAsmParser::isInlineConstant(const MCInst &Inst,
2136                                        unsigned OpIdx) const {
2137   const MCInstrDesc &Desc = MII.get(Inst.getOpcode());
2138 
2139   if (!AMDGPU::isSISrcOperand(Desc, OpIdx)) {
2140     return false;
2141   }
2142 
2143   const MCOperand &MO = Inst.getOperand(OpIdx);
2144 
2145   int64_t Val = MO.getImm();
2146   auto OpSize = AMDGPU::getOperandSize(Desc, OpIdx);
2147 
2148   switch (OpSize) { // expected operand size
2149   case 8:
2150     return AMDGPU::isInlinableLiteral64(Val, hasInv2PiInlineImm());
2151   case 4:
2152     return AMDGPU::isInlinableLiteral32(Val, hasInv2PiInlineImm());
2153   case 2: {
2154     const unsigned OperandType = Desc.OpInfo[OpIdx].OperandType;
2155     if (OperandType == AMDGPU::OPERAND_REG_INLINE_C_V2INT16 ||
2156         OperandType == AMDGPU::OPERAND_REG_INLINE_C_V2FP16) {
2157       return AMDGPU::isInlinableLiteralV216(Val, hasInv2PiInlineImm());
2158     } else {
2159       return AMDGPU::isInlinableLiteral16(Val, hasInv2PiInlineImm());
2160     }
2161   }
2162   default:
2163     llvm_unreachable("invalid operand size");
2164   }
2165 }
2166 
2167 bool AMDGPUAsmParser::usesConstantBus(const MCInst &Inst, unsigned OpIdx) {
2168   const MCOperand &MO = Inst.getOperand(OpIdx);
2169   if (MO.isImm()) {
2170     return !isInlineConstant(Inst, OpIdx);
2171   }
2172   return !MO.isReg() ||
2173          isSGPR(mc2PseudoReg(MO.getReg()), getContext().getRegisterInfo());
2174 }
2175 
2176 bool AMDGPUAsmParser::validateConstantBusLimitations(const MCInst &Inst) {
2177   const unsigned Opcode = Inst.getOpcode();
2178   const MCInstrDesc &Desc = MII.get(Opcode);
2179   unsigned ConstantBusUseCount = 0;
2180 
2181   if (Desc.TSFlags &
2182       (SIInstrFlags::VOPC |
2183        SIInstrFlags::VOP1 | SIInstrFlags::VOP2 |
2184        SIInstrFlags::VOP3 | SIInstrFlags::VOP3P |
2185        SIInstrFlags::SDWA)) {
2186     // Check special imm operands (used by madmk, etc)
2187     if (AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::imm) != -1) {
2188       ++ConstantBusUseCount;
2189     }
2190 
2191     unsigned SGPRUsed = findImplicitSGPRReadInVOP(Inst);
2192     if (SGPRUsed != AMDGPU::NoRegister) {
2193       ++ConstantBusUseCount;
2194     }
2195 
2196     const int Src0Idx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::src0);
2197     const int Src1Idx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::src1);
2198     const int Src2Idx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::src2);
2199 
2200     const int OpIndices[] = { Src0Idx, Src1Idx, Src2Idx };
2201 
2202     for (int OpIdx : OpIndices) {
2203       if (OpIdx == -1) break;
2204 
2205       const MCOperand &MO = Inst.getOperand(OpIdx);
2206       if (usesConstantBus(Inst, OpIdx)) {
2207         if (MO.isReg()) {
2208           const unsigned Reg = mc2PseudoReg(MO.getReg());
2209           // Pairs of registers with a partial intersections like these
2210           //   s0, s[0:1]
2211           //   flat_scratch_lo, flat_scratch
2212           //   flat_scratch_lo, flat_scratch_hi
2213           // are theoretically valid but they are disabled anyway.
2214           // Note that this code mimics SIInstrInfo::verifyInstruction
2215           if (Reg != SGPRUsed) {
2216             ++ConstantBusUseCount;
2217           }
2218           SGPRUsed = Reg;
2219         } else { // Expression or a literal
2220           ++ConstantBusUseCount;
2221         }
2222       }
2223     }
2224   }
2225 
2226   return ConstantBusUseCount <= 1;
2227 }
2228 
2229 bool AMDGPUAsmParser::validateEarlyClobberLimitations(const MCInst &Inst) {
2230   const unsigned Opcode = Inst.getOpcode();
2231   const MCInstrDesc &Desc = MII.get(Opcode);
2232 
2233   const int DstIdx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::vdst);
2234   if (DstIdx == -1 ||
2235       Desc.getOperandConstraint(DstIdx, MCOI::EARLY_CLOBBER) == -1) {
2236     return true;
2237   }
2238 
2239   const MCRegisterInfo *TRI = getContext().getRegisterInfo();
2240 
2241   const int Src0Idx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::src0);
2242   const int Src1Idx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::src1);
2243   const int Src2Idx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::src2);
2244 
2245   assert(DstIdx != -1);
2246   const MCOperand &Dst = Inst.getOperand(DstIdx);
2247   assert(Dst.isReg());
2248   const unsigned DstReg = mc2PseudoReg(Dst.getReg());
2249 
2250   const int SrcIndices[] = { Src0Idx, Src1Idx, Src2Idx };
2251 
2252   for (int SrcIdx : SrcIndices) {
2253     if (SrcIdx == -1) break;
2254     const MCOperand &Src = Inst.getOperand(SrcIdx);
2255     if (Src.isReg()) {
2256       const unsigned SrcReg = mc2PseudoReg(Src.getReg());
2257       if (isRegIntersect(DstReg, SrcReg, TRI)) {
2258         return false;
2259       }
2260     }
2261   }
2262 
2263   return true;
2264 }
2265 
2266 bool AMDGPUAsmParser::validateIntClampSupported(const MCInst &Inst) {
2267 
2268   const unsigned Opc = Inst.getOpcode();
2269   const MCInstrDesc &Desc = MII.get(Opc);
2270 
2271   if ((Desc.TSFlags & SIInstrFlags::IntClamp) != 0 && !hasIntClamp()) {
2272     int ClampIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::clamp);
2273     assert(ClampIdx != -1);
2274     return Inst.getOperand(ClampIdx).getImm() == 0;
2275   }
2276 
2277   return true;
2278 }
2279 
2280 bool AMDGPUAsmParser::validateMIMGDataSize(const MCInst &Inst) {
2281 
2282   const unsigned Opc = Inst.getOpcode();
2283   const MCInstrDesc &Desc = MII.get(Opc);
2284 
2285   if ((Desc.TSFlags & SIInstrFlags::MIMG) == 0)
2286     return true;
2287 
2288   // Gather4 instructions seem to have special rules not described in spec.
2289   if (Desc.TSFlags & SIInstrFlags::Gather4)
2290     return true;
2291 
2292   int VDataIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::vdata);
2293   int DMaskIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::dmask);
2294   int TFEIdx   = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::tfe);
2295 
2296   assert(VDataIdx != -1);
2297   assert(DMaskIdx != -1);
2298   assert(TFEIdx != -1);
2299 
2300   unsigned VDataSize = AMDGPU::getRegOperandSize(getMRI(), Desc, VDataIdx);
2301   unsigned TFESize = Inst.getOperand(TFEIdx).getImm()? 1 : 0;
2302   unsigned DMask = Inst.getOperand(DMaskIdx).getImm() & 0xf;
2303   if (DMask == 0)
2304     DMask = 1;
2305 
2306   return (VDataSize / 4) == countPopulation(DMask) + TFESize;
2307 }
2308 
2309 bool AMDGPUAsmParser::validateMIMGAtomicDMask(const MCInst &Inst) {
2310 
2311   const unsigned Opc = Inst.getOpcode();
2312   const MCInstrDesc &Desc = MII.get(Opc);
2313 
2314   if ((Desc.TSFlags & SIInstrFlags::MIMG) == 0)
2315     return true;
2316   if (!Desc.mayLoad() || !Desc.mayStore())
2317     return true; // Not atomic
2318 
2319   int DMaskIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::dmask);
2320   unsigned DMask = Inst.getOperand(DMaskIdx).getImm() & 0xf;
2321 
2322   // This is an incomplete check because image_atomic_cmpswap
2323   // may only use 0x3 and 0xf while other atomic operations
2324   // may use 0x1 and 0x3. However these limitations are
2325   // verified when we check that dmask matches dst size.
2326   return DMask == 0x1 || DMask == 0x3 || DMask == 0xf;
2327 }
2328 
2329 bool AMDGPUAsmParser::validateInstruction(const MCInst &Inst,
2330                                           const SMLoc &IDLoc) {
2331   if (!validateConstantBusLimitations(Inst)) {
2332     Error(IDLoc,
2333       "invalid operand (violates constant bus restrictions)");
2334     return false;
2335   }
2336   if (!validateEarlyClobberLimitations(Inst)) {
2337     Error(IDLoc,
2338       "destination must be different than all sources");
2339     return false;
2340   }
2341   if (!validateIntClampSupported(Inst)) {
2342     Error(IDLoc,
2343       "integer clamping is not supported on this GPU");
2344     return false;
2345   }
2346   if (!validateMIMGDataSize(Inst)) {
2347     Error(IDLoc,
2348       "image data size does not match dmask and tfe");
2349     return false;
2350   }
2351   if (!validateMIMGAtomicDMask(Inst)) {
2352     Error(IDLoc,
2353       "invalid atomic image dmask");
2354     return false;
2355   }
2356 
2357   return true;
2358 }
2359 
2360 static std::string AMDGPUMnemonicSpellCheck(StringRef S, uint64_t FBS,
2361                                             unsigned VariantID = 0);
2362 
2363 bool AMDGPUAsmParser::MatchAndEmitInstruction(SMLoc IDLoc, unsigned &Opcode,
2364                                               OperandVector &Operands,
2365                                               MCStreamer &Out,
2366                                               uint64_t &ErrorInfo,
2367                                               bool MatchingInlineAsm) {
2368   MCInst Inst;
2369   unsigned Result = Match_Success;
2370   for (auto Variant : getMatchedVariants()) {
2371     uint64_t EI;
2372     auto R = MatchInstructionImpl(Operands, Inst, EI, MatchingInlineAsm,
2373                                   Variant);
2374     // We order match statuses from least to most specific. We use most specific
2375     // status as resulting
2376     // Match_MnemonicFail < Match_InvalidOperand < Match_MissingFeature < Match_PreferE32
2377     if ((R == Match_Success) ||
2378         (R == Match_PreferE32) ||
2379         (R == Match_MissingFeature && Result != Match_PreferE32) ||
2380         (R == Match_InvalidOperand && Result != Match_MissingFeature
2381                                    && Result != Match_PreferE32) ||
2382         (R == Match_MnemonicFail   && Result != Match_InvalidOperand
2383                                    && Result != Match_MissingFeature
2384                                    && Result != Match_PreferE32)) {
2385       Result = R;
2386       ErrorInfo = EI;
2387     }
2388     if (R == Match_Success)
2389       break;
2390   }
2391 
2392   switch (Result) {
2393   default: break;
2394   case Match_Success:
2395     if (!validateInstruction(Inst, IDLoc)) {
2396       return true;
2397     }
2398     Inst.setLoc(IDLoc);
2399     Out.EmitInstruction(Inst, getSTI());
2400     return false;
2401 
2402   case Match_MissingFeature:
2403     return Error(IDLoc, "instruction not supported on this GPU");
2404 
2405   case Match_MnemonicFail: {
2406     uint64_t FBS = ComputeAvailableFeatures(getSTI().getFeatureBits());
2407     std::string Suggestion = AMDGPUMnemonicSpellCheck(
2408         ((AMDGPUOperand &)*Operands[0]).getToken(), FBS);
2409     return Error(IDLoc, "invalid instruction" + Suggestion,
2410                  ((AMDGPUOperand &)*Operands[0]).getLocRange());
2411   }
2412 
2413   case Match_InvalidOperand: {
2414     SMLoc ErrorLoc = IDLoc;
2415     if (ErrorInfo != ~0ULL) {
2416       if (ErrorInfo >= Operands.size()) {
2417         return Error(IDLoc, "too few operands for instruction");
2418       }
2419       ErrorLoc = ((AMDGPUOperand &)*Operands[ErrorInfo]).getStartLoc();
2420       if (ErrorLoc == SMLoc())
2421         ErrorLoc = IDLoc;
2422     }
2423     return Error(ErrorLoc, "invalid operand for instruction");
2424   }
2425 
2426   case Match_PreferE32:
2427     return Error(IDLoc, "internal error: instruction without _e64 suffix "
2428                         "should be encoded as e32");
2429   }
2430   llvm_unreachable("Implement any new match types added!");
2431 }
2432 
2433 bool AMDGPUAsmParser::ParseAsAbsoluteExpression(uint32_t &Ret) {
2434   int64_t Tmp = -1;
2435   if (getLexer().isNot(AsmToken::Integer) && getLexer().isNot(AsmToken::Identifier)) {
2436     return true;
2437   }
2438   if (getParser().parseAbsoluteExpression(Tmp)) {
2439     return true;
2440   }
2441   Ret = static_cast<uint32_t>(Tmp);
2442   return false;
2443 }
2444 
2445 bool AMDGPUAsmParser::ParseDirectiveMajorMinor(uint32_t &Major,
2446                                                uint32_t &Minor) {
2447   if (ParseAsAbsoluteExpression(Major))
2448     return TokError("invalid major version");
2449 
2450   if (getLexer().isNot(AsmToken::Comma))
2451     return TokError("minor version number required, comma expected");
2452   Lex();
2453 
2454   if (ParseAsAbsoluteExpression(Minor))
2455     return TokError("invalid minor version");
2456 
2457   return false;
2458 }
2459 
2460 bool AMDGPUAsmParser::ParseDirectiveHSACodeObjectVersion() {
2461   uint32_t Major;
2462   uint32_t Minor;
2463 
2464   if (ParseDirectiveMajorMinor(Major, Minor))
2465     return true;
2466 
2467   getTargetStreamer().EmitDirectiveHSACodeObjectVersion(Major, Minor);
2468   return false;
2469 }
2470 
2471 bool AMDGPUAsmParser::ParseDirectiveHSACodeObjectISA() {
2472   uint32_t Major;
2473   uint32_t Minor;
2474   uint32_t Stepping;
2475   StringRef VendorName;
2476   StringRef ArchName;
2477 
2478   // If this directive has no arguments, then use the ISA version for the
2479   // targeted GPU.
2480   if (getLexer().is(AsmToken::EndOfStatement)) {
2481     AMDGPU::IsaInfo::IsaVersion ISA =
2482         AMDGPU::IsaInfo::getIsaVersion(getFeatureBits());
2483     getTargetStreamer().EmitDirectiveHSACodeObjectISA(ISA.Major, ISA.Minor,
2484                                                       ISA.Stepping,
2485                                                       "AMD", "AMDGPU");
2486     return false;
2487   }
2488 
2489   if (ParseDirectiveMajorMinor(Major, Minor))
2490     return true;
2491 
2492   if (getLexer().isNot(AsmToken::Comma))
2493     return TokError("stepping version number required, comma expected");
2494   Lex();
2495 
2496   if (ParseAsAbsoluteExpression(Stepping))
2497     return TokError("invalid stepping version");
2498 
2499   if (getLexer().isNot(AsmToken::Comma))
2500     return TokError("vendor name required, comma expected");
2501   Lex();
2502 
2503   if (getLexer().isNot(AsmToken::String))
2504     return TokError("invalid vendor name");
2505 
2506   VendorName = getLexer().getTok().getStringContents();
2507   Lex();
2508 
2509   if (getLexer().isNot(AsmToken::Comma))
2510     return TokError("arch name required, comma expected");
2511   Lex();
2512 
2513   if (getLexer().isNot(AsmToken::String))
2514     return TokError("invalid arch name");
2515 
2516   ArchName = getLexer().getTok().getStringContents();
2517   Lex();
2518 
2519   getTargetStreamer().EmitDirectiveHSACodeObjectISA(Major, Minor, Stepping,
2520                                                     VendorName, ArchName);
2521   return false;
2522 }
2523 
2524 bool AMDGPUAsmParser::ParseAMDKernelCodeTValue(StringRef ID,
2525                                                amd_kernel_code_t &Header) {
2526   SmallString<40> ErrStr;
2527   raw_svector_ostream Err(ErrStr);
2528   if (!parseAmdKernelCodeField(ID, getParser(), Header, Err)) {
2529     return TokError(Err.str());
2530   }
2531   Lex();
2532   return false;
2533 }
2534 
2535 bool AMDGPUAsmParser::ParseDirectiveAMDKernelCodeT() {
2536   amd_kernel_code_t Header;
2537   AMDGPU::initDefaultAMDKernelCodeT(Header, getFeatureBits());
2538 
2539   while (true) {
2540     // Lex EndOfStatement.  This is in a while loop, because lexing a comment
2541     // will set the current token to EndOfStatement.
2542     while(getLexer().is(AsmToken::EndOfStatement))
2543       Lex();
2544 
2545     if (getLexer().isNot(AsmToken::Identifier))
2546       return TokError("expected value identifier or .end_amd_kernel_code_t");
2547 
2548     StringRef ID = getLexer().getTok().getIdentifier();
2549     Lex();
2550 
2551     if (ID == ".end_amd_kernel_code_t")
2552       break;
2553 
2554     if (ParseAMDKernelCodeTValue(ID, Header))
2555       return true;
2556   }
2557 
2558   getTargetStreamer().EmitAMDKernelCodeT(Header);
2559 
2560   return false;
2561 }
2562 
2563 bool AMDGPUAsmParser::ParseDirectiveAMDGPUHsaKernel() {
2564   if (getLexer().isNot(AsmToken::Identifier))
2565     return TokError("expected symbol name");
2566 
2567   StringRef KernelName = Parser.getTok().getString();
2568 
2569   getTargetStreamer().EmitAMDGPUSymbolType(KernelName,
2570                                            ELF::STT_AMDGPU_HSA_KERNEL);
2571   Lex();
2572   KernelScope.initialize(getContext());
2573   return false;
2574 }
2575 
2576 bool AMDGPUAsmParser::ParseDirectiveISAVersion() {
2577   if (getSTI().getTargetTriple().getArch() != Triple::amdgcn) {
2578     return Error(getParser().getTok().getLoc(),
2579                  ".amd_amdgpu_isa directive is not available on non-amdgcn "
2580                  "architectures");
2581   }
2582 
2583   auto ISAVersionStringFromASM = getLexer().getTok().getStringContents();
2584 
2585   std::string ISAVersionStringFromSTI;
2586   raw_string_ostream ISAVersionStreamFromSTI(ISAVersionStringFromSTI);
2587   IsaInfo::streamIsaVersion(&getSTI(), ISAVersionStreamFromSTI);
2588 
2589   if (ISAVersionStringFromASM != ISAVersionStreamFromSTI.str()) {
2590     return Error(getParser().getTok().getLoc(),
2591                  ".amd_amdgpu_isa directive does not match triple and/or mcpu "
2592                  "arguments specified through the command line");
2593   }
2594 
2595   getTargetStreamer().EmitISAVersion(ISAVersionStreamFromSTI.str());
2596   Lex();
2597 
2598   return false;
2599 }
2600 
2601 bool AMDGPUAsmParser::ParseDirectiveHSAMetadata() {
2602   if (getSTI().getTargetTriple().getOS() != Triple::AMDHSA) {
2603     return Error(getParser().getTok().getLoc(),
2604                  (Twine(HSAMD::AssemblerDirectiveBegin) + Twine(" directive is "
2605                  "not available on non-amdhsa OSes")).str());
2606   }
2607 
2608   std::string HSAMetadataString;
2609   raw_string_ostream YamlStream(HSAMetadataString);
2610 
2611   getLexer().setSkipSpace(false);
2612 
2613   bool FoundEnd = false;
2614   while (!getLexer().is(AsmToken::Eof)) {
2615     while (getLexer().is(AsmToken::Space)) {
2616       YamlStream << getLexer().getTok().getString();
2617       Lex();
2618     }
2619 
2620     if (getLexer().is(AsmToken::Identifier)) {
2621       StringRef ID = getLexer().getTok().getIdentifier();
2622       if (ID == AMDGPU::HSAMD::AssemblerDirectiveEnd) {
2623         Lex();
2624         FoundEnd = true;
2625         break;
2626       }
2627     }
2628 
2629     YamlStream << Parser.parseStringToEndOfStatement()
2630                << getContext().getAsmInfo()->getSeparatorString();
2631 
2632     Parser.eatToEndOfStatement();
2633   }
2634 
2635   getLexer().setSkipSpace(true);
2636 
2637   if (getLexer().is(AsmToken::Eof) && !FoundEnd) {
2638     return TokError(Twine("expected directive ") +
2639                     Twine(HSAMD::AssemblerDirectiveEnd) + Twine(" not found"));
2640   }
2641 
2642   YamlStream.flush();
2643 
2644   if (!getTargetStreamer().EmitHSAMetadata(HSAMetadataString))
2645     return Error(getParser().getTok().getLoc(), "invalid HSA metadata");
2646 
2647   return false;
2648 }
2649 
2650 bool AMDGPUAsmParser::ParseDirectivePALMetadata() {
2651   if (getSTI().getTargetTriple().getOS() != Triple::AMDPAL) {
2652     return Error(getParser().getTok().getLoc(),
2653                  (Twine(PALMD::AssemblerDirective) + Twine(" directive is "
2654                  "not available on non-amdpal OSes")).str());
2655   }
2656 
2657   PALMD::Metadata PALMetadata;
2658   for (;;) {
2659     uint32_t Value;
2660     if (ParseAsAbsoluteExpression(Value)) {
2661       return TokError(Twine("invalid value in ") +
2662                       Twine(PALMD::AssemblerDirective));
2663     }
2664     PALMetadata.push_back(Value);
2665     if (getLexer().isNot(AsmToken::Comma))
2666       break;
2667     Lex();
2668   }
2669   getTargetStreamer().EmitPALMetadata(PALMetadata);
2670   return false;
2671 }
2672 
2673 bool AMDGPUAsmParser::ParseDirective(AsmToken DirectiveID) {
2674   StringRef IDVal = DirectiveID.getString();
2675 
2676   if (IDVal == ".hsa_code_object_version")
2677     return ParseDirectiveHSACodeObjectVersion();
2678 
2679   if (IDVal == ".hsa_code_object_isa")
2680     return ParseDirectiveHSACodeObjectISA();
2681 
2682   if (IDVal == ".amd_kernel_code_t")
2683     return ParseDirectiveAMDKernelCodeT();
2684 
2685   if (IDVal == ".amdgpu_hsa_kernel")
2686     return ParseDirectiveAMDGPUHsaKernel();
2687 
2688   if (IDVal == ".amd_amdgpu_isa")
2689     return ParseDirectiveISAVersion();
2690 
2691   if (IDVal == AMDGPU::HSAMD::AssemblerDirectiveBegin)
2692     return ParseDirectiveHSAMetadata();
2693 
2694   if (IDVal == PALMD::AssemblerDirective)
2695     return ParseDirectivePALMetadata();
2696 
2697   return true;
2698 }
2699 
2700 bool AMDGPUAsmParser::subtargetHasRegister(const MCRegisterInfo &MRI,
2701                                            unsigned RegNo) const {
2702 
2703   for (MCRegAliasIterator R(AMDGPU::TTMP12_TTMP13_TTMP14_TTMP15, &MRI, true);
2704        R.isValid(); ++R) {
2705     if (*R == RegNo)
2706       return isGFX9();
2707   }
2708 
2709   switch (RegNo) {
2710   case AMDGPU::TBA:
2711   case AMDGPU::TBA_LO:
2712   case AMDGPU::TBA_HI:
2713   case AMDGPU::TMA:
2714   case AMDGPU::TMA_LO:
2715   case AMDGPU::TMA_HI:
2716     return !isGFX9();
2717   case AMDGPU::XNACK_MASK:
2718   case AMDGPU::XNACK_MASK_LO:
2719   case AMDGPU::XNACK_MASK_HI:
2720     return !isCI() && !isSI() && hasXNACK();
2721   default:
2722     break;
2723   }
2724 
2725   if (isCI())
2726     return true;
2727 
2728   if (isSI()) {
2729     // No flat_scr
2730     switch (RegNo) {
2731     case AMDGPU::FLAT_SCR:
2732     case AMDGPU::FLAT_SCR_LO:
2733     case AMDGPU::FLAT_SCR_HI:
2734       return false;
2735     default:
2736       return true;
2737     }
2738   }
2739 
2740   // VI only has 102 SGPRs, so make sure we aren't trying to use the 2 more that
2741   // SI/CI have.
2742   for (MCRegAliasIterator R(AMDGPU::SGPR102_SGPR103, &MRI, true);
2743        R.isValid(); ++R) {
2744     if (*R == RegNo)
2745       return false;
2746   }
2747 
2748   return true;
2749 }
2750 
2751 OperandMatchResultTy
2752 AMDGPUAsmParser::parseOperand(OperandVector &Operands, StringRef Mnemonic) {
2753   // Try to parse with a custom parser
2754   OperandMatchResultTy ResTy = MatchOperandParserImpl(Operands, Mnemonic);
2755 
2756   // If we successfully parsed the operand or if there as an error parsing,
2757   // we are done.
2758   //
2759   // If we are parsing after we reach EndOfStatement then this means we
2760   // are appending default values to the Operands list.  This is only done
2761   // by custom parser, so we shouldn't continue on to the generic parsing.
2762   if (ResTy == MatchOperand_Success || ResTy == MatchOperand_ParseFail ||
2763       getLexer().is(AsmToken::EndOfStatement))
2764     return ResTy;
2765 
2766   ResTy = parseRegOrImm(Operands);
2767 
2768   if (ResTy == MatchOperand_Success)
2769     return ResTy;
2770 
2771   const auto &Tok = Parser.getTok();
2772   SMLoc S = Tok.getLoc();
2773 
2774   const MCExpr *Expr = nullptr;
2775   if (!Parser.parseExpression(Expr)) {
2776     Operands.push_back(AMDGPUOperand::CreateExpr(this, Expr, S));
2777     return MatchOperand_Success;
2778   }
2779 
2780   // Possibly this is an instruction flag like 'gds'.
2781   if (Tok.getKind() == AsmToken::Identifier) {
2782     Operands.push_back(AMDGPUOperand::CreateToken(this, Tok.getString(), S));
2783     Parser.Lex();
2784     return MatchOperand_Success;
2785   }
2786 
2787   return MatchOperand_NoMatch;
2788 }
2789 
2790 StringRef AMDGPUAsmParser::parseMnemonicSuffix(StringRef Name) {
2791   // Clear any forced encodings from the previous instruction.
2792   setForcedEncodingSize(0);
2793   setForcedDPP(false);
2794   setForcedSDWA(false);
2795 
2796   if (Name.endswith("_e64")) {
2797     setForcedEncodingSize(64);
2798     return Name.substr(0, Name.size() - 4);
2799   } else if (Name.endswith("_e32")) {
2800     setForcedEncodingSize(32);
2801     return Name.substr(0, Name.size() - 4);
2802   } else if (Name.endswith("_dpp")) {
2803     setForcedDPP(true);
2804     return Name.substr(0, Name.size() - 4);
2805   } else if (Name.endswith("_sdwa")) {
2806     setForcedSDWA(true);
2807     return Name.substr(0, Name.size() - 5);
2808   }
2809   return Name;
2810 }
2811 
2812 bool AMDGPUAsmParser::ParseInstruction(ParseInstructionInfo &Info,
2813                                        StringRef Name,
2814                                        SMLoc NameLoc, OperandVector &Operands) {
2815   // Add the instruction mnemonic
2816   Name = parseMnemonicSuffix(Name);
2817   Operands.push_back(AMDGPUOperand::CreateToken(this, Name, NameLoc));
2818 
2819   while (!getLexer().is(AsmToken::EndOfStatement)) {
2820     OperandMatchResultTy Res = parseOperand(Operands, Name);
2821 
2822     // Eat the comma or space if there is one.
2823     if (getLexer().is(AsmToken::Comma))
2824       Parser.Lex();
2825 
2826     switch (Res) {
2827       case MatchOperand_Success: break;
2828       case MatchOperand_ParseFail:
2829         Error(getLexer().getLoc(), "failed parsing operand.");
2830         while (!getLexer().is(AsmToken::EndOfStatement)) {
2831           Parser.Lex();
2832         }
2833         return true;
2834       case MatchOperand_NoMatch:
2835         Error(getLexer().getLoc(), "not a valid operand.");
2836         while (!getLexer().is(AsmToken::EndOfStatement)) {
2837           Parser.Lex();
2838         }
2839         return true;
2840     }
2841   }
2842 
2843   return false;
2844 }
2845 
2846 //===----------------------------------------------------------------------===//
2847 // Utility functions
2848 //===----------------------------------------------------------------------===//
2849 
2850 OperandMatchResultTy
2851 AMDGPUAsmParser::parseIntWithPrefix(const char *Prefix, int64_t &Int) {
2852   switch(getLexer().getKind()) {
2853     default: return MatchOperand_NoMatch;
2854     case AsmToken::Identifier: {
2855       StringRef Name = Parser.getTok().getString();
2856       if (!Name.equals(Prefix)) {
2857         return MatchOperand_NoMatch;
2858       }
2859 
2860       Parser.Lex();
2861       if (getLexer().isNot(AsmToken::Colon))
2862         return MatchOperand_ParseFail;
2863 
2864       Parser.Lex();
2865 
2866       bool IsMinus = false;
2867       if (getLexer().getKind() == AsmToken::Minus) {
2868         Parser.Lex();
2869         IsMinus = true;
2870       }
2871 
2872       if (getLexer().isNot(AsmToken::Integer))
2873         return MatchOperand_ParseFail;
2874 
2875       if (getParser().parseAbsoluteExpression(Int))
2876         return MatchOperand_ParseFail;
2877 
2878       if (IsMinus)
2879         Int = -Int;
2880       break;
2881     }
2882   }
2883   return MatchOperand_Success;
2884 }
2885 
2886 OperandMatchResultTy
2887 AMDGPUAsmParser::parseIntWithPrefix(const char *Prefix, OperandVector &Operands,
2888                                     AMDGPUOperand::ImmTy ImmTy,
2889                                     bool (*ConvertResult)(int64_t&)) {
2890   SMLoc S = Parser.getTok().getLoc();
2891   int64_t Value = 0;
2892 
2893   OperandMatchResultTy Res = parseIntWithPrefix(Prefix, Value);
2894   if (Res != MatchOperand_Success)
2895     return Res;
2896 
2897   if (ConvertResult && !ConvertResult(Value)) {
2898     return MatchOperand_ParseFail;
2899   }
2900 
2901   Operands.push_back(AMDGPUOperand::CreateImm(this, Value, S, ImmTy));
2902   return MatchOperand_Success;
2903 }
2904 
2905 OperandMatchResultTy AMDGPUAsmParser::parseOperandArrayWithPrefix(
2906   const char *Prefix,
2907   OperandVector &Operands,
2908   AMDGPUOperand::ImmTy ImmTy,
2909   bool (*ConvertResult)(int64_t&)) {
2910   StringRef Name = Parser.getTok().getString();
2911   if (!Name.equals(Prefix))
2912     return MatchOperand_NoMatch;
2913 
2914   Parser.Lex();
2915   if (getLexer().isNot(AsmToken::Colon))
2916     return MatchOperand_ParseFail;
2917 
2918   Parser.Lex();
2919   if (getLexer().isNot(AsmToken::LBrac))
2920     return MatchOperand_ParseFail;
2921   Parser.Lex();
2922 
2923   unsigned Val = 0;
2924   SMLoc S = Parser.getTok().getLoc();
2925 
2926   // FIXME: How to verify the number of elements matches the number of src
2927   // operands?
2928   for (int I = 0; I < 4; ++I) {
2929     if (I != 0) {
2930       if (getLexer().is(AsmToken::RBrac))
2931         break;
2932 
2933       if (getLexer().isNot(AsmToken::Comma))
2934         return MatchOperand_ParseFail;
2935       Parser.Lex();
2936     }
2937 
2938     if (getLexer().isNot(AsmToken::Integer))
2939       return MatchOperand_ParseFail;
2940 
2941     int64_t Op;
2942     if (getParser().parseAbsoluteExpression(Op))
2943       return MatchOperand_ParseFail;
2944 
2945     if (Op != 0 && Op != 1)
2946       return MatchOperand_ParseFail;
2947     Val |= (Op << I);
2948   }
2949 
2950   Parser.Lex();
2951   Operands.push_back(AMDGPUOperand::CreateImm(this, Val, S, ImmTy));
2952   return MatchOperand_Success;
2953 }
2954 
2955 OperandMatchResultTy
2956 AMDGPUAsmParser::parseNamedBit(const char *Name, OperandVector &Operands,
2957                                AMDGPUOperand::ImmTy ImmTy) {
2958   int64_t Bit = 0;
2959   SMLoc S = Parser.getTok().getLoc();
2960 
2961   // We are at the end of the statement, and this is a default argument, so
2962   // use a default value.
2963   if (getLexer().isNot(AsmToken::EndOfStatement)) {
2964     switch(getLexer().getKind()) {
2965       case AsmToken::Identifier: {
2966         StringRef Tok = Parser.getTok().getString();
2967         if (Tok == Name) {
2968           Bit = 1;
2969           Parser.Lex();
2970         } else if (Tok.startswith("no") && Tok.endswith(Name)) {
2971           Bit = 0;
2972           Parser.Lex();
2973         } else {
2974           return MatchOperand_NoMatch;
2975         }
2976         break;
2977       }
2978       default:
2979         return MatchOperand_NoMatch;
2980     }
2981   }
2982 
2983   Operands.push_back(AMDGPUOperand::CreateImm(this, Bit, S, ImmTy));
2984   return MatchOperand_Success;
2985 }
2986 
2987 static void addOptionalImmOperand(
2988   MCInst& Inst, const OperandVector& Operands,
2989   AMDGPUAsmParser::OptionalImmIndexMap& OptionalIdx,
2990   AMDGPUOperand::ImmTy ImmT,
2991   int64_t Default = 0) {
2992   auto i = OptionalIdx.find(ImmT);
2993   if (i != OptionalIdx.end()) {
2994     unsigned Idx = i->second;
2995     ((AMDGPUOperand &)*Operands[Idx]).addImmOperands(Inst, 1);
2996   } else {
2997     Inst.addOperand(MCOperand::createImm(Default));
2998   }
2999 }
3000 
3001 OperandMatchResultTy
3002 AMDGPUAsmParser::parseStringWithPrefix(StringRef Prefix, StringRef &Value) {
3003   if (getLexer().isNot(AsmToken::Identifier)) {
3004     return MatchOperand_NoMatch;
3005   }
3006   StringRef Tok = Parser.getTok().getString();
3007   if (Tok != Prefix) {
3008     return MatchOperand_NoMatch;
3009   }
3010 
3011   Parser.Lex();
3012   if (getLexer().isNot(AsmToken::Colon)) {
3013     return MatchOperand_ParseFail;
3014   }
3015 
3016   Parser.Lex();
3017   if (getLexer().isNot(AsmToken::Identifier)) {
3018     return MatchOperand_ParseFail;
3019   }
3020 
3021   Value = Parser.getTok().getString();
3022   return MatchOperand_Success;
3023 }
3024 
3025 //===----------------------------------------------------------------------===//
3026 // ds
3027 //===----------------------------------------------------------------------===//
3028 
3029 void AMDGPUAsmParser::cvtDSOffset01(MCInst &Inst,
3030                                     const OperandVector &Operands) {
3031   OptionalImmIndexMap OptionalIdx;
3032 
3033   for (unsigned i = 1, e = Operands.size(); i != e; ++i) {
3034     AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[i]);
3035 
3036     // Add the register arguments
3037     if (Op.isReg()) {
3038       Op.addRegOperands(Inst, 1);
3039       continue;
3040     }
3041 
3042     // Handle optional arguments
3043     OptionalIdx[Op.getImmTy()] = i;
3044   }
3045 
3046   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyOffset0);
3047   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyOffset1);
3048   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyGDS);
3049 
3050   Inst.addOperand(MCOperand::createReg(AMDGPU::M0)); // m0
3051 }
3052 
3053 void AMDGPUAsmParser::cvtDSImpl(MCInst &Inst, const OperandVector &Operands,
3054                                 bool IsGdsHardcoded) {
3055   OptionalImmIndexMap OptionalIdx;
3056 
3057   for (unsigned i = 1, e = Operands.size(); i != e; ++i) {
3058     AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[i]);
3059 
3060     // Add the register arguments
3061     if (Op.isReg()) {
3062       Op.addRegOperands(Inst, 1);
3063       continue;
3064     }
3065 
3066     if (Op.isToken() && Op.getToken() == "gds") {
3067       IsGdsHardcoded = true;
3068       continue;
3069     }
3070 
3071     // Handle optional arguments
3072     OptionalIdx[Op.getImmTy()] = i;
3073   }
3074 
3075   AMDGPUOperand::ImmTy OffsetType =
3076     (Inst.getOpcode() == AMDGPU::DS_SWIZZLE_B32_si ||
3077      Inst.getOpcode() == AMDGPU::DS_SWIZZLE_B32_vi) ? AMDGPUOperand::ImmTySwizzle :
3078                                                       AMDGPUOperand::ImmTyOffset;
3079 
3080   addOptionalImmOperand(Inst, Operands, OptionalIdx, OffsetType);
3081 
3082   if (!IsGdsHardcoded) {
3083     addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyGDS);
3084   }
3085   Inst.addOperand(MCOperand::createReg(AMDGPU::M0)); // m0
3086 }
3087 
3088 void AMDGPUAsmParser::cvtExp(MCInst &Inst, const OperandVector &Operands) {
3089   OptionalImmIndexMap OptionalIdx;
3090 
3091   unsigned OperandIdx[4];
3092   unsigned EnMask = 0;
3093   int SrcIdx = 0;
3094 
3095   for (unsigned i = 1, e = Operands.size(); i != e; ++i) {
3096     AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[i]);
3097 
3098     // Add the register arguments
3099     if (Op.isReg()) {
3100       assert(SrcIdx < 4);
3101       OperandIdx[SrcIdx] = Inst.size();
3102       Op.addRegOperands(Inst, 1);
3103       ++SrcIdx;
3104       continue;
3105     }
3106 
3107     if (Op.isOff()) {
3108       assert(SrcIdx < 4);
3109       OperandIdx[SrcIdx] = Inst.size();
3110       Inst.addOperand(MCOperand::createReg(AMDGPU::NoRegister));
3111       ++SrcIdx;
3112       continue;
3113     }
3114 
3115     if (Op.isImm() && Op.getImmTy() == AMDGPUOperand::ImmTyExpTgt) {
3116       Op.addImmOperands(Inst, 1);
3117       continue;
3118     }
3119 
3120     if (Op.isToken() && Op.getToken() == "done")
3121       continue;
3122 
3123     // Handle optional arguments
3124     OptionalIdx[Op.getImmTy()] = i;
3125   }
3126 
3127   assert(SrcIdx == 4);
3128 
3129   bool Compr = false;
3130   if (OptionalIdx.find(AMDGPUOperand::ImmTyExpCompr) != OptionalIdx.end()) {
3131     Compr = true;
3132     Inst.getOperand(OperandIdx[1]) = Inst.getOperand(OperandIdx[2]);
3133     Inst.getOperand(OperandIdx[2]).setReg(AMDGPU::NoRegister);
3134     Inst.getOperand(OperandIdx[3]).setReg(AMDGPU::NoRegister);
3135   }
3136 
3137   for (auto i = 0; i < SrcIdx; ++i) {
3138     if (Inst.getOperand(OperandIdx[i]).getReg() != AMDGPU::NoRegister) {
3139       EnMask |= Compr? (0x3 << i * 2) : (0x1 << i);
3140     }
3141   }
3142 
3143   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyExpVM);
3144   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyExpCompr);
3145 
3146   Inst.addOperand(MCOperand::createImm(EnMask));
3147 }
3148 
3149 //===----------------------------------------------------------------------===//
3150 // s_waitcnt
3151 //===----------------------------------------------------------------------===//
3152 
3153 static bool
3154 encodeCnt(
3155   const AMDGPU::IsaInfo::IsaVersion ISA,
3156   int64_t &IntVal,
3157   int64_t CntVal,
3158   bool Saturate,
3159   unsigned (*encode)(const IsaInfo::IsaVersion &Version, unsigned, unsigned),
3160   unsigned (*decode)(const IsaInfo::IsaVersion &Version, unsigned))
3161 {
3162   bool Failed = false;
3163 
3164   IntVal = encode(ISA, IntVal, CntVal);
3165   if (CntVal != decode(ISA, IntVal)) {
3166     if (Saturate) {
3167       IntVal = encode(ISA, IntVal, -1);
3168     } else {
3169       Failed = true;
3170     }
3171   }
3172   return Failed;
3173 }
3174 
3175 bool AMDGPUAsmParser::parseCnt(int64_t &IntVal) {
3176   StringRef CntName = Parser.getTok().getString();
3177   int64_t CntVal;
3178 
3179   Parser.Lex();
3180   if (getLexer().isNot(AsmToken::LParen))
3181     return true;
3182 
3183   Parser.Lex();
3184   if (getLexer().isNot(AsmToken::Integer))
3185     return true;
3186 
3187   SMLoc ValLoc = Parser.getTok().getLoc();
3188   if (getParser().parseAbsoluteExpression(CntVal))
3189     return true;
3190 
3191   AMDGPU::IsaInfo::IsaVersion ISA =
3192       AMDGPU::IsaInfo::getIsaVersion(getFeatureBits());
3193 
3194   bool Failed = true;
3195   bool Sat = CntName.endswith("_sat");
3196 
3197   if (CntName == "vmcnt" || CntName == "vmcnt_sat") {
3198     Failed = encodeCnt(ISA, IntVal, CntVal, Sat, encodeVmcnt, decodeVmcnt);
3199   } else if (CntName == "expcnt" || CntName == "expcnt_sat") {
3200     Failed = encodeCnt(ISA, IntVal, CntVal, Sat, encodeExpcnt, decodeExpcnt);
3201   } else if (CntName == "lgkmcnt" || CntName == "lgkmcnt_sat") {
3202     Failed = encodeCnt(ISA, IntVal, CntVal, Sat, encodeLgkmcnt, decodeLgkmcnt);
3203   }
3204 
3205   if (Failed) {
3206     Error(ValLoc, "too large value for " + CntName);
3207     return true;
3208   }
3209 
3210   if (getLexer().isNot(AsmToken::RParen)) {
3211     return true;
3212   }
3213 
3214   Parser.Lex();
3215   if (getLexer().is(AsmToken::Amp) || getLexer().is(AsmToken::Comma)) {
3216     const AsmToken NextToken = getLexer().peekTok();
3217     if (NextToken.is(AsmToken::Identifier)) {
3218       Parser.Lex();
3219     }
3220   }
3221 
3222   return false;
3223 }
3224 
3225 OperandMatchResultTy
3226 AMDGPUAsmParser::parseSWaitCntOps(OperandVector &Operands) {
3227   AMDGPU::IsaInfo::IsaVersion ISA =
3228       AMDGPU::IsaInfo::getIsaVersion(getFeatureBits());
3229   int64_t Waitcnt = getWaitcntBitMask(ISA);
3230   SMLoc S = Parser.getTok().getLoc();
3231 
3232   switch(getLexer().getKind()) {
3233     default: return MatchOperand_ParseFail;
3234     case AsmToken::Integer:
3235       // The operand can be an integer value.
3236       if (getParser().parseAbsoluteExpression(Waitcnt))
3237         return MatchOperand_ParseFail;
3238       break;
3239 
3240     case AsmToken::Identifier:
3241       do {
3242         if (parseCnt(Waitcnt))
3243           return MatchOperand_ParseFail;
3244       } while(getLexer().isNot(AsmToken::EndOfStatement));
3245       break;
3246   }
3247   Operands.push_back(AMDGPUOperand::CreateImm(this, Waitcnt, S));
3248   return MatchOperand_Success;
3249 }
3250 
3251 bool AMDGPUAsmParser::parseHwregConstruct(OperandInfoTy &HwReg, int64_t &Offset,
3252                                           int64_t &Width) {
3253   using namespace llvm::AMDGPU::Hwreg;
3254 
3255   if (Parser.getTok().getString() != "hwreg")
3256     return true;
3257   Parser.Lex();
3258 
3259   if (getLexer().isNot(AsmToken::LParen))
3260     return true;
3261   Parser.Lex();
3262 
3263   if (getLexer().is(AsmToken::Identifier)) {
3264     HwReg.IsSymbolic = true;
3265     HwReg.Id = ID_UNKNOWN_;
3266     const StringRef tok = Parser.getTok().getString();
3267     int Last = ID_SYMBOLIC_LAST_;
3268     if (isSI() || isCI() || isVI())
3269       Last = ID_SYMBOLIC_FIRST_GFX9_;
3270     for (int i = ID_SYMBOLIC_FIRST_; i < Last; ++i) {
3271       if (tok == IdSymbolic[i]) {
3272         HwReg.Id = i;
3273         break;
3274       }
3275     }
3276     Parser.Lex();
3277   } else {
3278     HwReg.IsSymbolic = false;
3279     if (getLexer().isNot(AsmToken::Integer))
3280       return true;
3281     if (getParser().parseAbsoluteExpression(HwReg.Id))
3282       return true;
3283   }
3284 
3285   if (getLexer().is(AsmToken::RParen)) {
3286     Parser.Lex();
3287     return false;
3288   }
3289 
3290   // optional params
3291   if (getLexer().isNot(AsmToken::Comma))
3292     return true;
3293   Parser.Lex();
3294 
3295   if (getLexer().isNot(AsmToken::Integer))
3296     return true;
3297   if (getParser().parseAbsoluteExpression(Offset))
3298     return true;
3299 
3300   if (getLexer().isNot(AsmToken::Comma))
3301     return true;
3302   Parser.Lex();
3303 
3304   if (getLexer().isNot(AsmToken::Integer))
3305     return true;
3306   if (getParser().parseAbsoluteExpression(Width))
3307     return true;
3308 
3309   if (getLexer().isNot(AsmToken::RParen))
3310     return true;
3311   Parser.Lex();
3312 
3313   return false;
3314 }
3315 
3316 OperandMatchResultTy AMDGPUAsmParser::parseHwreg(OperandVector &Operands) {
3317   using namespace llvm::AMDGPU::Hwreg;
3318 
3319   int64_t Imm16Val = 0;
3320   SMLoc S = Parser.getTok().getLoc();
3321 
3322   switch(getLexer().getKind()) {
3323     default: return MatchOperand_NoMatch;
3324     case AsmToken::Integer:
3325       // The operand can be an integer value.
3326       if (getParser().parseAbsoluteExpression(Imm16Val))
3327         return MatchOperand_NoMatch;
3328       if (Imm16Val < 0 || !isUInt<16>(Imm16Val)) {
3329         Error(S, "invalid immediate: only 16-bit values are legal");
3330         // Do not return error code, but create an imm operand anyway and proceed
3331         // to the next operand, if any. That avoids unneccessary error messages.
3332       }
3333       break;
3334 
3335     case AsmToken::Identifier: {
3336         OperandInfoTy HwReg(ID_UNKNOWN_);
3337         int64_t Offset = OFFSET_DEFAULT_;
3338         int64_t Width = WIDTH_M1_DEFAULT_ + 1;
3339         if (parseHwregConstruct(HwReg, Offset, Width))
3340           return MatchOperand_ParseFail;
3341         if (HwReg.Id < 0 || !isUInt<ID_WIDTH_>(HwReg.Id)) {
3342           if (HwReg.IsSymbolic)
3343             Error(S, "invalid symbolic name of hardware register");
3344           else
3345             Error(S, "invalid code of hardware register: only 6-bit values are legal");
3346         }
3347         if (Offset < 0 || !isUInt<OFFSET_WIDTH_>(Offset))
3348           Error(S, "invalid bit offset: only 5-bit values are legal");
3349         if ((Width-1) < 0 || !isUInt<WIDTH_M1_WIDTH_>(Width-1))
3350           Error(S, "invalid bitfield width: only values from 1 to 32 are legal");
3351         Imm16Val = (HwReg.Id << ID_SHIFT_) | (Offset << OFFSET_SHIFT_) | ((Width-1) << WIDTH_M1_SHIFT_);
3352       }
3353       break;
3354   }
3355   Operands.push_back(AMDGPUOperand::CreateImm(this, Imm16Val, S, AMDGPUOperand::ImmTyHwreg));
3356   return MatchOperand_Success;
3357 }
3358 
3359 bool AMDGPUOperand::isSWaitCnt() const {
3360   return isImm();
3361 }
3362 
3363 bool AMDGPUOperand::isHwreg() const {
3364   return isImmTy(ImmTyHwreg);
3365 }
3366 
3367 bool AMDGPUAsmParser::parseSendMsgConstruct(OperandInfoTy &Msg, OperandInfoTy &Operation, int64_t &StreamId) {
3368   using namespace llvm::AMDGPU::SendMsg;
3369 
3370   if (Parser.getTok().getString() != "sendmsg")
3371     return true;
3372   Parser.Lex();
3373 
3374   if (getLexer().isNot(AsmToken::LParen))
3375     return true;
3376   Parser.Lex();
3377 
3378   if (getLexer().is(AsmToken::Identifier)) {
3379     Msg.IsSymbolic = true;
3380     Msg.Id = ID_UNKNOWN_;
3381     const std::string tok = Parser.getTok().getString();
3382     for (int i = ID_GAPS_FIRST_; i < ID_GAPS_LAST_; ++i) {
3383       switch(i) {
3384         default: continue; // Omit gaps.
3385         case ID_INTERRUPT: case ID_GS: case ID_GS_DONE:  case ID_SYSMSG: break;
3386       }
3387       if (tok == IdSymbolic[i]) {
3388         Msg.Id = i;
3389         break;
3390       }
3391     }
3392     Parser.Lex();
3393   } else {
3394     Msg.IsSymbolic = false;
3395     if (getLexer().isNot(AsmToken::Integer))
3396       return true;
3397     if (getParser().parseAbsoluteExpression(Msg.Id))
3398       return true;
3399     if (getLexer().is(AsmToken::Integer))
3400       if (getParser().parseAbsoluteExpression(Msg.Id))
3401         Msg.Id = ID_UNKNOWN_;
3402   }
3403   if (Msg.Id == ID_UNKNOWN_) // Don't know how to parse the rest.
3404     return false;
3405 
3406   if (!(Msg.Id == ID_GS || Msg.Id == ID_GS_DONE || Msg.Id == ID_SYSMSG)) {
3407     if (getLexer().isNot(AsmToken::RParen))
3408       return true;
3409     Parser.Lex();
3410     return false;
3411   }
3412 
3413   if (getLexer().isNot(AsmToken::Comma))
3414     return true;
3415   Parser.Lex();
3416 
3417   assert(Msg.Id == ID_GS || Msg.Id == ID_GS_DONE || Msg.Id == ID_SYSMSG);
3418   Operation.Id = ID_UNKNOWN_;
3419   if (getLexer().is(AsmToken::Identifier)) {
3420     Operation.IsSymbolic = true;
3421     const char* const *S = (Msg.Id == ID_SYSMSG) ? OpSysSymbolic : OpGsSymbolic;
3422     const int F = (Msg.Id == ID_SYSMSG) ? OP_SYS_FIRST_ : OP_GS_FIRST_;
3423     const int L = (Msg.Id == ID_SYSMSG) ? OP_SYS_LAST_ : OP_GS_LAST_;
3424     const StringRef Tok = Parser.getTok().getString();
3425     for (int i = F; i < L; ++i) {
3426       if (Tok == S[i]) {
3427         Operation.Id = i;
3428         break;
3429       }
3430     }
3431     Parser.Lex();
3432   } else {
3433     Operation.IsSymbolic = false;
3434     if (getLexer().isNot(AsmToken::Integer))
3435       return true;
3436     if (getParser().parseAbsoluteExpression(Operation.Id))
3437       return true;
3438   }
3439 
3440   if ((Msg.Id == ID_GS || Msg.Id == ID_GS_DONE) && Operation.Id != OP_GS_NOP) {
3441     // Stream id is optional.
3442     if (getLexer().is(AsmToken::RParen)) {
3443       Parser.Lex();
3444       return false;
3445     }
3446 
3447     if (getLexer().isNot(AsmToken::Comma))
3448       return true;
3449     Parser.Lex();
3450 
3451     if (getLexer().isNot(AsmToken::Integer))
3452       return true;
3453     if (getParser().parseAbsoluteExpression(StreamId))
3454       return true;
3455   }
3456 
3457   if (getLexer().isNot(AsmToken::RParen))
3458     return true;
3459   Parser.Lex();
3460   return false;
3461 }
3462 
3463 OperandMatchResultTy AMDGPUAsmParser::parseInterpSlot(OperandVector &Operands) {
3464   if (getLexer().getKind() != AsmToken::Identifier)
3465     return MatchOperand_NoMatch;
3466 
3467   StringRef Str = Parser.getTok().getString();
3468   int Slot = StringSwitch<int>(Str)
3469     .Case("p10", 0)
3470     .Case("p20", 1)
3471     .Case("p0", 2)
3472     .Default(-1);
3473 
3474   SMLoc S = Parser.getTok().getLoc();
3475   if (Slot == -1)
3476     return MatchOperand_ParseFail;
3477 
3478   Parser.Lex();
3479   Operands.push_back(AMDGPUOperand::CreateImm(this, Slot, S,
3480                                               AMDGPUOperand::ImmTyInterpSlot));
3481   return MatchOperand_Success;
3482 }
3483 
3484 OperandMatchResultTy AMDGPUAsmParser::parseInterpAttr(OperandVector &Operands) {
3485   if (getLexer().getKind() != AsmToken::Identifier)
3486     return MatchOperand_NoMatch;
3487 
3488   StringRef Str = Parser.getTok().getString();
3489   if (!Str.startswith("attr"))
3490     return MatchOperand_NoMatch;
3491 
3492   StringRef Chan = Str.take_back(2);
3493   int AttrChan = StringSwitch<int>(Chan)
3494     .Case(".x", 0)
3495     .Case(".y", 1)
3496     .Case(".z", 2)
3497     .Case(".w", 3)
3498     .Default(-1);
3499   if (AttrChan == -1)
3500     return MatchOperand_ParseFail;
3501 
3502   Str = Str.drop_back(2).drop_front(4);
3503 
3504   uint8_t Attr;
3505   if (Str.getAsInteger(10, Attr))
3506     return MatchOperand_ParseFail;
3507 
3508   SMLoc S = Parser.getTok().getLoc();
3509   Parser.Lex();
3510   if (Attr > 63) {
3511     Error(S, "out of bounds attr");
3512     return MatchOperand_Success;
3513   }
3514 
3515   SMLoc SChan = SMLoc::getFromPointer(Chan.data());
3516 
3517   Operands.push_back(AMDGPUOperand::CreateImm(this, Attr, S,
3518                                               AMDGPUOperand::ImmTyInterpAttr));
3519   Operands.push_back(AMDGPUOperand::CreateImm(this, AttrChan, SChan,
3520                                               AMDGPUOperand::ImmTyAttrChan));
3521   return MatchOperand_Success;
3522 }
3523 
3524 void AMDGPUAsmParser::errorExpTgt() {
3525   Error(Parser.getTok().getLoc(), "invalid exp target");
3526 }
3527 
3528 OperandMatchResultTy AMDGPUAsmParser::parseExpTgtImpl(StringRef Str,
3529                                                       uint8_t &Val) {
3530   if (Str == "null") {
3531     Val = 9;
3532     return MatchOperand_Success;
3533   }
3534 
3535   if (Str.startswith("mrt")) {
3536     Str = Str.drop_front(3);
3537     if (Str == "z") { // == mrtz
3538       Val = 8;
3539       return MatchOperand_Success;
3540     }
3541 
3542     if (Str.getAsInteger(10, Val))
3543       return MatchOperand_ParseFail;
3544 
3545     if (Val > 7)
3546       errorExpTgt();
3547 
3548     return MatchOperand_Success;
3549   }
3550 
3551   if (Str.startswith("pos")) {
3552     Str = Str.drop_front(3);
3553     if (Str.getAsInteger(10, Val))
3554       return MatchOperand_ParseFail;
3555 
3556     if (Val > 3)
3557       errorExpTgt();
3558 
3559     Val += 12;
3560     return MatchOperand_Success;
3561   }
3562 
3563   if (Str.startswith("param")) {
3564     Str = Str.drop_front(5);
3565     if (Str.getAsInteger(10, Val))
3566       return MatchOperand_ParseFail;
3567 
3568     if (Val >= 32)
3569       errorExpTgt();
3570 
3571     Val += 32;
3572     return MatchOperand_Success;
3573   }
3574 
3575   if (Str.startswith("invalid_target_")) {
3576     Str = Str.drop_front(15);
3577     if (Str.getAsInteger(10, Val))
3578       return MatchOperand_ParseFail;
3579 
3580     errorExpTgt();
3581     return MatchOperand_Success;
3582   }
3583 
3584   return MatchOperand_NoMatch;
3585 }
3586 
3587 OperandMatchResultTy AMDGPUAsmParser::parseExpTgt(OperandVector &Operands) {
3588   uint8_t Val;
3589   StringRef Str = Parser.getTok().getString();
3590 
3591   auto Res = parseExpTgtImpl(Str, Val);
3592   if (Res != MatchOperand_Success)
3593     return Res;
3594 
3595   SMLoc S = Parser.getTok().getLoc();
3596   Parser.Lex();
3597 
3598   Operands.push_back(AMDGPUOperand::CreateImm(this, Val, S,
3599                                               AMDGPUOperand::ImmTyExpTgt));
3600   return MatchOperand_Success;
3601 }
3602 
3603 OperandMatchResultTy
3604 AMDGPUAsmParser::parseSendMsgOp(OperandVector &Operands) {
3605   using namespace llvm::AMDGPU::SendMsg;
3606 
3607   int64_t Imm16Val = 0;
3608   SMLoc S = Parser.getTok().getLoc();
3609 
3610   switch(getLexer().getKind()) {
3611   default:
3612     return MatchOperand_NoMatch;
3613   case AsmToken::Integer:
3614     // The operand can be an integer value.
3615     if (getParser().parseAbsoluteExpression(Imm16Val))
3616       return MatchOperand_NoMatch;
3617     if (Imm16Val < 0 || !isUInt<16>(Imm16Val)) {
3618       Error(S, "invalid immediate: only 16-bit values are legal");
3619       // Do not return error code, but create an imm operand anyway and proceed
3620       // to the next operand, if any. That avoids unneccessary error messages.
3621     }
3622     break;
3623   case AsmToken::Identifier: {
3624       OperandInfoTy Msg(ID_UNKNOWN_);
3625       OperandInfoTy Operation(OP_UNKNOWN_);
3626       int64_t StreamId = STREAM_ID_DEFAULT_;
3627       if (parseSendMsgConstruct(Msg, Operation, StreamId))
3628         return MatchOperand_ParseFail;
3629       do {
3630         // Validate and encode message ID.
3631         if (! ((ID_INTERRUPT <= Msg.Id && Msg.Id <= ID_GS_DONE)
3632                 || Msg.Id == ID_SYSMSG)) {
3633           if (Msg.IsSymbolic)
3634             Error(S, "invalid/unsupported symbolic name of message");
3635           else
3636             Error(S, "invalid/unsupported code of message");
3637           break;
3638         }
3639         Imm16Val = (Msg.Id << ID_SHIFT_);
3640         // Validate and encode operation ID.
3641         if (Msg.Id == ID_GS || Msg.Id == ID_GS_DONE) {
3642           if (! (OP_GS_FIRST_ <= Operation.Id && Operation.Id < OP_GS_LAST_)) {
3643             if (Operation.IsSymbolic)
3644               Error(S, "invalid symbolic name of GS_OP");
3645             else
3646               Error(S, "invalid code of GS_OP: only 2-bit values are legal");
3647             break;
3648           }
3649           if (Operation.Id == OP_GS_NOP
3650               && Msg.Id != ID_GS_DONE) {
3651             Error(S, "invalid GS_OP: NOP is for GS_DONE only");
3652             break;
3653           }
3654           Imm16Val |= (Operation.Id << OP_SHIFT_);
3655         }
3656         if (Msg.Id == ID_SYSMSG) {
3657           if (! (OP_SYS_FIRST_ <= Operation.Id && Operation.Id < OP_SYS_LAST_)) {
3658             if (Operation.IsSymbolic)
3659               Error(S, "invalid/unsupported symbolic name of SYSMSG_OP");
3660             else
3661               Error(S, "invalid/unsupported code of SYSMSG_OP");
3662             break;
3663           }
3664           Imm16Val |= (Operation.Id << OP_SHIFT_);
3665         }
3666         // Validate and encode stream ID.
3667         if ((Msg.Id == ID_GS || Msg.Id == ID_GS_DONE) && Operation.Id != OP_GS_NOP) {
3668           if (! (STREAM_ID_FIRST_ <= StreamId && StreamId < STREAM_ID_LAST_)) {
3669             Error(S, "invalid stream id: only 2-bit values are legal");
3670             break;
3671           }
3672           Imm16Val |= (StreamId << STREAM_ID_SHIFT_);
3673         }
3674       } while (false);
3675     }
3676     break;
3677   }
3678   Operands.push_back(AMDGPUOperand::CreateImm(this, Imm16Val, S, AMDGPUOperand::ImmTySendMsg));
3679   return MatchOperand_Success;
3680 }
3681 
3682 bool AMDGPUOperand::isSendMsg() const {
3683   return isImmTy(ImmTySendMsg);
3684 }
3685 
3686 //===----------------------------------------------------------------------===//
3687 // parser helpers
3688 //===----------------------------------------------------------------------===//
3689 
3690 bool
3691 AMDGPUAsmParser::trySkipId(const StringRef Id) {
3692   if (getLexer().getKind() == AsmToken::Identifier &&
3693       Parser.getTok().getString() == Id) {
3694     Parser.Lex();
3695     return true;
3696   }
3697   return false;
3698 }
3699 
3700 bool
3701 AMDGPUAsmParser::trySkipToken(const AsmToken::TokenKind Kind) {
3702   if (getLexer().getKind() == Kind) {
3703     Parser.Lex();
3704     return true;
3705   }
3706   return false;
3707 }
3708 
3709 bool
3710 AMDGPUAsmParser::skipToken(const AsmToken::TokenKind Kind,
3711                            const StringRef ErrMsg) {
3712   if (!trySkipToken(Kind)) {
3713     Error(Parser.getTok().getLoc(), ErrMsg);
3714     return false;
3715   }
3716   return true;
3717 }
3718 
3719 bool
3720 AMDGPUAsmParser::parseExpr(int64_t &Imm) {
3721   return !getParser().parseAbsoluteExpression(Imm);
3722 }
3723 
3724 bool
3725 AMDGPUAsmParser::parseString(StringRef &Val, const StringRef ErrMsg) {
3726   SMLoc S = Parser.getTok().getLoc();
3727   if (getLexer().getKind() == AsmToken::String) {
3728     Val = Parser.getTok().getStringContents();
3729     Parser.Lex();
3730     return true;
3731   } else {
3732     Error(S, ErrMsg);
3733     return false;
3734   }
3735 }
3736 
3737 //===----------------------------------------------------------------------===//
3738 // swizzle
3739 //===----------------------------------------------------------------------===//
3740 
3741 LLVM_READNONE
3742 static unsigned
3743 encodeBitmaskPerm(const unsigned AndMask,
3744                   const unsigned OrMask,
3745                   const unsigned XorMask) {
3746   using namespace llvm::AMDGPU::Swizzle;
3747 
3748   return BITMASK_PERM_ENC |
3749          (AndMask << BITMASK_AND_SHIFT) |
3750          (OrMask  << BITMASK_OR_SHIFT)  |
3751          (XorMask << BITMASK_XOR_SHIFT);
3752 }
3753 
3754 bool
3755 AMDGPUAsmParser::parseSwizzleOperands(const unsigned OpNum, int64_t* Op,
3756                                       const unsigned MinVal,
3757                                       const unsigned MaxVal,
3758                                       const StringRef ErrMsg) {
3759   for (unsigned i = 0; i < OpNum; ++i) {
3760     if (!skipToken(AsmToken::Comma, "expected a comma")){
3761       return false;
3762     }
3763     SMLoc ExprLoc = Parser.getTok().getLoc();
3764     if (!parseExpr(Op[i])) {
3765       return false;
3766     }
3767     if (Op[i] < MinVal || Op[i] > MaxVal) {
3768       Error(ExprLoc, ErrMsg);
3769       return false;
3770     }
3771   }
3772 
3773   return true;
3774 }
3775 
3776 bool
3777 AMDGPUAsmParser::parseSwizzleQuadPerm(int64_t &Imm) {
3778   using namespace llvm::AMDGPU::Swizzle;
3779 
3780   int64_t Lane[LANE_NUM];
3781   if (parseSwizzleOperands(LANE_NUM, Lane, 0, LANE_MAX,
3782                            "expected a 2-bit lane id")) {
3783     Imm = QUAD_PERM_ENC;
3784     for (auto i = 0; i < LANE_NUM; ++i) {
3785       Imm |= Lane[i] << (LANE_SHIFT * i);
3786     }
3787     return true;
3788   }
3789   return false;
3790 }
3791 
3792 bool
3793 AMDGPUAsmParser::parseSwizzleBroadcast(int64_t &Imm) {
3794   using namespace llvm::AMDGPU::Swizzle;
3795 
3796   SMLoc S = Parser.getTok().getLoc();
3797   int64_t GroupSize;
3798   int64_t LaneIdx;
3799 
3800   if (!parseSwizzleOperands(1, &GroupSize,
3801                             2, 32,
3802                             "group size must be in the interval [2,32]")) {
3803     return false;
3804   }
3805   if (!isPowerOf2_64(GroupSize)) {
3806     Error(S, "group size must be a power of two");
3807     return false;
3808   }
3809   if (parseSwizzleOperands(1, &LaneIdx,
3810                            0, GroupSize - 1,
3811                            "lane id must be in the interval [0,group size - 1]")) {
3812     Imm = encodeBitmaskPerm(BITMASK_MAX - GroupSize + 1, LaneIdx, 0);
3813     return true;
3814   }
3815   return false;
3816 }
3817 
3818 bool
3819 AMDGPUAsmParser::parseSwizzleReverse(int64_t &Imm) {
3820   using namespace llvm::AMDGPU::Swizzle;
3821 
3822   SMLoc S = Parser.getTok().getLoc();
3823   int64_t GroupSize;
3824 
3825   if (!parseSwizzleOperands(1, &GroupSize,
3826       2, 32, "group size must be in the interval [2,32]")) {
3827     return false;
3828   }
3829   if (!isPowerOf2_64(GroupSize)) {
3830     Error(S, "group size must be a power of two");
3831     return false;
3832   }
3833 
3834   Imm = encodeBitmaskPerm(BITMASK_MAX, 0, GroupSize - 1);
3835   return true;
3836 }
3837 
3838 bool
3839 AMDGPUAsmParser::parseSwizzleSwap(int64_t &Imm) {
3840   using namespace llvm::AMDGPU::Swizzle;
3841 
3842   SMLoc S = Parser.getTok().getLoc();
3843   int64_t GroupSize;
3844 
3845   if (!parseSwizzleOperands(1, &GroupSize,
3846       1, 16, "group size must be in the interval [1,16]")) {
3847     return false;
3848   }
3849   if (!isPowerOf2_64(GroupSize)) {
3850     Error(S, "group size must be a power of two");
3851     return false;
3852   }
3853 
3854   Imm = encodeBitmaskPerm(BITMASK_MAX, 0, GroupSize);
3855   return true;
3856 }
3857 
3858 bool
3859 AMDGPUAsmParser::parseSwizzleBitmaskPerm(int64_t &Imm) {
3860   using namespace llvm::AMDGPU::Swizzle;
3861 
3862   if (!skipToken(AsmToken::Comma, "expected a comma")) {
3863     return false;
3864   }
3865 
3866   StringRef Ctl;
3867   SMLoc StrLoc = Parser.getTok().getLoc();
3868   if (!parseString(Ctl)) {
3869     return false;
3870   }
3871   if (Ctl.size() != BITMASK_WIDTH) {
3872     Error(StrLoc, "expected a 5-character mask");
3873     return false;
3874   }
3875 
3876   unsigned AndMask = 0;
3877   unsigned OrMask = 0;
3878   unsigned XorMask = 0;
3879 
3880   for (size_t i = 0; i < Ctl.size(); ++i) {
3881     unsigned Mask = 1 << (BITMASK_WIDTH - 1 - i);
3882     switch(Ctl[i]) {
3883     default:
3884       Error(StrLoc, "invalid mask");
3885       return false;
3886     case '0':
3887       break;
3888     case '1':
3889       OrMask |= Mask;
3890       break;
3891     case 'p':
3892       AndMask |= Mask;
3893       break;
3894     case 'i':
3895       AndMask |= Mask;
3896       XorMask |= Mask;
3897       break;
3898     }
3899   }
3900 
3901   Imm = encodeBitmaskPerm(AndMask, OrMask, XorMask);
3902   return true;
3903 }
3904 
3905 bool
3906 AMDGPUAsmParser::parseSwizzleOffset(int64_t &Imm) {
3907 
3908   SMLoc OffsetLoc = Parser.getTok().getLoc();
3909 
3910   if (!parseExpr(Imm)) {
3911     return false;
3912   }
3913   if (!isUInt<16>(Imm)) {
3914     Error(OffsetLoc, "expected a 16-bit offset");
3915     return false;
3916   }
3917   return true;
3918 }
3919 
3920 bool
3921 AMDGPUAsmParser::parseSwizzleMacro(int64_t &Imm) {
3922   using namespace llvm::AMDGPU::Swizzle;
3923 
3924   if (skipToken(AsmToken::LParen, "expected a left parentheses")) {
3925 
3926     SMLoc ModeLoc = Parser.getTok().getLoc();
3927     bool Ok = false;
3928 
3929     if (trySkipId(IdSymbolic[ID_QUAD_PERM])) {
3930       Ok = parseSwizzleQuadPerm(Imm);
3931     } else if (trySkipId(IdSymbolic[ID_BITMASK_PERM])) {
3932       Ok = parseSwizzleBitmaskPerm(Imm);
3933     } else if (trySkipId(IdSymbolic[ID_BROADCAST])) {
3934       Ok = parseSwizzleBroadcast(Imm);
3935     } else if (trySkipId(IdSymbolic[ID_SWAP])) {
3936       Ok = parseSwizzleSwap(Imm);
3937     } else if (trySkipId(IdSymbolic[ID_REVERSE])) {
3938       Ok = parseSwizzleReverse(Imm);
3939     } else {
3940       Error(ModeLoc, "expected a swizzle mode");
3941     }
3942 
3943     return Ok && skipToken(AsmToken::RParen, "expected a closing parentheses");
3944   }
3945 
3946   return false;
3947 }
3948 
3949 OperandMatchResultTy
3950 AMDGPUAsmParser::parseSwizzleOp(OperandVector &Operands) {
3951   SMLoc S = Parser.getTok().getLoc();
3952   int64_t Imm = 0;
3953 
3954   if (trySkipId("offset")) {
3955 
3956     bool Ok = false;
3957     if (skipToken(AsmToken::Colon, "expected a colon")) {
3958       if (trySkipId("swizzle")) {
3959         Ok = parseSwizzleMacro(Imm);
3960       } else {
3961         Ok = parseSwizzleOffset(Imm);
3962       }
3963     }
3964 
3965     Operands.push_back(AMDGPUOperand::CreateImm(this, Imm, S, AMDGPUOperand::ImmTySwizzle));
3966 
3967     return Ok? MatchOperand_Success : MatchOperand_ParseFail;
3968   } else {
3969     // Swizzle "offset" operand is optional.
3970     // If it is omitted, try parsing other optional operands.
3971     return parseOptionalOpr(Operands);
3972   }
3973 }
3974 
3975 bool
3976 AMDGPUOperand::isSwizzle() const {
3977   return isImmTy(ImmTySwizzle);
3978 }
3979 
3980 //===----------------------------------------------------------------------===//
3981 // sopp branch targets
3982 //===----------------------------------------------------------------------===//
3983 
3984 OperandMatchResultTy
3985 AMDGPUAsmParser::parseSOppBrTarget(OperandVector &Operands) {
3986   SMLoc S = Parser.getTok().getLoc();
3987 
3988   switch (getLexer().getKind()) {
3989     default: return MatchOperand_ParseFail;
3990     case AsmToken::Integer: {
3991       int64_t Imm;
3992       if (getParser().parseAbsoluteExpression(Imm))
3993         return MatchOperand_ParseFail;
3994       Operands.push_back(AMDGPUOperand::CreateImm(this, Imm, S));
3995       return MatchOperand_Success;
3996     }
3997 
3998     case AsmToken::Identifier:
3999       Operands.push_back(AMDGPUOperand::CreateExpr(this,
4000           MCSymbolRefExpr::create(getContext().getOrCreateSymbol(
4001                                   Parser.getTok().getString()), getContext()), S));
4002       Parser.Lex();
4003       return MatchOperand_Success;
4004   }
4005 }
4006 
4007 //===----------------------------------------------------------------------===//
4008 // mubuf
4009 //===----------------------------------------------------------------------===//
4010 
4011 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultGLC() const {
4012   return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTyGLC);
4013 }
4014 
4015 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultSLC() const {
4016   return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTySLC);
4017 }
4018 
4019 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultTFE() const {
4020   return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTyTFE);
4021 }
4022 
4023 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultD16() const {
4024   return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTyD16);
4025 }
4026 
4027 void AMDGPUAsmParser::cvtMubufImpl(MCInst &Inst,
4028                                const OperandVector &Operands,
4029                                bool IsAtomic, bool IsAtomicReturn) {
4030   OptionalImmIndexMap OptionalIdx;
4031   assert(IsAtomicReturn ? IsAtomic : true);
4032 
4033   for (unsigned i = 1, e = Operands.size(); i != e; ++i) {
4034     AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[i]);
4035 
4036     // Add the register arguments
4037     if (Op.isReg()) {
4038       Op.addRegOperands(Inst, 1);
4039       continue;
4040     }
4041 
4042     // Handle the case where soffset is an immediate
4043     if (Op.isImm() && Op.getImmTy() == AMDGPUOperand::ImmTyNone) {
4044       Op.addImmOperands(Inst, 1);
4045       continue;
4046     }
4047 
4048     // Handle tokens like 'offen' which are sometimes hard-coded into the
4049     // asm string.  There are no MCInst operands for these.
4050     if (Op.isToken()) {
4051       continue;
4052     }
4053     assert(Op.isImm());
4054 
4055     // Handle optional arguments
4056     OptionalIdx[Op.getImmTy()] = i;
4057   }
4058 
4059   // Copy $vdata_in operand and insert as $vdata for MUBUF_Atomic RTN insns.
4060   if (IsAtomicReturn) {
4061     MCInst::iterator I = Inst.begin(); // $vdata_in is always at the beginning.
4062     Inst.insert(I, *I);
4063   }
4064 
4065   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyOffset);
4066   if (!IsAtomic) { // glc is hard-coded.
4067     addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyGLC);
4068   }
4069   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySLC);
4070   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyTFE);
4071 }
4072 
4073 void AMDGPUAsmParser::cvtMtbuf(MCInst &Inst, const OperandVector &Operands) {
4074   OptionalImmIndexMap OptionalIdx;
4075 
4076   for (unsigned i = 1, e = Operands.size(); i != e; ++i) {
4077     AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[i]);
4078 
4079     // Add the register arguments
4080     if (Op.isReg()) {
4081       Op.addRegOperands(Inst, 1);
4082       continue;
4083     }
4084 
4085     // Handle the case where soffset is an immediate
4086     if (Op.isImm() && Op.getImmTy() == AMDGPUOperand::ImmTyNone) {
4087       Op.addImmOperands(Inst, 1);
4088       continue;
4089     }
4090 
4091     // Handle tokens like 'offen' which are sometimes hard-coded into the
4092     // asm string.  There are no MCInst operands for these.
4093     if (Op.isToken()) {
4094       continue;
4095     }
4096     assert(Op.isImm());
4097 
4098     // Handle optional arguments
4099     OptionalIdx[Op.getImmTy()] = i;
4100   }
4101 
4102   addOptionalImmOperand(Inst, Operands, OptionalIdx,
4103                         AMDGPUOperand::ImmTyOffset);
4104   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyDFMT);
4105   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyNFMT);
4106   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyGLC);
4107   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySLC);
4108   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyTFE);
4109 }
4110 
4111 //===----------------------------------------------------------------------===//
4112 // mimg
4113 //===----------------------------------------------------------------------===//
4114 
4115 void AMDGPUAsmParser::cvtMIMG(MCInst &Inst, const OperandVector &Operands,
4116                               bool IsAtomic) {
4117   unsigned I = 1;
4118   const MCInstrDesc &Desc = MII.get(Inst.getOpcode());
4119   for (unsigned J = 0; J < Desc.getNumDefs(); ++J) {
4120     ((AMDGPUOperand &)*Operands[I++]).addRegOperands(Inst, 1);
4121   }
4122 
4123   if (IsAtomic) {
4124     // Add src, same as dst
4125     assert(Desc.getNumDefs() == 1);
4126     ((AMDGPUOperand &)*Operands[I - 1]).addRegOperands(Inst, 1);
4127   }
4128 
4129   OptionalImmIndexMap OptionalIdx;
4130 
4131   for (unsigned E = Operands.size(); I != E; ++I) {
4132     AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[I]);
4133 
4134     // Add the register arguments
4135     if (Op.isReg()) {
4136       Op.addRegOperands(Inst, 1);
4137     } else if (Op.isImmModifier()) {
4138       OptionalIdx[Op.getImmTy()] = I;
4139     } else {
4140       llvm_unreachable("unexpected operand type");
4141     }
4142   }
4143 
4144   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyDMask);
4145   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyUNorm);
4146   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyGLC);
4147   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySLC);
4148   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyR128);
4149   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyTFE);
4150   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyLWE);
4151   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyDA);
4152 }
4153 
4154 void AMDGPUAsmParser::cvtMIMGAtomic(MCInst &Inst, const OperandVector &Operands) {
4155   cvtMIMG(Inst, Operands, true);
4156 }
4157 
4158 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultDMask() const {
4159   return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTyDMask);
4160 }
4161 
4162 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultUNorm() const {
4163   return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTyUNorm);
4164 }
4165 
4166 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultDA() const {
4167   return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTyDA);
4168 }
4169 
4170 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultR128() const {
4171   return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTyR128);
4172 }
4173 
4174 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultLWE() const {
4175   return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTyLWE);
4176 }
4177 
4178 //===----------------------------------------------------------------------===//
4179 // smrd
4180 //===----------------------------------------------------------------------===//
4181 
4182 bool AMDGPUOperand::isSMRDOffset8() const {
4183   return isImm() && isUInt<8>(getImm());
4184 }
4185 
4186 bool AMDGPUOperand::isSMRDOffset20() const {
4187   return isImm() && isUInt<20>(getImm());
4188 }
4189 
4190 bool AMDGPUOperand::isSMRDLiteralOffset() const {
4191   // 32-bit literals are only supported on CI and we only want to use them
4192   // when the offset is > 8-bits.
4193   return isImm() && !isUInt<8>(getImm()) && isUInt<32>(getImm());
4194 }
4195 
4196 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultSMRDOffset8() const {
4197   return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTyOffset);
4198 }
4199 
4200 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultSMRDOffset20() const {
4201   return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTyOffset);
4202 }
4203 
4204 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultSMRDLiteralOffset() const {
4205   return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTyOffset);
4206 }
4207 
4208 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultOffsetU12() const {
4209   return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTyOffset);
4210 }
4211 
4212 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultOffsetS13() const {
4213   return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTyOffset);
4214 }
4215 
4216 //===----------------------------------------------------------------------===//
4217 // vop3
4218 //===----------------------------------------------------------------------===//
4219 
4220 static bool ConvertOmodMul(int64_t &Mul) {
4221   if (Mul != 1 && Mul != 2 && Mul != 4)
4222     return false;
4223 
4224   Mul >>= 1;
4225   return true;
4226 }
4227 
4228 static bool ConvertOmodDiv(int64_t &Div) {
4229   if (Div == 1) {
4230     Div = 0;
4231     return true;
4232   }
4233 
4234   if (Div == 2) {
4235     Div = 3;
4236     return true;
4237   }
4238 
4239   return false;
4240 }
4241 
4242 static bool ConvertBoundCtrl(int64_t &BoundCtrl) {
4243   if (BoundCtrl == 0) {
4244     BoundCtrl = 1;
4245     return true;
4246   }
4247 
4248   if (BoundCtrl == -1) {
4249     BoundCtrl = 0;
4250     return true;
4251   }
4252 
4253   return false;
4254 }
4255 
4256 // Note: the order in this table matches the order of operands in AsmString.
4257 static const OptionalOperand AMDGPUOptionalOperandTable[] = {
4258   {"offen",   AMDGPUOperand::ImmTyOffen, true, nullptr},
4259   {"idxen",   AMDGPUOperand::ImmTyIdxen, true, nullptr},
4260   {"addr64",  AMDGPUOperand::ImmTyAddr64, true, nullptr},
4261   {"offset0", AMDGPUOperand::ImmTyOffset0, false, nullptr},
4262   {"offset1", AMDGPUOperand::ImmTyOffset1, false, nullptr},
4263   {"gds",     AMDGPUOperand::ImmTyGDS, true, nullptr},
4264   {"offset",  AMDGPUOperand::ImmTyOffset, false, nullptr},
4265   {"inst_offset", AMDGPUOperand::ImmTyInstOffset, false, nullptr},
4266   {"dfmt",    AMDGPUOperand::ImmTyDFMT, false, nullptr},
4267   {"nfmt",    AMDGPUOperand::ImmTyNFMT, false, nullptr},
4268   {"glc",     AMDGPUOperand::ImmTyGLC, true, nullptr},
4269   {"slc",     AMDGPUOperand::ImmTySLC, true, nullptr},
4270   {"tfe",     AMDGPUOperand::ImmTyTFE, true, nullptr},
4271   {"d16",     AMDGPUOperand::ImmTyD16, true, nullptr},
4272   {"high",    AMDGPUOperand::ImmTyHigh, true, nullptr},
4273   {"clamp",   AMDGPUOperand::ImmTyClampSI, true, nullptr},
4274   {"omod",    AMDGPUOperand::ImmTyOModSI, false, ConvertOmodMul},
4275   {"unorm",   AMDGPUOperand::ImmTyUNorm, true, nullptr},
4276   {"da",      AMDGPUOperand::ImmTyDA,    true, nullptr},
4277   {"r128",    AMDGPUOperand::ImmTyR128,  true, nullptr},
4278   {"lwe",     AMDGPUOperand::ImmTyLWE,   true, nullptr},
4279   {"dmask",   AMDGPUOperand::ImmTyDMask, false, nullptr},
4280   {"row_mask",   AMDGPUOperand::ImmTyDppRowMask, false, nullptr},
4281   {"bank_mask",  AMDGPUOperand::ImmTyDppBankMask, false, nullptr},
4282   {"bound_ctrl", AMDGPUOperand::ImmTyDppBoundCtrl, false, ConvertBoundCtrl},
4283   {"dst_sel",    AMDGPUOperand::ImmTySdwaDstSel, false, nullptr},
4284   {"src0_sel",   AMDGPUOperand::ImmTySdwaSrc0Sel, false, nullptr},
4285   {"src1_sel",   AMDGPUOperand::ImmTySdwaSrc1Sel, false, nullptr},
4286   {"dst_unused", AMDGPUOperand::ImmTySdwaDstUnused, false, nullptr},
4287   {"compr", AMDGPUOperand::ImmTyExpCompr, true, nullptr },
4288   {"vm", AMDGPUOperand::ImmTyExpVM, true, nullptr},
4289   {"op_sel", AMDGPUOperand::ImmTyOpSel, false, nullptr},
4290   {"op_sel_hi", AMDGPUOperand::ImmTyOpSelHi, false, nullptr},
4291   {"neg_lo", AMDGPUOperand::ImmTyNegLo, false, nullptr},
4292   {"neg_hi", AMDGPUOperand::ImmTyNegHi, false, nullptr}
4293 };
4294 
4295 OperandMatchResultTy AMDGPUAsmParser::parseOptionalOperand(OperandVector &Operands) {
4296   unsigned size = Operands.size();
4297   assert(size > 0);
4298 
4299   OperandMatchResultTy res = parseOptionalOpr(Operands);
4300 
4301   // This is a hack to enable hardcoded mandatory operands which follow
4302   // optional operands.
4303   //
4304   // Current design assumes that all operands after the first optional operand
4305   // are also optional. However implementation of some instructions violates
4306   // this rule (see e.g. flat/global atomic which have hardcoded 'glc' operands).
4307   //
4308   // To alleviate this problem, we have to (implicitly) parse extra operands
4309   // to make sure autogenerated parser of custom operands never hit hardcoded
4310   // mandatory operands.
4311 
4312   if (size == 1 || ((AMDGPUOperand &)*Operands[size - 1]).isRegKind()) {
4313 
4314     // We have parsed the first optional operand.
4315     // Parse as many operands as necessary to skip all mandatory operands.
4316 
4317     for (unsigned i = 0; i < MAX_OPR_LOOKAHEAD; ++i) {
4318       if (res != MatchOperand_Success ||
4319           getLexer().is(AsmToken::EndOfStatement)) break;
4320       if (getLexer().is(AsmToken::Comma)) Parser.Lex();
4321       res = parseOptionalOpr(Operands);
4322     }
4323   }
4324 
4325   return res;
4326 }
4327 
4328 OperandMatchResultTy AMDGPUAsmParser::parseOptionalOpr(OperandVector &Operands) {
4329   OperandMatchResultTy res;
4330   for (const OptionalOperand &Op : AMDGPUOptionalOperandTable) {
4331     // try to parse any optional operand here
4332     if (Op.IsBit) {
4333       res = parseNamedBit(Op.Name, Operands, Op.Type);
4334     } else if (Op.Type == AMDGPUOperand::ImmTyOModSI) {
4335       res = parseOModOperand(Operands);
4336     } else if (Op.Type == AMDGPUOperand::ImmTySdwaDstSel ||
4337                Op.Type == AMDGPUOperand::ImmTySdwaSrc0Sel ||
4338                Op.Type == AMDGPUOperand::ImmTySdwaSrc1Sel) {
4339       res = parseSDWASel(Operands, Op.Name, Op.Type);
4340     } else if (Op.Type == AMDGPUOperand::ImmTySdwaDstUnused) {
4341       res = parseSDWADstUnused(Operands);
4342     } else if (Op.Type == AMDGPUOperand::ImmTyOpSel ||
4343                Op.Type == AMDGPUOperand::ImmTyOpSelHi ||
4344                Op.Type == AMDGPUOperand::ImmTyNegLo ||
4345                Op.Type == AMDGPUOperand::ImmTyNegHi) {
4346       res = parseOperandArrayWithPrefix(Op.Name, Operands, Op.Type,
4347                                         Op.ConvertResult);
4348     } else {
4349       res = parseIntWithPrefix(Op.Name, Operands, Op.Type, Op.ConvertResult);
4350     }
4351     if (res != MatchOperand_NoMatch) {
4352       return res;
4353     }
4354   }
4355   return MatchOperand_NoMatch;
4356 }
4357 
4358 OperandMatchResultTy AMDGPUAsmParser::parseOModOperand(OperandVector &Operands) {
4359   StringRef Name = Parser.getTok().getString();
4360   if (Name == "mul") {
4361     return parseIntWithPrefix("mul", Operands,
4362                               AMDGPUOperand::ImmTyOModSI, ConvertOmodMul);
4363   }
4364 
4365   if (Name == "div") {
4366     return parseIntWithPrefix("div", Operands,
4367                               AMDGPUOperand::ImmTyOModSI, ConvertOmodDiv);
4368   }
4369 
4370   return MatchOperand_NoMatch;
4371 }
4372 
4373 void AMDGPUAsmParser::cvtVOP3OpSel(MCInst &Inst, const OperandVector &Operands) {
4374   cvtVOP3P(Inst, Operands);
4375 
4376   int Opc = Inst.getOpcode();
4377 
4378   int SrcNum;
4379   const int Ops[] = { AMDGPU::OpName::src0,
4380                       AMDGPU::OpName::src1,
4381                       AMDGPU::OpName::src2 };
4382   for (SrcNum = 0;
4383        SrcNum < 3 && AMDGPU::getNamedOperandIdx(Opc, Ops[SrcNum]) != -1;
4384        ++SrcNum);
4385   assert(SrcNum > 0);
4386 
4387   int OpSelIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::op_sel);
4388   unsigned OpSel = Inst.getOperand(OpSelIdx).getImm();
4389 
4390   if ((OpSel & (1 << SrcNum)) != 0) {
4391     int ModIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src0_modifiers);
4392     uint32_t ModVal = Inst.getOperand(ModIdx).getImm();
4393     Inst.getOperand(ModIdx).setImm(ModVal | SISrcMods::DST_OP_SEL);
4394   }
4395 }
4396 
4397 static bool isRegOrImmWithInputMods(const MCInstrDesc &Desc, unsigned OpNum) {
4398       // 1. This operand is input modifiers
4399   return Desc.OpInfo[OpNum].OperandType == AMDGPU::OPERAND_INPUT_MODS
4400       // 2. This is not last operand
4401       && Desc.NumOperands > (OpNum + 1)
4402       // 3. Next operand is register class
4403       && Desc.OpInfo[OpNum + 1].RegClass != -1
4404       // 4. Next register is not tied to any other operand
4405       && Desc.getOperandConstraint(OpNum + 1, MCOI::OperandConstraint::TIED_TO) == -1;
4406 }
4407 
4408 void AMDGPUAsmParser::cvtVOP3Interp(MCInst &Inst, const OperandVector &Operands)
4409 {
4410   OptionalImmIndexMap OptionalIdx;
4411   unsigned Opc = Inst.getOpcode();
4412 
4413   unsigned I = 1;
4414   const MCInstrDesc &Desc = MII.get(Inst.getOpcode());
4415   for (unsigned J = 0; J < Desc.getNumDefs(); ++J) {
4416     ((AMDGPUOperand &)*Operands[I++]).addRegOperands(Inst, 1);
4417   }
4418 
4419   for (unsigned E = Operands.size(); I != E; ++I) {
4420     AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[I]);
4421     if (isRegOrImmWithInputMods(Desc, Inst.getNumOperands())) {
4422       Op.addRegOrImmWithFPInputModsOperands(Inst, 2);
4423     } else if (Op.isInterpSlot() ||
4424                Op.isInterpAttr() ||
4425                Op.isAttrChan()) {
4426       Inst.addOperand(MCOperand::createImm(Op.Imm.Val));
4427     } else if (Op.isImmModifier()) {
4428       OptionalIdx[Op.getImmTy()] = I;
4429     } else {
4430       llvm_unreachable("unhandled operand type");
4431     }
4432   }
4433 
4434   if (AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::high) != -1) {
4435     addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyHigh);
4436   }
4437 
4438   if (AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::clamp) != -1) {
4439     addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyClampSI);
4440   }
4441 
4442   if (AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::omod) != -1) {
4443     addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyOModSI);
4444   }
4445 }
4446 
4447 void AMDGPUAsmParser::cvtVOP3(MCInst &Inst, const OperandVector &Operands,
4448                               OptionalImmIndexMap &OptionalIdx) {
4449   unsigned Opc = Inst.getOpcode();
4450 
4451   unsigned I = 1;
4452   const MCInstrDesc &Desc = MII.get(Inst.getOpcode());
4453   for (unsigned J = 0; J < Desc.getNumDefs(); ++J) {
4454     ((AMDGPUOperand &)*Operands[I++]).addRegOperands(Inst, 1);
4455   }
4456 
4457   if (AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src0_modifiers) != -1) {
4458     // This instruction has src modifiers
4459     for (unsigned E = Operands.size(); I != E; ++I) {
4460       AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[I]);
4461       if (isRegOrImmWithInputMods(Desc, Inst.getNumOperands())) {
4462         Op.addRegOrImmWithFPInputModsOperands(Inst, 2);
4463       } else if (Op.isImmModifier()) {
4464         OptionalIdx[Op.getImmTy()] = I;
4465       } else if (Op.isRegOrImm()) {
4466         Op.addRegOrImmOperands(Inst, 1);
4467       } else {
4468         llvm_unreachable("unhandled operand type");
4469       }
4470     }
4471   } else {
4472     // No src modifiers
4473     for (unsigned E = Operands.size(); I != E; ++I) {
4474       AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[I]);
4475       if (Op.isMod()) {
4476         OptionalIdx[Op.getImmTy()] = I;
4477       } else {
4478         Op.addRegOrImmOperands(Inst, 1);
4479       }
4480     }
4481   }
4482 
4483   if (AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::clamp) != -1) {
4484     addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyClampSI);
4485   }
4486 
4487   if (AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::omod) != -1) {
4488     addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyOModSI);
4489   }
4490 
4491   // special case v_mac_{f16, f32}:
4492   // it has src2 register operand that is tied to dst operand
4493   // we don't allow modifiers for this operand in assembler so src2_modifiers
4494   // should be 0
4495   if (Opc == AMDGPU::V_MAC_F32_e64_si || Opc == AMDGPU::V_MAC_F32_e64_vi ||
4496       Opc == AMDGPU::V_MAC_F16_e64_vi) {
4497     auto it = Inst.begin();
4498     std::advance(it, AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src2_modifiers));
4499     it = Inst.insert(it, MCOperand::createImm(0)); // no modifiers for src2
4500     ++it;
4501     Inst.insert(it, Inst.getOperand(0)); // src2 = dst
4502   }
4503 }
4504 
4505 void AMDGPUAsmParser::cvtVOP3(MCInst &Inst, const OperandVector &Operands) {
4506   OptionalImmIndexMap OptionalIdx;
4507   cvtVOP3(Inst, Operands, OptionalIdx);
4508 }
4509 
4510 void AMDGPUAsmParser::cvtVOP3P(MCInst &Inst,
4511                                const OperandVector &Operands) {
4512   OptionalImmIndexMap OptIdx;
4513   const int Opc = Inst.getOpcode();
4514   const MCInstrDesc &Desc = MII.get(Opc);
4515 
4516   const bool IsPacked = (Desc.TSFlags & SIInstrFlags::IsPacked) != 0;
4517 
4518   cvtVOP3(Inst, Operands, OptIdx);
4519 
4520   if (AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::vdst_in) != -1) {
4521     assert(!IsPacked);
4522     Inst.addOperand(Inst.getOperand(0));
4523   }
4524 
4525   // FIXME: This is messy. Parse the modifiers as if it was a normal VOP3
4526   // instruction, and then figure out where to actually put the modifiers
4527 
4528   addOptionalImmOperand(Inst, Operands, OptIdx, AMDGPUOperand::ImmTyOpSel);
4529 
4530   int OpSelHiIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::op_sel_hi);
4531   if (OpSelHiIdx != -1) {
4532     int DefaultVal = IsPacked ? -1 : 0;
4533     addOptionalImmOperand(Inst, Operands, OptIdx, AMDGPUOperand::ImmTyOpSelHi,
4534                           DefaultVal);
4535   }
4536 
4537   int NegLoIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::neg_lo);
4538   if (NegLoIdx != -1) {
4539     assert(IsPacked);
4540     addOptionalImmOperand(Inst, Operands, OptIdx, AMDGPUOperand::ImmTyNegLo);
4541     addOptionalImmOperand(Inst, Operands, OptIdx, AMDGPUOperand::ImmTyNegHi);
4542   }
4543 
4544   const int Ops[] = { AMDGPU::OpName::src0,
4545                       AMDGPU::OpName::src1,
4546                       AMDGPU::OpName::src2 };
4547   const int ModOps[] = { AMDGPU::OpName::src0_modifiers,
4548                          AMDGPU::OpName::src1_modifiers,
4549                          AMDGPU::OpName::src2_modifiers };
4550 
4551   int OpSelIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::op_sel);
4552 
4553   unsigned OpSel = Inst.getOperand(OpSelIdx).getImm();
4554   unsigned OpSelHi = 0;
4555   unsigned NegLo = 0;
4556   unsigned NegHi = 0;
4557 
4558   if (OpSelHiIdx != -1) {
4559     OpSelHi = Inst.getOperand(OpSelHiIdx).getImm();
4560   }
4561 
4562   if (NegLoIdx != -1) {
4563     int NegHiIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::neg_hi);
4564     NegLo = Inst.getOperand(NegLoIdx).getImm();
4565     NegHi = Inst.getOperand(NegHiIdx).getImm();
4566   }
4567 
4568   for (int J = 0; J < 3; ++J) {
4569     int OpIdx = AMDGPU::getNamedOperandIdx(Opc, Ops[J]);
4570     if (OpIdx == -1)
4571       break;
4572 
4573     uint32_t ModVal = 0;
4574 
4575     if ((OpSel & (1 << J)) != 0)
4576       ModVal |= SISrcMods::OP_SEL_0;
4577 
4578     if ((OpSelHi & (1 << J)) != 0)
4579       ModVal |= SISrcMods::OP_SEL_1;
4580 
4581     if ((NegLo & (1 << J)) != 0)
4582       ModVal |= SISrcMods::NEG;
4583 
4584     if ((NegHi & (1 << J)) != 0)
4585       ModVal |= SISrcMods::NEG_HI;
4586 
4587     int ModIdx = AMDGPU::getNamedOperandIdx(Opc, ModOps[J]);
4588 
4589     Inst.getOperand(ModIdx).setImm(Inst.getOperand(ModIdx).getImm() | ModVal);
4590   }
4591 }
4592 
4593 //===----------------------------------------------------------------------===//
4594 // dpp
4595 //===----------------------------------------------------------------------===//
4596 
4597 bool AMDGPUOperand::isDPPCtrl() const {
4598   bool result = isImm() && getImmTy() == ImmTyDppCtrl && isUInt<9>(getImm());
4599   if (result) {
4600     int64_t Imm = getImm();
4601     return ((Imm >= 0x000) && (Imm <= 0x0ff)) ||
4602            ((Imm >= 0x101) && (Imm <= 0x10f)) ||
4603            ((Imm >= 0x111) && (Imm <= 0x11f)) ||
4604            ((Imm >= 0x121) && (Imm <= 0x12f)) ||
4605            (Imm == 0x130) ||
4606            (Imm == 0x134) ||
4607            (Imm == 0x138) ||
4608            (Imm == 0x13c) ||
4609            (Imm == 0x140) ||
4610            (Imm == 0x141) ||
4611            (Imm == 0x142) ||
4612            (Imm == 0x143);
4613   }
4614   return false;
4615 }
4616 
4617 bool AMDGPUOperand::isGPRIdxMode() const {
4618   return isImm() && isUInt<4>(getImm());
4619 }
4620 
4621 bool AMDGPUOperand::isS16Imm() const {
4622   return isImm() && (isInt<16>(getImm()) || isUInt<16>(getImm()));
4623 }
4624 
4625 bool AMDGPUOperand::isU16Imm() const {
4626   return isImm() && isUInt<16>(getImm());
4627 }
4628 
4629 OperandMatchResultTy
4630 AMDGPUAsmParser::parseDPPCtrl(OperandVector &Operands) {
4631   SMLoc S = Parser.getTok().getLoc();
4632   StringRef Prefix;
4633   int64_t Int;
4634 
4635   if (getLexer().getKind() == AsmToken::Identifier) {
4636     Prefix = Parser.getTok().getString();
4637   } else {
4638     return MatchOperand_NoMatch;
4639   }
4640 
4641   if (Prefix == "row_mirror") {
4642     Int = 0x140;
4643     Parser.Lex();
4644   } else if (Prefix == "row_half_mirror") {
4645     Int = 0x141;
4646     Parser.Lex();
4647   } else {
4648     // Check to prevent parseDPPCtrlOps from eating invalid tokens
4649     if (Prefix != "quad_perm"
4650         && Prefix != "row_shl"
4651         && Prefix != "row_shr"
4652         && Prefix != "row_ror"
4653         && Prefix != "wave_shl"
4654         && Prefix != "wave_rol"
4655         && Prefix != "wave_shr"
4656         && Prefix != "wave_ror"
4657         && Prefix != "row_bcast") {
4658       return MatchOperand_NoMatch;
4659     }
4660 
4661     Parser.Lex();
4662     if (getLexer().isNot(AsmToken::Colon))
4663       return MatchOperand_ParseFail;
4664 
4665     if (Prefix == "quad_perm") {
4666       // quad_perm:[%d,%d,%d,%d]
4667       Parser.Lex();
4668       if (getLexer().isNot(AsmToken::LBrac))
4669         return MatchOperand_ParseFail;
4670       Parser.Lex();
4671 
4672       if (getParser().parseAbsoluteExpression(Int) || !(0 <= Int && Int <=3))
4673         return MatchOperand_ParseFail;
4674 
4675       for (int i = 0; i < 3; ++i) {
4676         if (getLexer().isNot(AsmToken::Comma))
4677           return MatchOperand_ParseFail;
4678         Parser.Lex();
4679 
4680         int64_t Temp;
4681         if (getParser().parseAbsoluteExpression(Temp) || !(0 <= Temp && Temp <=3))
4682           return MatchOperand_ParseFail;
4683         const int shift = i*2 + 2;
4684         Int += (Temp << shift);
4685       }
4686 
4687       if (getLexer().isNot(AsmToken::RBrac))
4688         return MatchOperand_ParseFail;
4689       Parser.Lex();
4690     } else {
4691       // sel:%d
4692       Parser.Lex();
4693       if (getParser().parseAbsoluteExpression(Int))
4694         return MatchOperand_ParseFail;
4695 
4696       if (Prefix == "row_shl" && 1 <= Int && Int <= 15) {
4697         Int |= 0x100;
4698       } else if (Prefix == "row_shr" && 1 <= Int && Int <= 15) {
4699         Int |= 0x110;
4700       } else if (Prefix == "row_ror" && 1 <= Int && Int <= 15) {
4701         Int |= 0x120;
4702       } else if (Prefix == "wave_shl" && 1 == Int) {
4703         Int = 0x130;
4704       } else if (Prefix == "wave_rol" && 1 == Int) {
4705         Int = 0x134;
4706       } else if (Prefix == "wave_shr" && 1 == Int) {
4707         Int = 0x138;
4708       } else if (Prefix == "wave_ror" && 1 == Int) {
4709         Int = 0x13C;
4710       } else if (Prefix == "row_bcast") {
4711         if (Int == 15) {
4712           Int = 0x142;
4713         } else if (Int == 31) {
4714           Int = 0x143;
4715         } else {
4716           return MatchOperand_ParseFail;
4717         }
4718       } else {
4719         return MatchOperand_ParseFail;
4720       }
4721     }
4722   }
4723 
4724   Operands.push_back(AMDGPUOperand::CreateImm(this, Int, S, AMDGPUOperand::ImmTyDppCtrl));
4725   return MatchOperand_Success;
4726 }
4727 
4728 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultRowMask() const {
4729   return AMDGPUOperand::CreateImm(this, 0xf, SMLoc(), AMDGPUOperand::ImmTyDppRowMask);
4730 }
4731 
4732 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultBankMask() const {
4733   return AMDGPUOperand::CreateImm(this, 0xf, SMLoc(), AMDGPUOperand::ImmTyDppBankMask);
4734 }
4735 
4736 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultBoundCtrl() const {
4737   return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTyDppBoundCtrl);
4738 }
4739 
4740 void AMDGPUAsmParser::cvtDPP(MCInst &Inst, const OperandVector &Operands) {
4741   OptionalImmIndexMap OptionalIdx;
4742 
4743   unsigned I = 1;
4744   const MCInstrDesc &Desc = MII.get(Inst.getOpcode());
4745   for (unsigned J = 0; J < Desc.getNumDefs(); ++J) {
4746     ((AMDGPUOperand &)*Operands[I++]).addRegOperands(Inst, 1);
4747   }
4748 
4749   // All DPP instructions with at least one source operand have a fake "old"
4750   // source at the beginning that's tied to the dst operand. Handle it here.
4751   if (Desc.getNumOperands() >= 2)
4752     Inst.addOperand(Inst.getOperand(0));
4753 
4754   for (unsigned E = Operands.size(); I != E; ++I) {
4755     AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[I]);
4756     // Add the register arguments
4757     if (Op.isReg() && Op.Reg.RegNo == AMDGPU::VCC) {
4758       // VOP2b (v_add_u32, v_sub_u32 ...) dpp use "vcc" token.
4759       // Skip it.
4760       continue;
4761     } if (isRegOrImmWithInputMods(Desc, Inst.getNumOperands())) {
4762       Op.addRegWithFPInputModsOperands(Inst, 2);
4763     } else if (Op.isDPPCtrl()) {
4764       Op.addImmOperands(Inst, 1);
4765     } else if (Op.isImm()) {
4766       // Handle optional arguments
4767       OptionalIdx[Op.getImmTy()] = I;
4768     } else {
4769       llvm_unreachable("Invalid operand type");
4770     }
4771   }
4772 
4773   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyDppRowMask, 0xf);
4774   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyDppBankMask, 0xf);
4775   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyDppBoundCtrl);
4776 }
4777 
4778 //===----------------------------------------------------------------------===//
4779 // sdwa
4780 //===----------------------------------------------------------------------===//
4781 
4782 OperandMatchResultTy
4783 AMDGPUAsmParser::parseSDWASel(OperandVector &Operands, StringRef Prefix,
4784                               AMDGPUOperand::ImmTy Type) {
4785   using namespace llvm::AMDGPU::SDWA;
4786 
4787   SMLoc S = Parser.getTok().getLoc();
4788   StringRef Value;
4789   OperandMatchResultTy res;
4790 
4791   res = parseStringWithPrefix(Prefix, Value);
4792   if (res != MatchOperand_Success) {
4793     return res;
4794   }
4795 
4796   int64_t Int;
4797   Int = StringSwitch<int64_t>(Value)
4798         .Case("BYTE_0", SdwaSel::BYTE_0)
4799         .Case("BYTE_1", SdwaSel::BYTE_1)
4800         .Case("BYTE_2", SdwaSel::BYTE_2)
4801         .Case("BYTE_3", SdwaSel::BYTE_3)
4802         .Case("WORD_0", SdwaSel::WORD_0)
4803         .Case("WORD_1", SdwaSel::WORD_1)
4804         .Case("DWORD", SdwaSel::DWORD)
4805         .Default(0xffffffff);
4806   Parser.Lex(); // eat last token
4807 
4808   if (Int == 0xffffffff) {
4809     return MatchOperand_ParseFail;
4810   }
4811 
4812   Operands.push_back(AMDGPUOperand::CreateImm(this, Int, S, Type));
4813   return MatchOperand_Success;
4814 }
4815 
4816 OperandMatchResultTy
4817 AMDGPUAsmParser::parseSDWADstUnused(OperandVector &Operands) {
4818   using namespace llvm::AMDGPU::SDWA;
4819 
4820   SMLoc S = Parser.getTok().getLoc();
4821   StringRef Value;
4822   OperandMatchResultTy res;
4823 
4824   res = parseStringWithPrefix("dst_unused", Value);
4825   if (res != MatchOperand_Success) {
4826     return res;
4827   }
4828 
4829   int64_t Int;
4830   Int = StringSwitch<int64_t>(Value)
4831         .Case("UNUSED_PAD", DstUnused::UNUSED_PAD)
4832         .Case("UNUSED_SEXT", DstUnused::UNUSED_SEXT)
4833         .Case("UNUSED_PRESERVE", DstUnused::UNUSED_PRESERVE)
4834         .Default(0xffffffff);
4835   Parser.Lex(); // eat last token
4836 
4837   if (Int == 0xffffffff) {
4838     return MatchOperand_ParseFail;
4839   }
4840 
4841   Operands.push_back(AMDGPUOperand::CreateImm(this, Int, S, AMDGPUOperand::ImmTySdwaDstUnused));
4842   return MatchOperand_Success;
4843 }
4844 
4845 void AMDGPUAsmParser::cvtSdwaVOP1(MCInst &Inst, const OperandVector &Operands) {
4846   cvtSDWA(Inst, Operands, SIInstrFlags::VOP1);
4847 }
4848 
4849 void AMDGPUAsmParser::cvtSdwaVOP2(MCInst &Inst, const OperandVector &Operands) {
4850   cvtSDWA(Inst, Operands, SIInstrFlags::VOP2);
4851 }
4852 
4853 void AMDGPUAsmParser::cvtSdwaVOP2b(MCInst &Inst, const OperandVector &Operands) {
4854   cvtSDWA(Inst, Operands, SIInstrFlags::VOP2, true);
4855 }
4856 
4857 void AMDGPUAsmParser::cvtSdwaVOPC(MCInst &Inst, const OperandVector &Operands) {
4858   cvtSDWA(Inst, Operands, SIInstrFlags::VOPC, isVI());
4859 }
4860 
4861 void AMDGPUAsmParser::cvtSDWA(MCInst &Inst, const OperandVector &Operands,
4862                               uint64_t BasicInstType, bool skipVcc) {
4863   using namespace llvm::AMDGPU::SDWA;
4864 
4865   OptionalImmIndexMap OptionalIdx;
4866   bool skippedVcc = false;
4867 
4868   unsigned I = 1;
4869   const MCInstrDesc &Desc = MII.get(Inst.getOpcode());
4870   for (unsigned J = 0; J < Desc.getNumDefs(); ++J) {
4871     ((AMDGPUOperand &)*Operands[I++]).addRegOperands(Inst, 1);
4872   }
4873 
4874   for (unsigned E = Operands.size(); I != E; ++I) {
4875     AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[I]);
4876     if (skipVcc && !skippedVcc && Op.isReg() && Op.Reg.RegNo == AMDGPU::VCC) {
4877       // VOP2b (v_add_u32, v_sub_u32 ...) sdwa use "vcc" token as dst.
4878       // Skip it if it's 2nd (e.g. v_add_i32_sdwa v1, vcc, v2, v3)
4879       // or 4th (v_addc_u32_sdwa v1, vcc, v2, v3, vcc) operand.
4880       // Skip VCC only if we didn't skip it on previous iteration.
4881       if (BasicInstType == SIInstrFlags::VOP2 &&
4882           (Inst.getNumOperands() == 1 || Inst.getNumOperands() == 5)) {
4883         skippedVcc = true;
4884         continue;
4885       } else if (BasicInstType == SIInstrFlags::VOPC &&
4886                  Inst.getNumOperands() == 0) {
4887         skippedVcc = true;
4888         continue;
4889       }
4890     }
4891     if (isRegOrImmWithInputMods(Desc, Inst.getNumOperands())) {
4892       Op.addRegOrImmWithInputModsOperands(Inst, 2);
4893     } else if (Op.isImm()) {
4894       // Handle optional arguments
4895       OptionalIdx[Op.getImmTy()] = I;
4896     } else {
4897       llvm_unreachable("Invalid operand type");
4898     }
4899     skippedVcc = false;
4900   }
4901 
4902   if (Inst.getOpcode() != AMDGPU::V_NOP_sdwa_gfx9 &&
4903       Inst.getOpcode() != AMDGPU::V_NOP_sdwa_vi) {
4904     // v_nop_sdwa_sdwa_vi/gfx9 has no optional sdwa arguments
4905     switch (BasicInstType) {
4906     case SIInstrFlags::VOP1:
4907       addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyClampSI, 0);
4908       if (AMDGPU::getNamedOperandIdx(Inst.getOpcode(), AMDGPU::OpName::omod) != -1) {
4909         addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyOModSI, 0);
4910       }
4911       addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySdwaDstSel, SdwaSel::DWORD);
4912       addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySdwaDstUnused, DstUnused::UNUSED_PRESERVE);
4913       addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySdwaSrc0Sel, SdwaSel::DWORD);
4914       break;
4915 
4916     case SIInstrFlags::VOP2:
4917       addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyClampSI, 0);
4918       if (AMDGPU::getNamedOperandIdx(Inst.getOpcode(), AMDGPU::OpName::omod) != -1) {
4919         addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyOModSI, 0);
4920       }
4921       addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySdwaDstSel, SdwaSel::DWORD);
4922       addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySdwaDstUnused, DstUnused::UNUSED_PRESERVE);
4923       addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySdwaSrc0Sel, SdwaSel::DWORD);
4924       addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySdwaSrc1Sel, SdwaSel::DWORD);
4925       break;
4926 
4927     case SIInstrFlags::VOPC:
4928       addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyClampSI, 0);
4929       addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySdwaSrc0Sel, SdwaSel::DWORD);
4930       addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySdwaSrc1Sel, SdwaSel::DWORD);
4931       break;
4932 
4933     default:
4934       llvm_unreachable("Invalid instruction type. Only VOP1, VOP2 and VOPC allowed");
4935     }
4936   }
4937 
4938   // special case v_mac_{f16, f32}:
4939   // it has src2 register operand that is tied to dst operand
4940   if (Inst.getOpcode() == AMDGPU::V_MAC_F32_sdwa_vi ||
4941       Inst.getOpcode() == AMDGPU::V_MAC_F16_sdwa_vi)  {
4942     auto it = Inst.begin();
4943     std::advance(
4944       it, AMDGPU::getNamedOperandIdx(Inst.getOpcode(), AMDGPU::OpName::src2));
4945     Inst.insert(it, Inst.getOperand(0)); // src2 = dst
4946   }
4947 }
4948 
4949 /// Force static initialization.
4950 extern "C" void LLVMInitializeAMDGPUAsmParser() {
4951   RegisterMCAsmParser<AMDGPUAsmParser> A(getTheAMDGPUTarget());
4952   RegisterMCAsmParser<AMDGPUAsmParser> B(getTheGCNTarget());
4953 }
4954 
4955 #define GET_REGISTER_MATCHER
4956 #define GET_MATCHER_IMPLEMENTATION
4957 #define GET_MNEMONIC_SPELL_CHECKER
4958 #include "AMDGPUGenAsmMatcher.inc"
4959 
4960 // This fuction should be defined after auto-generated include so that we have
4961 // MatchClassKind enum defined
4962 unsigned AMDGPUAsmParser::validateTargetOperandClass(MCParsedAsmOperand &Op,
4963                                                      unsigned Kind) {
4964   // Tokens like "glc" would be parsed as immediate operands in ParseOperand().
4965   // But MatchInstructionImpl() expects to meet token and fails to validate
4966   // operand. This method checks if we are given immediate operand but expect to
4967   // get corresponding token.
4968   AMDGPUOperand &Operand = (AMDGPUOperand&)Op;
4969   switch (Kind) {
4970   case MCK_addr64:
4971     return Operand.isAddr64() ? Match_Success : Match_InvalidOperand;
4972   case MCK_gds:
4973     return Operand.isGDS() ? Match_Success : Match_InvalidOperand;
4974   case MCK_glc:
4975     return Operand.isGLC() ? Match_Success : Match_InvalidOperand;
4976   case MCK_d16:
4977     return Operand.isD16() ? Match_Success : Match_InvalidOperand;
4978   case MCK_idxen:
4979     return Operand.isIdxen() ? Match_Success : Match_InvalidOperand;
4980   case MCK_offen:
4981     return Operand.isOffen() ? Match_Success : Match_InvalidOperand;
4982   case MCK_SSrcB32:
4983     // When operands have expression values, they will return true for isToken,
4984     // because it is not possible to distinguish between a token and an
4985     // expression at parse time. MatchInstructionImpl() will always try to
4986     // match an operand as a token, when isToken returns true, and when the
4987     // name of the expression is not a valid token, the match will fail,
4988     // so we need to handle it here.
4989     return Operand.isSSrcB32() ? Match_Success : Match_InvalidOperand;
4990   case MCK_SSrcF32:
4991     return Operand.isSSrcF32() ? Match_Success : Match_InvalidOperand;
4992   case MCK_SoppBrTarget:
4993     return Operand.isSoppBrTarget() ? Match_Success : Match_InvalidOperand;
4994   case MCK_VReg32OrOff:
4995     return Operand.isVReg32OrOff() ? Match_Success : Match_InvalidOperand;
4996   case MCK_InterpSlot:
4997     return Operand.isInterpSlot() ? Match_Success : Match_InvalidOperand;
4998   case MCK_Attr:
4999     return Operand.isInterpAttr() ? Match_Success : Match_InvalidOperand;
5000   case MCK_AttrChan:
5001     return Operand.isAttrChan() ? Match_Success : Match_InvalidOperand;
5002   default:
5003     return Match_InvalidOperand;
5004   }
5005 }
5006