1 //===- AMDGPUAsmParser.cpp - Parse SI asm to MCInst instructions ----------===//
2 //
3 //                     The LLVM Compiler Infrastructure
4 //
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
7 //
8 //===----------------------------------------------------------------------===//
9 
10 #include "AMDGPU.h"
11 #include "AMDKernelCodeT.h"
12 #include "MCTargetDesc/AMDGPUMCTargetDesc.h"
13 #include "MCTargetDesc/AMDGPUTargetStreamer.h"
14 #include "SIDefines.h"
15 #include "Utils/AMDGPUAsmUtils.h"
16 #include "Utils/AMDGPUBaseInfo.h"
17 #include "Utils/AMDKernelCodeTUtils.h"
18 #include "llvm/ADT/APFloat.h"
19 #include "llvm/ADT/APInt.h"
20 #include "llvm/ADT/ArrayRef.h"
21 #include "llvm/ADT/STLExtras.h"
22 #include "llvm/ADT/SmallBitVector.h"
23 #include "llvm/ADT/SmallString.h"
24 #include "llvm/ADT/StringRef.h"
25 #include "llvm/ADT/StringSwitch.h"
26 #include "llvm/ADT/Twine.h"
27 #include "llvm/BinaryFormat/ELF.h"
28 #include "llvm/CodeGen/MachineValueType.h"
29 #include "llvm/MC/MCAsmInfo.h"
30 #include "llvm/MC/MCContext.h"
31 #include "llvm/MC/MCExpr.h"
32 #include "llvm/MC/MCInst.h"
33 #include "llvm/MC/MCInstrDesc.h"
34 #include "llvm/MC/MCInstrInfo.h"
35 #include "llvm/MC/MCParser/MCAsmLexer.h"
36 #include "llvm/MC/MCParser/MCAsmParser.h"
37 #include "llvm/MC/MCParser/MCAsmParserExtension.h"
38 #include "llvm/MC/MCParser/MCParsedAsmOperand.h"
39 #include "llvm/MC/MCParser/MCTargetAsmParser.h"
40 #include "llvm/MC/MCRegisterInfo.h"
41 #include "llvm/MC/MCStreamer.h"
42 #include "llvm/MC/MCSubtargetInfo.h"
43 #include "llvm/MC/MCSymbol.h"
44 #include "llvm/Support/AMDGPUMetadata.h"
45 #include "llvm/Support/Casting.h"
46 #include "llvm/Support/Compiler.h"
47 #include "llvm/Support/ErrorHandling.h"
48 #include "llvm/Support/MathExtras.h"
49 #include "llvm/Support/SMLoc.h"
50 #include "llvm/Support/TargetRegistry.h"
51 #include "llvm/Support/raw_ostream.h"
52 #include <algorithm>
53 #include <cassert>
54 #include <cstdint>
55 #include <cstring>
56 #include <iterator>
57 #include <map>
58 #include <memory>
59 #include <string>
60 
61 using namespace llvm;
62 using namespace llvm::AMDGPU;
63 
64 namespace {
65 
66 class AMDGPUAsmParser;
67 
68 enum RegisterKind { IS_UNKNOWN, IS_VGPR, IS_SGPR, IS_TTMP, IS_SPECIAL };
69 
70 //===----------------------------------------------------------------------===//
71 // Operand
72 //===----------------------------------------------------------------------===//
73 
74 class AMDGPUOperand : public MCParsedAsmOperand {
75   enum KindTy {
76     Token,
77     Immediate,
78     Register,
79     Expression
80   } Kind;
81 
82   SMLoc StartLoc, EndLoc;
83   const AMDGPUAsmParser *AsmParser;
84 
85 public:
86   AMDGPUOperand(KindTy Kind_, const AMDGPUAsmParser *AsmParser_)
87     : MCParsedAsmOperand(), Kind(Kind_), AsmParser(AsmParser_) {}
88 
89   using Ptr = std::unique_ptr<AMDGPUOperand>;
90 
91   struct Modifiers {
92     bool Abs = false;
93     bool Neg = false;
94     bool Sext = false;
95 
96     bool hasFPModifiers() const { return Abs || Neg; }
97     bool hasIntModifiers() const { return Sext; }
98     bool hasModifiers() const { return hasFPModifiers() || hasIntModifiers(); }
99 
100     int64_t getFPModifiersOperand() const {
101       int64_t Operand = 0;
102       Operand |= Abs ? SISrcMods::ABS : 0;
103       Operand |= Neg ? SISrcMods::NEG : 0;
104       return Operand;
105     }
106 
107     int64_t getIntModifiersOperand() const {
108       int64_t Operand = 0;
109       Operand |= Sext ? SISrcMods::SEXT : 0;
110       return Operand;
111     }
112 
113     int64_t getModifiersOperand() const {
114       assert(!(hasFPModifiers() && hasIntModifiers())
115            && "fp and int modifiers should not be used simultaneously");
116       if (hasFPModifiers()) {
117         return getFPModifiersOperand();
118       } else if (hasIntModifiers()) {
119         return getIntModifiersOperand();
120       } else {
121         return 0;
122       }
123     }
124 
125     friend raw_ostream &operator <<(raw_ostream &OS, AMDGPUOperand::Modifiers Mods);
126   };
127 
128   enum ImmTy {
129     ImmTyNone,
130     ImmTyGDS,
131     ImmTyOffen,
132     ImmTyIdxen,
133     ImmTyAddr64,
134     ImmTyOffset,
135     ImmTyInstOffset,
136     ImmTyOffset0,
137     ImmTyOffset1,
138     ImmTyGLC,
139     ImmTySLC,
140     ImmTyTFE,
141     ImmTyClampSI,
142     ImmTyOModSI,
143     ImmTyDppCtrl,
144     ImmTyDppRowMask,
145     ImmTyDppBankMask,
146     ImmTyDppBoundCtrl,
147     ImmTySdwaDstSel,
148     ImmTySdwaSrc0Sel,
149     ImmTySdwaSrc1Sel,
150     ImmTySdwaDstUnused,
151     ImmTyDMask,
152     ImmTyUNorm,
153     ImmTyDA,
154     ImmTyR128,
155     ImmTyLWE,
156     ImmTyExpTgt,
157     ImmTyExpCompr,
158     ImmTyExpVM,
159     ImmTyDFMT,
160     ImmTyNFMT,
161     ImmTyHwreg,
162     ImmTyOff,
163     ImmTySendMsg,
164     ImmTyInterpSlot,
165     ImmTyInterpAttr,
166     ImmTyAttrChan,
167     ImmTyOpSel,
168     ImmTyOpSelHi,
169     ImmTyNegLo,
170     ImmTyNegHi,
171     ImmTySwizzle,
172     ImmTyHigh
173   };
174 
175   struct TokOp {
176     const char *Data;
177     unsigned Length;
178   };
179 
180   struct ImmOp {
181     int64_t Val;
182     ImmTy Type;
183     bool IsFPImm;
184     Modifiers Mods;
185   };
186 
187   struct RegOp {
188     unsigned RegNo;
189     bool IsForcedVOP3;
190     Modifiers Mods;
191   };
192 
193   union {
194     TokOp Tok;
195     ImmOp Imm;
196     RegOp Reg;
197     const MCExpr *Expr;
198   };
199 
200   bool isToken() const override {
201     if (Kind == Token)
202       return true;
203 
204     if (Kind != Expression || !Expr)
205       return false;
206 
207     // When parsing operands, we can't always tell if something was meant to be
208     // a token, like 'gds', or an expression that references a global variable.
209     // In this case, we assume the string is an expression, and if we need to
210     // interpret is a token, then we treat the symbol name as the token.
211     return isa<MCSymbolRefExpr>(Expr);
212   }
213 
214   bool isImm() const override {
215     return Kind == Immediate;
216   }
217 
218   bool isInlinableImm(MVT type) const;
219   bool isLiteralImm(MVT type) const;
220 
221   bool isRegKind() const {
222     return Kind == Register;
223   }
224 
225   bool isReg() const override {
226     return isRegKind() && !hasModifiers();
227   }
228 
229   bool isRegOrImmWithInputMods(MVT type) const {
230     return isRegKind() || isInlinableImm(type);
231   }
232 
233   bool isRegOrImmWithInt16InputMods() const {
234     return isRegOrImmWithInputMods(MVT::i16);
235   }
236 
237   bool isRegOrImmWithInt32InputMods() const {
238     return isRegOrImmWithInputMods(MVT::i32);
239   }
240 
241   bool isRegOrImmWithInt64InputMods() const {
242     return isRegOrImmWithInputMods(MVT::i64);
243   }
244 
245   bool isRegOrImmWithFP16InputMods() const {
246     return isRegOrImmWithInputMods(MVT::f16);
247   }
248 
249   bool isRegOrImmWithFP32InputMods() const {
250     return isRegOrImmWithInputMods(MVT::f32);
251   }
252 
253   bool isRegOrImmWithFP64InputMods() const {
254     return isRegOrImmWithInputMods(MVT::f64);
255   }
256 
257   bool isVReg() const {
258     return isRegClass(AMDGPU::VGPR_32RegClassID) ||
259            isRegClass(AMDGPU::VReg_64RegClassID) ||
260            isRegClass(AMDGPU::VReg_96RegClassID) ||
261            isRegClass(AMDGPU::VReg_128RegClassID) ||
262            isRegClass(AMDGPU::VReg_256RegClassID) ||
263            isRegClass(AMDGPU::VReg_512RegClassID);
264   }
265 
266   bool isVReg32OrOff() const {
267     return isOff() || isRegClass(AMDGPU::VGPR_32RegClassID);
268   }
269 
270   bool isSDWAOperand(MVT type) const;
271   bool isSDWAFP16Operand() const;
272   bool isSDWAFP32Operand() const;
273   bool isSDWAInt16Operand() const;
274   bool isSDWAInt32Operand() const;
275 
276   bool isImmTy(ImmTy ImmT) const {
277     return isImm() && Imm.Type == ImmT;
278   }
279 
280   bool isImmModifier() const {
281     return isImm() && Imm.Type != ImmTyNone;
282   }
283 
284   bool isClampSI() const { return isImmTy(ImmTyClampSI); }
285   bool isOModSI() const { return isImmTy(ImmTyOModSI); }
286   bool isDMask() const { return isImmTy(ImmTyDMask); }
287   bool isUNorm() const { return isImmTy(ImmTyUNorm); }
288   bool isDA() const { return isImmTy(ImmTyDA); }
289   bool isR128() const { return isImmTy(ImmTyUNorm); }
290   bool isLWE() const { return isImmTy(ImmTyLWE); }
291   bool isOff() const { return isImmTy(ImmTyOff); }
292   bool isExpTgt() const { return isImmTy(ImmTyExpTgt); }
293   bool isExpVM() const { return isImmTy(ImmTyExpVM); }
294   bool isExpCompr() const { return isImmTy(ImmTyExpCompr); }
295   bool isOffen() const { return isImmTy(ImmTyOffen); }
296   bool isIdxen() const { return isImmTy(ImmTyIdxen); }
297   bool isAddr64() const { return isImmTy(ImmTyAddr64); }
298   bool isOffset() const { return isImmTy(ImmTyOffset) && isUInt<16>(getImm()); }
299   bool isOffset0() const { return isImmTy(ImmTyOffset0) && isUInt<16>(getImm()); }
300   bool isOffset1() const { return isImmTy(ImmTyOffset1) && isUInt<8>(getImm()); }
301 
302   bool isOffsetU12() const { return (isImmTy(ImmTyOffset) || isImmTy(ImmTyInstOffset)) && isUInt<12>(getImm()); }
303   bool isOffsetS13() const { return (isImmTy(ImmTyOffset) || isImmTy(ImmTyInstOffset)) && isInt<13>(getImm()); }
304   bool isGDS() const { return isImmTy(ImmTyGDS); }
305   bool isGLC() const { return isImmTy(ImmTyGLC); }
306   bool isSLC() const { return isImmTy(ImmTySLC); }
307   bool isTFE() const { return isImmTy(ImmTyTFE); }
308   bool isDFMT() const { return isImmTy(ImmTyDFMT) && isUInt<8>(getImm()); }
309   bool isNFMT() const { return isImmTy(ImmTyNFMT) && isUInt<8>(getImm()); }
310   bool isBankMask() const { return isImmTy(ImmTyDppBankMask); }
311   bool isRowMask() const { return isImmTy(ImmTyDppRowMask); }
312   bool isBoundCtrl() const { return isImmTy(ImmTyDppBoundCtrl); }
313   bool isSDWADstSel() const { return isImmTy(ImmTySdwaDstSel); }
314   bool isSDWASrc0Sel() const { return isImmTy(ImmTySdwaSrc0Sel); }
315   bool isSDWASrc1Sel() const { return isImmTy(ImmTySdwaSrc1Sel); }
316   bool isSDWADstUnused() const { return isImmTy(ImmTySdwaDstUnused); }
317   bool isInterpSlot() const { return isImmTy(ImmTyInterpSlot); }
318   bool isInterpAttr() const { return isImmTy(ImmTyInterpAttr); }
319   bool isAttrChan() const { return isImmTy(ImmTyAttrChan); }
320   bool isOpSel() const { return isImmTy(ImmTyOpSel); }
321   bool isOpSelHi() const { return isImmTy(ImmTyOpSelHi); }
322   bool isNegLo() const { return isImmTy(ImmTyNegLo); }
323   bool isNegHi() const { return isImmTy(ImmTyNegHi); }
324   bool isHigh() const { return isImmTy(ImmTyHigh); }
325 
326   bool isMod() const {
327     return isClampSI() || isOModSI();
328   }
329 
330   bool isRegOrImm() const {
331     return isReg() || isImm();
332   }
333 
334   bool isRegClass(unsigned RCID) const;
335 
336   bool isRegOrInlineNoMods(unsigned RCID, MVT type) const {
337     return (isRegClass(RCID) || isInlinableImm(type)) && !hasModifiers();
338   }
339 
340   bool isSCSrcB16() const {
341     return isRegOrInlineNoMods(AMDGPU::SReg_32RegClassID, MVT::i16);
342   }
343 
344   bool isSCSrcV2B16() const {
345     return isSCSrcB16();
346   }
347 
348   bool isSCSrcB32() const {
349     return isRegOrInlineNoMods(AMDGPU::SReg_32RegClassID, MVT::i32);
350   }
351 
352   bool isSCSrcB64() const {
353     return isRegOrInlineNoMods(AMDGPU::SReg_64RegClassID, MVT::i64);
354   }
355 
356   bool isSCSrcF16() const {
357     return isRegOrInlineNoMods(AMDGPU::SReg_32RegClassID, MVT::f16);
358   }
359 
360   bool isSCSrcV2F16() const {
361     return isSCSrcF16();
362   }
363 
364   bool isSCSrcF32() const {
365     return isRegOrInlineNoMods(AMDGPU::SReg_32RegClassID, MVT::f32);
366   }
367 
368   bool isSCSrcF64() const {
369     return isRegOrInlineNoMods(AMDGPU::SReg_64RegClassID, MVT::f64);
370   }
371 
372   bool isSSrcB32() const {
373     return isSCSrcB32() || isLiteralImm(MVT::i32) || isExpr();
374   }
375 
376   bool isSSrcB16() const {
377     return isSCSrcB16() || isLiteralImm(MVT::i16);
378   }
379 
380   bool isSSrcV2B16() const {
381     llvm_unreachable("cannot happen");
382     return isSSrcB16();
383   }
384 
385   bool isSSrcB64() const {
386     // TODO: Find out how SALU supports extension of 32-bit literals to 64 bits.
387     // See isVSrc64().
388     return isSCSrcB64() || isLiteralImm(MVT::i64);
389   }
390 
391   bool isSSrcF32() const {
392     return isSCSrcB32() || isLiteralImm(MVT::f32) || isExpr();
393   }
394 
395   bool isSSrcF64() const {
396     return isSCSrcB64() || isLiteralImm(MVT::f64);
397   }
398 
399   bool isSSrcF16() const {
400     return isSCSrcB16() || isLiteralImm(MVT::f16);
401   }
402 
403   bool isSSrcV2F16() const {
404     llvm_unreachable("cannot happen");
405     return isSSrcF16();
406   }
407 
408   bool isVCSrcB32() const {
409     return isRegOrInlineNoMods(AMDGPU::VS_32RegClassID, MVT::i32);
410   }
411 
412   bool isVCSrcB64() const {
413     return isRegOrInlineNoMods(AMDGPU::VS_64RegClassID, MVT::i64);
414   }
415 
416   bool isVCSrcB16() const {
417     return isRegOrInlineNoMods(AMDGPU::VS_32RegClassID, MVT::i16);
418   }
419 
420   bool isVCSrcV2B16() const {
421     return isVCSrcB16();
422   }
423 
424   bool isVCSrcF32() const {
425     return isRegOrInlineNoMods(AMDGPU::VS_32RegClassID, MVT::f32);
426   }
427 
428   bool isVCSrcF64() const {
429     return isRegOrInlineNoMods(AMDGPU::VS_64RegClassID, MVT::f64);
430   }
431 
432   bool isVCSrcF16() const {
433     return isRegOrInlineNoMods(AMDGPU::VS_32RegClassID, MVT::f16);
434   }
435 
436   bool isVCSrcV2F16() const {
437     return isVCSrcF16();
438   }
439 
440   bool isVSrcB32() const {
441     return isVCSrcF32() || isLiteralImm(MVT::i32);
442   }
443 
444   bool isVSrcB64() const {
445     return isVCSrcF64() || isLiteralImm(MVT::i64);
446   }
447 
448   bool isVSrcB16() const {
449     return isVCSrcF16() || isLiteralImm(MVT::i16);
450   }
451 
452   bool isVSrcV2B16() const {
453     llvm_unreachable("cannot happen");
454     return isVSrcB16();
455   }
456 
457   bool isVSrcF32() const {
458     return isVCSrcF32() || isLiteralImm(MVT::f32);
459   }
460 
461   bool isVSrcF64() const {
462     return isVCSrcF64() || isLiteralImm(MVT::f64);
463   }
464 
465   bool isVSrcF16() const {
466     return isVCSrcF16() || isLiteralImm(MVT::f16);
467   }
468 
469   bool isVSrcV2F16() const {
470     llvm_unreachable("cannot happen");
471     return isVSrcF16();
472   }
473 
474   bool isKImmFP32() const {
475     return isLiteralImm(MVT::f32);
476   }
477 
478   bool isKImmFP16() const {
479     return isLiteralImm(MVT::f16);
480   }
481 
482   bool isMem() const override {
483     return false;
484   }
485 
486   bool isExpr() const {
487     return Kind == Expression;
488   }
489 
490   bool isSoppBrTarget() const {
491     return isExpr() || isImm();
492   }
493 
494   bool isSWaitCnt() const;
495   bool isHwreg() const;
496   bool isSendMsg() const;
497   bool isSwizzle() const;
498   bool isSMRDOffset8() const;
499   bool isSMRDOffset20() const;
500   bool isSMRDLiteralOffset() const;
501   bool isDPPCtrl() const;
502   bool isGPRIdxMode() const;
503   bool isS16Imm() const;
504   bool isU16Imm() const;
505 
506   StringRef getExpressionAsToken() const {
507     assert(isExpr());
508     const MCSymbolRefExpr *S = cast<MCSymbolRefExpr>(Expr);
509     return S->getSymbol().getName();
510   }
511 
512   StringRef getToken() const {
513     assert(isToken());
514 
515     if (Kind == Expression)
516       return getExpressionAsToken();
517 
518     return StringRef(Tok.Data, Tok.Length);
519   }
520 
521   int64_t getImm() const {
522     assert(isImm());
523     return Imm.Val;
524   }
525 
526   ImmTy getImmTy() const {
527     assert(isImm());
528     return Imm.Type;
529   }
530 
531   unsigned getReg() const override {
532     return Reg.RegNo;
533   }
534 
535   SMLoc getStartLoc() const override {
536     return StartLoc;
537   }
538 
539   SMLoc getEndLoc() const override {
540     return EndLoc;
541   }
542 
543   SMRange getLocRange() const {
544     return SMRange(StartLoc, EndLoc);
545   }
546 
547   Modifiers getModifiers() const {
548     assert(isRegKind() || isImmTy(ImmTyNone));
549     return isRegKind() ? Reg.Mods : Imm.Mods;
550   }
551 
552   void setModifiers(Modifiers Mods) {
553     assert(isRegKind() || isImmTy(ImmTyNone));
554     if (isRegKind())
555       Reg.Mods = Mods;
556     else
557       Imm.Mods = Mods;
558   }
559 
560   bool hasModifiers() const {
561     return getModifiers().hasModifiers();
562   }
563 
564   bool hasFPModifiers() const {
565     return getModifiers().hasFPModifiers();
566   }
567 
568   bool hasIntModifiers() const {
569     return getModifiers().hasIntModifiers();
570   }
571 
572   uint64_t applyInputFPModifiers(uint64_t Val, unsigned Size) const;
573 
574   void addImmOperands(MCInst &Inst, unsigned N, bool ApplyModifiers = true) const;
575 
576   void addLiteralImmOperand(MCInst &Inst, int64_t Val, bool ApplyModifiers) const;
577 
578   template <unsigned Bitwidth>
579   void addKImmFPOperands(MCInst &Inst, unsigned N) const;
580 
581   void addKImmFP16Operands(MCInst &Inst, unsigned N) const {
582     addKImmFPOperands<16>(Inst, N);
583   }
584 
585   void addKImmFP32Operands(MCInst &Inst, unsigned N) const {
586     addKImmFPOperands<32>(Inst, N);
587   }
588 
589   void addRegOperands(MCInst &Inst, unsigned N) const;
590 
591   void addRegOrImmOperands(MCInst &Inst, unsigned N) const {
592     if (isRegKind())
593       addRegOperands(Inst, N);
594     else if (isExpr())
595       Inst.addOperand(MCOperand::createExpr(Expr));
596     else
597       addImmOperands(Inst, N);
598   }
599 
600   void addRegOrImmWithInputModsOperands(MCInst &Inst, unsigned N) const {
601     Modifiers Mods = getModifiers();
602     Inst.addOperand(MCOperand::createImm(Mods.getModifiersOperand()));
603     if (isRegKind()) {
604       addRegOperands(Inst, N);
605     } else {
606       addImmOperands(Inst, N, false);
607     }
608   }
609 
610   void addRegOrImmWithFPInputModsOperands(MCInst &Inst, unsigned N) const {
611     assert(!hasIntModifiers());
612     addRegOrImmWithInputModsOperands(Inst, N);
613   }
614 
615   void addRegOrImmWithIntInputModsOperands(MCInst &Inst, unsigned N) const {
616     assert(!hasFPModifiers());
617     addRegOrImmWithInputModsOperands(Inst, N);
618   }
619 
620   void addRegWithInputModsOperands(MCInst &Inst, unsigned N) const {
621     Modifiers Mods = getModifiers();
622     Inst.addOperand(MCOperand::createImm(Mods.getModifiersOperand()));
623     assert(isRegKind());
624     addRegOperands(Inst, N);
625   }
626 
627   void addRegWithFPInputModsOperands(MCInst &Inst, unsigned N) const {
628     assert(!hasIntModifiers());
629     addRegWithInputModsOperands(Inst, N);
630   }
631 
632   void addRegWithIntInputModsOperands(MCInst &Inst, unsigned N) const {
633     assert(!hasFPModifiers());
634     addRegWithInputModsOperands(Inst, N);
635   }
636 
637   void addSoppBrTargetOperands(MCInst &Inst, unsigned N) const {
638     if (isImm())
639       addImmOperands(Inst, N);
640     else {
641       assert(isExpr());
642       Inst.addOperand(MCOperand::createExpr(Expr));
643     }
644   }
645 
646   static void printImmTy(raw_ostream& OS, ImmTy Type) {
647     switch (Type) {
648     case ImmTyNone: OS << "None"; break;
649     case ImmTyGDS: OS << "GDS"; break;
650     case ImmTyOffen: OS << "Offen"; break;
651     case ImmTyIdxen: OS << "Idxen"; break;
652     case ImmTyAddr64: OS << "Addr64"; break;
653     case ImmTyOffset: OS << "Offset"; break;
654     case ImmTyInstOffset: OS << "InstOffset"; break;
655     case ImmTyOffset0: OS << "Offset0"; break;
656     case ImmTyOffset1: OS << "Offset1"; break;
657     case ImmTyGLC: OS << "GLC"; break;
658     case ImmTySLC: OS << "SLC"; break;
659     case ImmTyTFE: OS << "TFE"; break;
660     case ImmTyDFMT: OS << "DFMT"; break;
661     case ImmTyNFMT: OS << "NFMT"; break;
662     case ImmTyClampSI: OS << "ClampSI"; break;
663     case ImmTyOModSI: OS << "OModSI"; break;
664     case ImmTyDppCtrl: OS << "DppCtrl"; break;
665     case ImmTyDppRowMask: OS << "DppRowMask"; break;
666     case ImmTyDppBankMask: OS << "DppBankMask"; break;
667     case ImmTyDppBoundCtrl: OS << "DppBoundCtrl"; break;
668     case ImmTySdwaDstSel: OS << "SdwaDstSel"; break;
669     case ImmTySdwaSrc0Sel: OS << "SdwaSrc0Sel"; break;
670     case ImmTySdwaSrc1Sel: OS << "SdwaSrc1Sel"; break;
671     case ImmTySdwaDstUnused: OS << "SdwaDstUnused"; break;
672     case ImmTyDMask: OS << "DMask"; break;
673     case ImmTyUNorm: OS << "UNorm"; break;
674     case ImmTyDA: OS << "DA"; break;
675     case ImmTyR128: OS << "R128"; break;
676     case ImmTyLWE: OS << "LWE"; break;
677     case ImmTyOff: OS << "Off"; break;
678     case ImmTyExpTgt: OS << "ExpTgt"; break;
679     case ImmTyExpCompr: OS << "ExpCompr"; break;
680     case ImmTyExpVM: OS << "ExpVM"; break;
681     case ImmTyHwreg: OS << "Hwreg"; break;
682     case ImmTySendMsg: OS << "SendMsg"; break;
683     case ImmTyInterpSlot: OS << "InterpSlot"; break;
684     case ImmTyInterpAttr: OS << "InterpAttr"; break;
685     case ImmTyAttrChan: OS << "AttrChan"; break;
686     case ImmTyOpSel: OS << "OpSel"; break;
687     case ImmTyOpSelHi: OS << "OpSelHi"; break;
688     case ImmTyNegLo: OS << "NegLo"; break;
689     case ImmTyNegHi: OS << "NegHi"; break;
690     case ImmTySwizzle: OS << "Swizzle"; break;
691     case ImmTyHigh: OS << "High"; break;
692     }
693   }
694 
695   void print(raw_ostream &OS) const override {
696     switch (Kind) {
697     case Register:
698       OS << "<register " << getReg() << " mods: " << Reg.Mods << '>';
699       break;
700     case Immediate:
701       OS << '<' << getImm();
702       if (getImmTy() != ImmTyNone) {
703         OS << " type: "; printImmTy(OS, getImmTy());
704       }
705       OS << " mods: " << Imm.Mods << '>';
706       break;
707     case Token:
708       OS << '\'' << getToken() << '\'';
709       break;
710     case Expression:
711       OS << "<expr " << *Expr << '>';
712       break;
713     }
714   }
715 
716   static AMDGPUOperand::Ptr CreateImm(const AMDGPUAsmParser *AsmParser,
717                                       int64_t Val, SMLoc Loc,
718                                       ImmTy Type = ImmTyNone,
719                                       bool IsFPImm = false) {
720     auto Op = llvm::make_unique<AMDGPUOperand>(Immediate, AsmParser);
721     Op->Imm.Val = Val;
722     Op->Imm.IsFPImm = IsFPImm;
723     Op->Imm.Type = Type;
724     Op->Imm.Mods = Modifiers();
725     Op->StartLoc = Loc;
726     Op->EndLoc = Loc;
727     return Op;
728   }
729 
730   static AMDGPUOperand::Ptr CreateToken(const AMDGPUAsmParser *AsmParser,
731                                         StringRef Str, SMLoc Loc,
732                                         bool HasExplicitEncodingSize = true) {
733     auto Res = llvm::make_unique<AMDGPUOperand>(Token, AsmParser);
734     Res->Tok.Data = Str.data();
735     Res->Tok.Length = Str.size();
736     Res->StartLoc = Loc;
737     Res->EndLoc = Loc;
738     return Res;
739   }
740 
741   static AMDGPUOperand::Ptr CreateReg(const AMDGPUAsmParser *AsmParser,
742                                       unsigned RegNo, SMLoc S,
743                                       SMLoc E,
744                                       bool ForceVOP3) {
745     auto Op = llvm::make_unique<AMDGPUOperand>(Register, AsmParser);
746     Op->Reg.RegNo = RegNo;
747     Op->Reg.Mods = Modifiers();
748     Op->Reg.IsForcedVOP3 = ForceVOP3;
749     Op->StartLoc = S;
750     Op->EndLoc = E;
751     return Op;
752   }
753 
754   static AMDGPUOperand::Ptr CreateExpr(const AMDGPUAsmParser *AsmParser,
755                                        const class MCExpr *Expr, SMLoc S) {
756     auto Op = llvm::make_unique<AMDGPUOperand>(Expression, AsmParser);
757     Op->Expr = Expr;
758     Op->StartLoc = S;
759     Op->EndLoc = S;
760     return Op;
761   }
762 };
763 
764 raw_ostream &operator <<(raw_ostream &OS, AMDGPUOperand::Modifiers Mods) {
765   OS << "abs:" << Mods.Abs << " neg: " << Mods.Neg << " sext:" << Mods.Sext;
766   return OS;
767 }
768 
769 //===----------------------------------------------------------------------===//
770 // AsmParser
771 //===----------------------------------------------------------------------===//
772 
773 // Holds info related to the current kernel, e.g. count of SGPRs used.
774 // Kernel scope begins at .amdgpu_hsa_kernel directive, ends at next
775 // .amdgpu_hsa_kernel or at EOF.
776 class KernelScopeInfo {
777   int SgprIndexUnusedMin = -1;
778   int VgprIndexUnusedMin = -1;
779   MCContext *Ctx = nullptr;
780 
781   void usesSgprAt(int i) {
782     if (i >= SgprIndexUnusedMin) {
783       SgprIndexUnusedMin = ++i;
784       if (Ctx) {
785         MCSymbol * const Sym = Ctx->getOrCreateSymbol(Twine(".kernel.sgpr_count"));
786         Sym->setVariableValue(MCConstantExpr::create(SgprIndexUnusedMin, *Ctx));
787       }
788     }
789   }
790 
791   void usesVgprAt(int i) {
792     if (i >= VgprIndexUnusedMin) {
793       VgprIndexUnusedMin = ++i;
794       if (Ctx) {
795         MCSymbol * const Sym = Ctx->getOrCreateSymbol(Twine(".kernel.vgpr_count"));
796         Sym->setVariableValue(MCConstantExpr::create(VgprIndexUnusedMin, *Ctx));
797       }
798     }
799   }
800 
801 public:
802   KernelScopeInfo() = default;
803 
804   void initialize(MCContext &Context) {
805     Ctx = &Context;
806     usesSgprAt(SgprIndexUnusedMin = -1);
807     usesVgprAt(VgprIndexUnusedMin = -1);
808   }
809 
810   void usesRegister(RegisterKind RegKind, unsigned DwordRegIndex, unsigned RegWidth) {
811     switch (RegKind) {
812       case IS_SGPR: usesSgprAt(DwordRegIndex + RegWidth - 1); break;
813       case IS_VGPR: usesVgprAt(DwordRegIndex + RegWidth - 1); break;
814       default: break;
815     }
816   }
817 };
818 
819 class AMDGPUAsmParser : public MCTargetAsmParser {
820   MCAsmParser &Parser;
821 
822   // Number of extra operands parsed after the first optional operand.
823   // This may be necessary to skip hardcoded mandatory operands.
824   static const unsigned MAX_OPR_LOOKAHEAD = 1;
825 
826   unsigned ForcedEncodingSize = 0;
827   bool ForcedDPP = false;
828   bool ForcedSDWA = false;
829   KernelScopeInfo KernelScope;
830 
831   /// @name Auto-generated Match Functions
832   /// {
833 
834 #define GET_ASSEMBLER_HEADER
835 #include "AMDGPUGenAsmMatcher.inc"
836 
837   /// }
838 
839 private:
840   bool ParseAsAbsoluteExpression(uint32_t &Ret);
841   bool ParseDirectiveMajorMinor(uint32_t &Major, uint32_t &Minor);
842   bool ParseDirectiveHSACodeObjectVersion();
843   bool ParseDirectiveHSACodeObjectISA();
844   bool ParseAMDKernelCodeTValue(StringRef ID, amd_kernel_code_t &Header);
845   bool ParseDirectiveAMDKernelCodeT();
846   bool subtargetHasRegister(const MCRegisterInfo &MRI, unsigned RegNo) const;
847   bool ParseDirectiveAMDGPUHsaKernel();
848 
849   bool ParseDirectiveISAVersion();
850   bool ParseDirectiveHSAMetadata();
851   bool ParseDirectivePALMetadata();
852 
853   bool AddNextRegisterToList(unsigned& Reg, unsigned& RegWidth,
854                              RegisterKind RegKind, unsigned Reg1,
855                              unsigned RegNum);
856   bool ParseAMDGPURegister(RegisterKind& RegKind, unsigned& Reg,
857                            unsigned& RegNum, unsigned& RegWidth,
858                            unsigned *DwordRegIndex);
859   void cvtMubufImpl(MCInst &Inst, const OperandVector &Operands,
860                     bool IsAtomic, bool IsAtomicReturn);
861   void cvtDSImpl(MCInst &Inst, const OperandVector &Operands,
862                  bool IsGdsHardcoded);
863 
864 public:
865   enum AMDGPUMatchResultTy {
866     Match_PreferE32 = FIRST_TARGET_MATCH_RESULT_TY
867   };
868 
869   using OptionalImmIndexMap = std::map<AMDGPUOperand::ImmTy, unsigned>;
870 
871   AMDGPUAsmParser(const MCSubtargetInfo &STI, MCAsmParser &_Parser,
872                const MCInstrInfo &MII,
873                const MCTargetOptions &Options)
874       : MCTargetAsmParser(Options, STI, MII), Parser(_Parser) {
875     MCAsmParserExtension::Initialize(Parser);
876 
877     if (getFeatureBits().none()) {
878       // Set default features.
879       copySTI().ToggleFeature("SOUTHERN_ISLANDS");
880     }
881 
882     setAvailableFeatures(ComputeAvailableFeatures(getFeatureBits()));
883 
884     {
885       // TODO: make those pre-defined variables read-only.
886       // Currently there is none suitable machinery in the core llvm-mc for this.
887       // MCSymbol::isRedefinable is intended for another purpose, and
888       // AsmParser::parseDirectiveSet() cannot be specialized for specific target.
889       AMDGPU::IsaInfo::IsaVersion ISA =
890           AMDGPU::IsaInfo::getIsaVersion(getFeatureBits());
891       MCContext &Ctx = getContext();
892       MCSymbol *Sym =
893           Ctx.getOrCreateSymbol(Twine(".option.machine_version_major"));
894       Sym->setVariableValue(MCConstantExpr::create(ISA.Major, Ctx));
895       Sym = Ctx.getOrCreateSymbol(Twine(".option.machine_version_minor"));
896       Sym->setVariableValue(MCConstantExpr::create(ISA.Minor, Ctx));
897       Sym = Ctx.getOrCreateSymbol(Twine(".option.machine_version_stepping"));
898       Sym->setVariableValue(MCConstantExpr::create(ISA.Stepping, Ctx));
899     }
900     KernelScope.initialize(getContext());
901   }
902 
903   bool hasXNACK() const {
904     return AMDGPU::hasXNACK(getSTI());
905   }
906 
907   bool isSI() const {
908     return AMDGPU::isSI(getSTI());
909   }
910 
911   bool isCI() const {
912     return AMDGPU::isCI(getSTI());
913   }
914 
915   bool isVI() const {
916     return AMDGPU::isVI(getSTI());
917   }
918 
919   bool isGFX9() const {
920     return AMDGPU::isGFX9(getSTI());
921   }
922 
923   bool hasInv2PiInlineImm() const {
924     return getFeatureBits()[AMDGPU::FeatureInv2PiInlineImm];
925   }
926 
927   bool hasFlatOffsets() const {
928     return getFeatureBits()[AMDGPU::FeatureFlatInstOffsets];
929   }
930 
931   bool hasSGPR102_SGPR103() const {
932     return !isVI();
933   }
934 
935   bool hasIntClamp() const {
936     return getFeatureBits()[AMDGPU::FeatureIntClamp];
937   }
938 
939   AMDGPUTargetStreamer &getTargetStreamer() {
940     MCTargetStreamer &TS = *getParser().getStreamer().getTargetStreamer();
941     return static_cast<AMDGPUTargetStreamer &>(TS);
942   }
943 
944   const MCRegisterInfo *getMRI() const {
945     // We need this const_cast because for some reason getContext() is not const
946     // in MCAsmParser.
947     return const_cast<AMDGPUAsmParser*>(this)->getContext().getRegisterInfo();
948   }
949 
950   const MCInstrInfo *getMII() const {
951     return &MII;
952   }
953 
954   const FeatureBitset &getFeatureBits() const {
955     return getSTI().getFeatureBits();
956   }
957 
958   void setForcedEncodingSize(unsigned Size) { ForcedEncodingSize = Size; }
959   void setForcedDPP(bool ForceDPP_) { ForcedDPP = ForceDPP_; }
960   void setForcedSDWA(bool ForceSDWA_) { ForcedSDWA = ForceSDWA_; }
961 
962   unsigned getForcedEncodingSize() const { return ForcedEncodingSize; }
963   bool isForcedVOP3() const { return ForcedEncodingSize == 64; }
964   bool isForcedDPP() const { return ForcedDPP; }
965   bool isForcedSDWA() const { return ForcedSDWA; }
966   ArrayRef<unsigned> getMatchedVariants() const;
967 
968   std::unique_ptr<AMDGPUOperand> parseRegister();
969   bool ParseRegister(unsigned &RegNo, SMLoc &StartLoc, SMLoc &EndLoc) override;
970   unsigned checkTargetMatchPredicate(MCInst &Inst) override;
971   unsigned validateTargetOperandClass(MCParsedAsmOperand &Op,
972                                       unsigned Kind) override;
973   bool MatchAndEmitInstruction(SMLoc IDLoc, unsigned &Opcode,
974                                OperandVector &Operands, MCStreamer &Out,
975                                uint64_t &ErrorInfo,
976                                bool MatchingInlineAsm) override;
977   bool ParseDirective(AsmToken DirectiveID) override;
978   OperandMatchResultTy parseOperand(OperandVector &Operands, StringRef Mnemonic);
979   StringRef parseMnemonicSuffix(StringRef Name);
980   bool ParseInstruction(ParseInstructionInfo &Info, StringRef Name,
981                         SMLoc NameLoc, OperandVector &Operands) override;
982   //bool ProcessInstruction(MCInst &Inst);
983 
984   OperandMatchResultTy parseIntWithPrefix(const char *Prefix, int64_t &Int);
985 
986   OperandMatchResultTy
987   parseIntWithPrefix(const char *Prefix, OperandVector &Operands,
988                      AMDGPUOperand::ImmTy ImmTy = AMDGPUOperand::ImmTyNone,
989                      bool (*ConvertResult)(int64_t &) = nullptr);
990 
991   OperandMatchResultTy parseOperandArrayWithPrefix(
992     const char *Prefix,
993     OperandVector &Operands,
994     AMDGPUOperand::ImmTy ImmTy = AMDGPUOperand::ImmTyNone,
995     bool (*ConvertResult)(int64_t&) = nullptr);
996 
997   OperandMatchResultTy
998   parseNamedBit(const char *Name, OperandVector &Operands,
999                 AMDGPUOperand::ImmTy ImmTy = AMDGPUOperand::ImmTyNone);
1000   OperandMatchResultTy parseStringWithPrefix(StringRef Prefix,
1001                                              StringRef &Value);
1002 
1003   bool parseAbsoluteExpr(int64_t &Val, bool AbsMod = false);
1004   OperandMatchResultTy parseImm(OperandVector &Operands, bool AbsMod = false);
1005   OperandMatchResultTy parseReg(OperandVector &Operands);
1006   OperandMatchResultTy parseRegOrImm(OperandVector &Operands, bool AbsMod = false);
1007   OperandMatchResultTy parseRegOrImmWithFPInputMods(OperandVector &Operands, bool AllowImm = true);
1008   OperandMatchResultTy parseRegOrImmWithIntInputMods(OperandVector &Operands, bool AllowImm = true);
1009   OperandMatchResultTy parseRegWithFPInputMods(OperandVector &Operands);
1010   OperandMatchResultTy parseRegWithIntInputMods(OperandVector &Operands);
1011   OperandMatchResultTy parseVReg32OrOff(OperandVector &Operands);
1012 
1013   void cvtDSOffset01(MCInst &Inst, const OperandVector &Operands);
1014   void cvtDS(MCInst &Inst, const OperandVector &Operands) { cvtDSImpl(Inst, Operands, false); }
1015   void cvtDSGds(MCInst &Inst, const OperandVector &Operands) { cvtDSImpl(Inst, Operands, true); }
1016   void cvtExp(MCInst &Inst, const OperandVector &Operands);
1017 
1018   bool parseCnt(int64_t &IntVal);
1019   OperandMatchResultTy parseSWaitCntOps(OperandVector &Operands);
1020   OperandMatchResultTy parseHwreg(OperandVector &Operands);
1021 
1022 private:
1023   struct OperandInfoTy {
1024     int64_t Id;
1025     bool IsSymbolic = false;
1026 
1027     OperandInfoTy(int64_t Id_) : Id(Id_) {}
1028   };
1029 
1030   bool parseSendMsgConstruct(OperandInfoTy &Msg, OperandInfoTy &Operation, int64_t &StreamId);
1031   bool parseHwregConstruct(OperandInfoTy &HwReg, int64_t &Offset, int64_t &Width);
1032 
1033   void errorExpTgt();
1034   OperandMatchResultTy parseExpTgtImpl(StringRef Str, uint8_t &Val);
1035 
1036   bool validateInstruction(const MCInst &Inst, const SMLoc &IDLoc);
1037   bool validateConstantBusLimitations(const MCInst &Inst);
1038   bool validateEarlyClobberLimitations(const MCInst &Inst);
1039   bool validateIntClampSupported(const MCInst &Inst);
1040   bool usesConstantBus(const MCInst &Inst, unsigned OpIdx);
1041   bool isInlineConstant(const MCInst &Inst, unsigned OpIdx) const;
1042   unsigned findImplicitSGPRReadInVOP(const MCInst &Inst) const;
1043 
1044   bool trySkipId(const StringRef Id);
1045   bool trySkipToken(const AsmToken::TokenKind Kind);
1046   bool skipToken(const AsmToken::TokenKind Kind, const StringRef ErrMsg);
1047   bool parseString(StringRef &Val, const StringRef ErrMsg = "expected a string");
1048   bool parseExpr(int64_t &Imm);
1049 
1050 public:
1051   OperandMatchResultTy parseOptionalOperand(OperandVector &Operands);
1052   OperandMatchResultTy parseOptionalOpr(OperandVector &Operands);
1053 
1054   OperandMatchResultTy parseExpTgt(OperandVector &Operands);
1055   OperandMatchResultTy parseSendMsgOp(OperandVector &Operands);
1056   OperandMatchResultTy parseInterpSlot(OperandVector &Operands);
1057   OperandMatchResultTy parseInterpAttr(OperandVector &Operands);
1058   OperandMatchResultTy parseSOppBrTarget(OperandVector &Operands);
1059 
1060   bool parseSwizzleOperands(const unsigned OpNum, int64_t* Op,
1061                             const unsigned MinVal,
1062                             const unsigned MaxVal,
1063                             const StringRef ErrMsg);
1064   OperandMatchResultTy parseSwizzleOp(OperandVector &Operands);
1065   bool parseSwizzleOffset(int64_t &Imm);
1066   bool parseSwizzleMacro(int64_t &Imm);
1067   bool parseSwizzleQuadPerm(int64_t &Imm);
1068   bool parseSwizzleBitmaskPerm(int64_t &Imm);
1069   bool parseSwizzleBroadcast(int64_t &Imm);
1070   bool parseSwizzleSwap(int64_t &Imm);
1071   bool parseSwizzleReverse(int64_t &Imm);
1072 
1073   void cvtMubuf(MCInst &Inst, const OperandVector &Operands) { cvtMubufImpl(Inst, Operands, false, false); }
1074   void cvtMubufAtomic(MCInst &Inst, const OperandVector &Operands) { cvtMubufImpl(Inst, Operands, true, false); }
1075   void cvtMubufAtomicReturn(MCInst &Inst, const OperandVector &Operands) { cvtMubufImpl(Inst, Operands, true, true); }
1076   void cvtMtbuf(MCInst &Inst, const OperandVector &Operands);
1077 
1078   AMDGPUOperand::Ptr defaultGLC() const;
1079   AMDGPUOperand::Ptr defaultSLC() const;
1080   AMDGPUOperand::Ptr defaultTFE() const;
1081 
1082   AMDGPUOperand::Ptr defaultDMask() const;
1083   AMDGPUOperand::Ptr defaultUNorm() const;
1084   AMDGPUOperand::Ptr defaultDA() const;
1085   AMDGPUOperand::Ptr defaultR128() const;
1086   AMDGPUOperand::Ptr defaultLWE() const;
1087   AMDGPUOperand::Ptr defaultSMRDOffset8() const;
1088   AMDGPUOperand::Ptr defaultSMRDOffset20() const;
1089   AMDGPUOperand::Ptr defaultSMRDLiteralOffset() const;
1090   AMDGPUOperand::Ptr defaultOffsetU12() const;
1091   AMDGPUOperand::Ptr defaultOffsetS13() const;
1092 
1093   OperandMatchResultTy parseOModOperand(OperandVector &Operands);
1094 
1095   void cvtVOP3(MCInst &Inst, const OperandVector &Operands,
1096                OptionalImmIndexMap &OptionalIdx);
1097   void cvtVOP3OpSel(MCInst &Inst, const OperandVector &Operands);
1098   void cvtVOP3(MCInst &Inst, const OperandVector &Operands);
1099   void cvtVOP3P(MCInst &Inst, const OperandVector &Operands);
1100 
1101   void cvtVOP3Interp(MCInst &Inst, const OperandVector &Operands);
1102 
1103   void cvtMIMG(MCInst &Inst, const OperandVector &Operands,
1104                bool IsAtomic = false);
1105   void cvtMIMGAtomic(MCInst &Inst, const OperandVector &Operands);
1106 
1107   OperandMatchResultTy parseDPPCtrl(OperandVector &Operands);
1108   AMDGPUOperand::Ptr defaultRowMask() const;
1109   AMDGPUOperand::Ptr defaultBankMask() const;
1110   AMDGPUOperand::Ptr defaultBoundCtrl() const;
1111   void cvtDPP(MCInst &Inst, const OperandVector &Operands);
1112 
1113   OperandMatchResultTy parseSDWASel(OperandVector &Operands, StringRef Prefix,
1114                                     AMDGPUOperand::ImmTy Type);
1115   OperandMatchResultTy parseSDWADstUnused(OperandVector &Operands);
1116   void cvtSdwaVOP1(MCInst &Inst, const OperandVector &Operands);
1117   void cvtSdwaVOP2(MCInst &Inst, const OperandVector &Operands);
1118   void cvtSdwaVOP2b(MCInst &Inst, const OperandVector &Operands);
1119   void cvtSdwaVOPC(MCInst &Inst, const OperandVector &Operands);
1120   void cvtSDWA(MCInst &Inst, const OperandVector &Operands,
1121                 uint64_t BasicInstType, bool skipVcc = false);
1122 };
1123 
1124 struct OptionalOperand {
1125   const char *Name;
1126   AMDGPUOperand::ImmTy Type;
1127   bool IsBit;
1128   bool (*ConvertResult)(int64_t&);
1129 };
1130 
1131 } // end anonymous namespace
1132 
1133 // May be called with integer type with equivalent bitwidth.
1134 static const fltSemantics *getFltSemantics(unsigned Size) {
1135   switch (Size) {
1136   case 4:
1137     return &APFloat::IEEEsingle();
1138   case 8:
1139     return &APFloat::IEEEdouble();
1140   case 2:
1141     return &APFloat::IEEEhalf();
1142   default:
1143     llvm_unreachable("unsupported fp type");
1144   }
1145 }
1146 
1147 static const fltSemantics *getFltSemantics(MVT VT) {
1148   return getFltSemantics(VT.getSizeInBits() / 8);
1149 }
1150 
1151 static const fltSemantics *getOpFltSemantics(uint8_t OperandType) {
1152   switch (OperandType) {
1153   case AMDGPU::OPERAND_REG_IMM_INT32:
1154   case AMDGPU::OPERAND_REG_IMM_FP32:
1155   case AMDGPU::OPERAND_REG_INLINE_C_INT32:
1156   case AMDGPU::OPERAND_REG_INLINE_C_FP32:
1157     return &APFloat::IEEEsingle();
1158   case AMDGPU::OPERAND_REG_IMM_INT64:
1159   case AMDGPU::OPERAND_REG_IMM_FP64:
1160   case AMDGPU::OPERAND_REG_INLINE_C_INT64:
1161   case AMDGPU::OPERAND_REG_INLINE_C_FP64:
1162     return &APFloat::IEEEdouble();
1163   case AMDGPU::OPERAND_REG_IMM_INT16:
1164   case AMDGPU::OPERAND_REG_IMM_FP16:
1165   case AMDGPU::OPERAND_REG_INLINE_C_INT16:
1166   case AMDGPU::OPERAND_REG_INLINE_C_FP16:
1167   case AMDGPU::OPERAND_REG_INLINE_C_V2INT16:
1168   case AMDGPU::OPERAND_REG_INLINE_C_V2FP16:
1169     return &APFloat::IEEEhalf();
1170   default:
1171     llvm_unreachable("unsupported fp type");
1172   }
1173 }
1174 
1175 //===----------------------------------------------------------------------===//
1176 // Operand
1177 //===----------------------------------------------------------------------===//
1178 
1179 static bool canLosslesslyConvertToFPType(APFloat &FPLiteral, MVT VT) {
1180   bool Lost;
1181 
1182   // Convert literal to single precision
1183   APFloat::opStatus Status = FPLiteral.convert(*getFltSemantics(VT),
1184                                                APFloat::rmNearestTiesToEven,
1185                                                &Lost);
1186   // We allow precision lost but not overflow or underflow
1187   if (Status != APFloat::opOK &&
1188       Lost &&
1189       ((Status & APFloat::opOverflow)  != 0 ||
1190        (Status & APFloat::opUnderflow) != 0)) {
1191     return false;
1192   }
1193 
1194   return true;
1195 }
1196 
1197 bool AMDGPUOperand::isInlinableImm(MVT type) const {
1198   if (!isImmTy(ImmTyNone)) {
1199     // Only plain immediates are inlinable (e.g. "clamp" attribute is not)
1200     return false;
1201   }
1202   // TODO: We should avoid using host float here. It would be better to
1203   // check the float bit values which is what a few other places do.
1204   // We've had bot failures before due to weird NaN support on mips hosts.
1205 
1206   APInt Literal(64, Imm.Val);
1207 
1208   if (Imm.IsFPImm) { // We got fp literal token
1209     if (type == MVT::f64 || type == MVT::i64) { // Expected 64-bit operand
1210       return AMDGPU::isInlinableLiteral64(Imm.Val,
1211                                           AsmParser->hasInv2PiInlineImm());
1212     }
1213 
1214     APFloat FPLiteral(APFloat::IEEEdouble(), APInt(64, Imm.Val));
1215     if (!canLosslesslyConvertToFPType(FPLiteral, type))
1216       return false;
1217 
1218     if (type.getScalarSizeInBits() == 16) {
1219       return AMDGPU::isInlinableLiteral16(
1220         static_cast<int16_t>(FPLiteral.bitcastToAPInt().getZExtValue()),
1221         AsmParser->hasInv2PiInlineImm());
1222     }
1223 
1224     // Check if single precision literal is inlinable
1225     return AMDGPU::isInlinableLiteral32(
1226       static_cast<int32_t>(FPLiteral.bitcastToAPInt().getZExtValue()),
1227       AsmParser->hasInv2PiInlineImm());
1228   }
1229 
1230   // We got int literal token.
1231   if (type == MVT::f64 || type == MVT::i64) { // Expected 64-bit operand
1232     return AMDGPU::isInlinableLiteral64(Imm.Val,
1233                                         AsmParser->hasInv2PiInlineImm());
1234   }
1235 
1236   if (type.getScalarSizeInBits() == 16) {
1237     return AMDGPU::isInlinableLiteral16(
1238       static_cast<int16_t>(Literal.getLoBits(16).getSExtValue()),
1239       AsmParser->hasInv2PiInlineImm());
1240   }
1241 
1242   return AMDGPU::isInlinableLiteral32(
1243     static_cast<int32_t>(Literal.getLoBits(32).getZExtValue()),
1244     AsmParser->hasInv2PiInlineImm());
1245 }
1246 
1247 bool AMDGPUOperand::isLiteralImm(MVT type) const {
1248   // Check that this immediate can be added as literal
1249   if (!isImmTy(ImmTyNone)) {
1250     return false;
1251   }
1252 
1253   if (!Imm.IsFPImm) {
1254     // We got int literal token.
1255 
1256     if (type == MVT::f64 && hasFPModifiers()) {
1257       // Cannot apply fp modifiers to int literals preserving the same semantics
1258       // for VOP1/2/C and VOP3 because of integer truncation. To avoid ambiguity,
1259       // disable these cases.
1260       return false;
1261     }
1262 
1263     unsigned Size = type.getSizeInBits();
1264     if (Size == 64)
1265       Size = 32;
1266 
1267     // FIXME: 64-bit operands can zero extend, sign extend, or pad zeroes for FP
1268     // types.
1269     return isUIntN(Size, Imm.Val) || isIntN(Size, Imm.Val);
1270   }
1271 
1272   // We got fp literal token
1273   if (type == MVT::f64) { // Expected 64-bit fp operand
1274     // We would set low 64-bits of literal to zeroes but we accept this literals
1275     return true;
1276   }
1277 
1278   if (type == MVT::i64) { // Expected 64-bit int operand
1279     // We don't allow fp literals in 64-bit integer instructions. It is
1280     // unclear how we should encode them.
1281     return false;
1282   }
1283 
1284   APFloat FPLiteral(APFloat::IEEEdouble(), APInt(64, Imm.Val));
1285   return canLosslesslyConvertToFPType(FPLiteral, type);
1286 }
1287 
1288 bool AMDGPUOperand::isRegClass(unsigned RCID) const {
1289   return isRegKind() && AsmParser->getMRI()->getRegClass(RCID).contains(getReg());
1290 }
1291 
1292 bool AMDGPUOperand::isSDWAOperand(MVT type) const {
1293   if (AsmParser->isVI())
1294     return isVReg();
1295   else if (AsmParser->isGFX9())
1296     return isRegKind() || isInlinableImm(type);
1297   else
1298     return false;
1299 }
1300 
1301 bool AMDGPUOperand::isSDWAFP16Operand() const {
1302   return isSDWAOperand(MVT::f16);
1303 }
1304 
1305 bool AMDGPUOperand::isSDWAFP32Operand() const {
1306   return isSDWAOperand(MVT::f32);
1307 }
1308 
1309 bool AMDGPUOperand::isSDWAInt16Operand() const {
1310   return isSDWAOperand(MVT::i16);
1311 }
1312 
1313 bool AMDGPUOperand::isSDWAInt32Operand() const {
1314   return isSDWAOperand(MVT::i32);
1315 }
1316 
1317 uint64_t AMDGPUOperand::applyInputFPModifiers(uint64_t Val, unsigned Size) const
1318 {
1319   assert(isImmTy(ImmTyNone) && Imm.Mods.hasFPModifiers());
1320   assert(Size == 2 || Size == 4 || Size == 8);
1321 
1322   const uint64_t FpSignMask = (1ULL << (Size * 8 - 1));
1323 
1324   if (Imm.Mods.Abs) {
1325     Val &= ~FpSignMask;
1326   }
1327   if (Imm.Mods.Neg) {
1328     Val ^= FpSignMask;
1329   }
1330 
1331   return Val;
1332 }
1333 
1334 void AMDGPUOperand::addImmOperands(MCInst &Inst, unsigned N, bool ApplyModifiers) const {
1335   if (AMDGPU::isSISrcOperand(AsmParser->getMII()->get(Inst.getOpcode()),
1336                              Inst.getNumOperands())) {
1337     addLiteralImmOperand(Inst, Imm.Val,
1338                          ApplyModifiers &
1339                          isImmTy(ImmTyNone) && Imm.Mods.hasFPModifiers());
1340   } else {
1341     assert(!isImmTy(ImmTyNone) || !hasModifiers());
1342     Inst.addOperand(MCOperand::createImm(Imm.Val));
1343   }
1344 }
1345 
1346 void AMDGPUOperand::addLiteralImmOperand(MCInst &Inst, int64_t Val, bool ApplyModifiers) const {
1347   const auto& InstDesc = AsmParser->getMII()->get(Inst.getOpcode());
1348   auto OpNum = Inst.getNumOperands();
1349   // Check that this operand accepts literals
1350   assert(AMDGPU::isSISrcOperand(InstDesc, OpNum));
1351 
1352   if (ApplyModifiers) {
1353     assert(AMDGPU::isSISrcFPOperand(InstDesc, OpNum));
1354     const unsigned Size = Imm.IsFPImm ? sizeof(double) : getOperandSize(InstDesc, OpNum);
1355     Val = applyInputFPModifiers(Val, Size);
1356   }
1357 
1358   APInt Literal(64, Val);
1359   uint8_t OpTy = InstDesc.OpInfo[OpNum].OperandType;
1360 
1361   if (Imm.IsFPImm) { // We got fp literal token
1362     switch (OpTy) {
1363     case AMDGPU::OPERAND_REG_IMM_INT64:
1364     case AMDGPU::OPERAND_REG_IMM_FP64:
1365     case AMDGPU::OPERAND_REG_INLINE_C_INT64:
1366     case AMDGPU::OPERAND_REG_INLINE_C_FP64:
1367       if (AMDGPU::isInlinableLiteral64(Literal.getZExtValue(),
1368                                        AsmParser->hasInv2PiInlineImm())) {
1369         Inst.addOperand(MCOperand::createImm(Literal.getZExtValue()));
1370         return;
1371       }
1372 
1373       // Non-inlineable
1374       if (AMDGPU::isSISrcFPOperand(InstDesc, OpNum)) { // Expected 64-bit fp operand
1375         // For fp operands we check if low 32 bits are zeros
1376         if (Literal.getLoBits(32) != 0) {
1377           const_cast<AMDGPUAsmParser *>(AsmParser)->Warning(Inst.getLoc(),
1378           "Can't encode literal as exact 64-bit floating-point operand. "
1379           "Low 32-bits will be set to zero");
1380         }
1381 
1382         Inst.addOperand(MCOperand::createImm(Literal.lshr(32).getZExtValue()));
1383         return;
1384       }
1385 
1386       // We don't allow fp literals in 64-bit integer instructions. It is
1387       // unclear how we should encode them. This case should be checked earlier
1388       // in predicate methods (isLiteralImm())
1389       llvm_unreachable("fp literal in 64-bit integer instruction.");
1390 
1391     case AMDGPU::OPERAND_REG_IMM_INT32:
1392     case AMDGPU::OPERAND_REG_IMM_FP32:
1393     case AMDGPU::OPERAND_REG_INLINE_C_INT32:
1394     case AMDGPU::OPERAND_REG_INLINE_C_FP32:
1395     case AMDGPU::OPERAND_REG_IMM_INT16:
1396     case AMDGPU::OPERAND_REG_IMM_FP16:
1397     case AMDGPU::OPERAND_REG_INLINE_C_INT16:
1398     case AMDGPU::OPERAND_REG_INLINE_C_FP16:
1399     case AMDGPU::OPERAND_REG_INLINE_C_V2INT16:
1400     case AMDGPU::OPERAND_REG_INLINE_C_V2FP16: {
1401       bool lost;
1402       APFloat FPLiteral(APFloat::IEEEdouble(), Literal);
1403       // Convert literal to single precision
1404       FPLiteral.convert(*getOpFltSemantics(OpTy),
1405                         APFloat::rmNearestTiesToEven, &lost);
1406       // We allow precision lost but not overflow or underflow. This should be
1407       // checked earlier in isLiteralImm()
1408 
1409       uint64_t ImmVal = FPLiteral.bitcastToAPInt().getZExtValue();
1410       if (OpTy == AMDGPU::OPERAND_REG_INLINE_C_V2INT16 ||
1411           OpTy == AMDGPU::OPERAND_REG_INLINE_C_V2FP16) {
1412         ImmVal |= (ImmVal << 16);
1413       }
1414 
1415       Inst.addOperand(MCOperand::createImm(ImmVal));
1416       return;
1417     }
1418     default:
1419       llvm_unreachable("invalid operand size");
1420     }
1421 
1422     return;
1423   }
1424 
1425    // We got int literal token.
1426   // Only sign extend inline immediates.
1427   // FIXME: No errors on truncation
1428   switch (OpTy) {
1429   case AMDGPU::OPERAND_REG_IMM_INT32:
1430   case AMDGPU::OPERAND_REG_IMM_FP32:
1431   case AMDGPU::OPERAND_REG_INLINE_C_INT32:
1432   case AMDGPU::OPERAND_REG_INLINE_C_FP32:
1433     if (isInt<32>(Val) &&
1434         AMDGPU::isInlinableLiteral32(static_cast<int32_t>(Val),
1435                                      AsmParser->hasInv2PiInlineImm())) {
1436       Inst.addOperand(MCOperand::createImm(Val));
1437       return;
1438     }
1439 
1440     Inst.addOperand(MCOperand::createImm(Val & 0xffffffff));
1441     return;
1442 
1443   case AMDGPU::OPERAND_REG_IMM_INT64:
1444   case AMDGPU::OPERAND_REG_IMM_FP64:
1445   case AMDGPU::OPERAND_REG_INLINE_C_INT64:
1446   case AMDGPU::OPERAND_REG_INLINE_C_FP64:
1447     if (AMDGPU::isInlinableLiteral64(Val, AsmParser->hasInv2PiInlineImm())) {
1448       Inst.addOperand(MCOperand::createImm(Val));
1449       return;
1450     }
1451 
1452     Inst.addOperand(MCOperand::createImm(Lo_32(Val)));
1453     return;
1454 
1455   case AMDGPU::OPERAND_REG_IMM_INT16:
1456   case AMDGPU::OPERAND_REG_IMM_FP16:
1457   case AMDGPU::OPERAND_REG_INLINE_C_INT16:
1458   case AMDGPU::OPERAND_REG_INLINE_C_FP16:
1459     if (isInt<16>(Val) &&
1460         AMDGPU::isInlinableLiteral16(static_cast<int16_t>(Val),
1461                                      AsmParser->hasInv2PiInlineImm())) {
1462       Inst.addOperand(MCOperand::createImm(Val));
1463       return;
1464     }
1465 
1466     Inst.addOperand(MCOperand::createImm(Val & 0xffff));
1467     return;
1468 
1469   case AMDGPU::OPERAND_REG_INLINE_C_V2INT16:
1470   case AMDGPU::OPERAND_REG_INLINE_C_V2FP16: {
1471     auto LiteralVal = static_cast<uint16_t>(Literal.getLoBits(16).getZExtValue());
1472     assert(AMDGPU::isInlinableLiteral16(LiteralVal,
1473                                         AsmParser->hasInv2PiInlineImm()));
1474 
1475     uint32_t ImmVal = static_cast<uint32_t>(LiteralVal) << 16 |
1476                       static_cast<uint32_t>(LiteralVal);
1477     Inst.addOperand(MCOperand::createImm(ImmVal));
1478     return;
1479   }
1480   default:
1481     llvm_unreachable("invalid operand size");
1482   }
1483 }
1484 
1485 template <unsigned Bitwidth>
1486 void AMDGPUOperand::addKImmFPOperands(MCInst &Inst, unsigned N) const {
1487   APInt Literal(64, Imm.Val);
1488 
1489   if (!Imm.IsFPImm) {
1490     // We got int literal token.
1491     Inst.addOperand(MCOperand::createImm(Literal.getLoBits(Bitwidth).getZExtValue()));
1492     return;
1493   }
1494 
1495   bool Lost;
1496   APFloat FPLiteral(APFloat::IEEEdouble(), Literal);
1497   FPLiteral.convert(*getFltSemantics(Bitwidth / 8),
1498                     APFloat::rmNearestTiesToEven, &Lost);
1499   Inst.addOperand(MCOperand::createImm(FPLiteral.bitcastToAPInt().getZExtValue()));
1500 }
1501 
1502 void AMDGPUOperand::addRegOperands(MCInst &Inst, unsigned N) const {
1503   Inst.addOperand(MCOperand::createReg(AMDGPU::getMCReg(getReg(), AsmParser->getSTI())));
1504 }
1505 
1506 //===----------------------------------------------------------------------===//
1507 // AsmParser
1508 //===----------------------------------------------------------------------===//
1509 
1510 static int getRegClass(RegisterKind Is, unsigned RegWidth) {
1511   if (Is == IS_VGPR) {
1512     switch (RegWidth) {
1513       default: return -1;
1514       case 1: return AMDGPU::VGPR_32RegClassID;
1515       case 2: return AMDGPU::VReg_64RegClassID;
1516       case 3: return AMDGPU::VReg_96RegClassID;
1517       case 4: return AMDGPU::VReg_128RegClassID;
1518       case 8: return AMDGPU::VReg_256RegClassID;
1519       case 16: return AMDGPU::VReg_512RegClassID;
1520     }
1521   } else if (Is == IS_TTMP) {
1522     switch (RegWidth) {
1523       default: return -1;
1524       case 1: return AMDGPU::TTMP_32RegClassID;
1525       case 2: return AMDGPU::TTMP_64RegClassID;
1526       case 4: return AMDGPU::TTMP_128RegClassID;
1527       case 8: return AMDGPU::TTMP_256RegClassID;
1528       case 16: return AMDGPU::TTMP_512RegClassID;
1529     }
1530   } else if (Is == IS_SGPR) {
1531     switch (RegWidth) {
1532       default: return -1;
1533       case 1: return AMDGPU::SGPR_32RegClassID;
1534       case 2: return AMDGPU::SGPR_64RegClassID;
1535       case 4: return AMDGPU::SGPR_128RegClassID;
1536       case 8: return AMDGPU::SGPR_256RegClassID;
1537       case 16: return AMDGPU::SGPR_512RegClassID;
1538     }
1539   }
1540   return -1;
1541 }
1542 
1543 static unsigned getSpecialRegForName(StringRef RegName) {
1544   return StringSwitch<unsigned>(RegName)
1545     .Case("exec", AMDGPU::EXEC)
1546     .Case("vcc", AMDGPU::VCC)
1547     .Case("flat_scratch", AMDGPU::FLAT_SCR)
1548     .Case("xnack_mask", AMDGPU::XNACK_MASK)
1549     .Case("m0", AMDGPU::M0)
1550     .Case("scc", AMDGPU::SCC)
1551     .Case("tba", AMDGPU::TBA)
1552     .Case("tma", AMDGPU::TMA)
1553     .Case("flat_scratch_lo", AMDGPU::FLAT_SCR_LO)
1554     .Case("flat_scratch_hi", AMDGPU::FLAT_SCR_HI)
1555     .Case("xnack_mask_lo", AMDGPU::XNACK_MASK_LO)
1556     .Case("xnack_mask_hi", AMDGPU::XNACK_MASK_HI)
1557     .Case("vcc_lo", AMDGPU::VCC_LO)
1558     .Case("vcc_hi", AMDGPU::VCC_HI)
1559     .Case("exec_lo", AMDGPU::EXEC_LO)
1560     .Case("exec_hi", AMDGPU::EXEC_HI)
1561     .Case("tma_lo", AMDGPU::TMA_LO)
1562     .Case("tma_hi", AMDGPU::TMA_HI)
1563     .Case("tba_lo", AMDGPU::TBA_LO)
1564     .Case("tba_hi", AMDGPU::TBA_HI)
1565     .Default(0);
1566 }
1567 
1568 bool AMDGPUAsmParser::ParseRegister(unsigned &RegNo, SMLoc &StartLoc,
1569                                     SMLoc &EndLoc) {
1570   auto R = parseRegister();
1571   if (!R) return true;
1572   assert(R->isReg());
1573   RegNo = R->getReg();
1574   StartLoc = R->getStartLoc();
1575   EndLoc = R->getEndLoc();
1576   return false;
1577 }
1578 
1579 bool AMDGPUAsmParser::AddNextRegisterToList(unsigned &Reg, unsigned &RegWidth,
1580                                             RegisterKind RegKind, unsigned Reg1,
1581                                             unsigned RegNum) {
1582   switch (RegKind) {
1583   case IS_SPECIAL:
1584     if (Reg == AMDGPU::EXEC_LO && Reg1 == AMDGPU::EXEC_HI) {
1585       Reg = AMDGPU::EXEC;
1586       RegWidth = 2;
1587       return true;
1588     }
1589     if (Reg == AMDGPU::FLAT_SCR_LO && Reg1 == AMDGPU::FLAT_SCR_HI) {
1590       Reg = AMDGPU::FLAT_SCR;
1591       RegWidth = 2;
1592       return true;
1593     }
1594     if (Reg == AMDGPU::XNACK_MASK_LO && Reg1 == AMDGPU::XNACK_MASK_HI) {
1595       Reg = AMDGPU::XNACK_MASK;
1596       RegWidth = 2;
1597       return true;
1598     }
1599     if (Reg == AMDGPU::VCC_LO && Reg1 == AMDGPU::VCC_HI) {
1600       Reg = AMDGPU::VCC;
1601       RegWidth = 2;
1602       return true;
1603     }
1604     if (Reg == AMDGPU::TBA_LO && Reg1 == AMDGPU::TBA_HI) {
1605       Reg = AMDGPU::TBA;
1606       RegWidth = 2;
1607       return true;
1608     }
1609     if (Reg == AMDGPU::TMA_LO && Reg1 == AMDGPU::TMA_HI) {
1610       Reg = AMDGPU::TMA;
1611       RegWidth = 2;
1612       return true;
1613     }
1614     return false;
1615   case IS_VGPR:
1616   case IS_SGPR:
1617   case IS_TTMP:
1618     if (Reg1 != Reg + RegWidth) {
1619       return false;
1620     }
1621     RegWidth++;
1622     return true;
1623   default:
1624     llvm_unreachable("unexpected register kind");
1625   }
1626 }
1627 
1628 bool AMDGPUAsmParser::ParseAMDGPURegister(RegisterKind &RegKind, unsigned &Reg,
1629                                           unsigned &RegNum, unsigned &RegWidth,
1630                                           unsigned *DwordRegIndex) {
1631   if (DwordRegIndex) { *DwordRegIndex = 0; }
1632   const MCRegisterInfo *TRI = getContext().getRegisterInfo();
1633   if (getLexer().is(AsmToken::Identifier)) {
1634     StringRef RegName = Parser.getTok().getString();
1635     if ((Reg = getSpecialRegForName(RegName))) {
1636       Parser.Lex();
1637       RegKind = IS_SPECIAL;
1638     } else {
1639       unsigned RegNumIndex = 0;
1640       if (RegName[0] == 'v') {
1641         RegNumIndex = 1;
1642         RegKind = IS_VGPR;
1643       } else if (RegName[0] == 's') {
1644         RegNumIndex = 1;
1645         RegKind = IS_SGPR;
1646       } else if (RegName.startswith("ttmp")) {
1647         RegNumIndex = strlen("ttmp");
1648         RegKind = IS_TTMP;
1649       } else {
1650         return false;
1651       }
1652       if (RegName.size() > RegNumIndex) {
1653         // Single 32-bit register: vXX.
1654         if (RegName.substr(RegNumIndex).getAsInteger(10, RegNum))
1655           return false;
1656         Parser.Lex();
1657         RegWidth = 1;
1658       } else {
1659         // Range of registers: v[XX:YY]. ":YY" is optional.
1660         Parser.Lex();
1661         int64_t RegLo, RegHi;
1662         if (getLexer().isNot(AsmToken::LBrac))
1663           return false;
1664         Parser.Lex();
1665 
1666         if (getParser().parseAbsoluteExpression(RegLo))
1667           return false;
1668 
1669         const bool isRBrace = getLexer().is(AsmToken::RBrac);
1670         if (!isRBrace && getLexer().isNot(AsmToken::Colon))
1671           return false;
1672         Parser.Lex();
1673 
1674         if (isRBrace) {
1675           RegHi = RegLo;
1676         } else {
1677           if (getParser().parseAbsoluteExpression(RegHi))
1678             return false;
1679 
1680           if (getLexer().isNot(AsmToken::RBrac))
1681             return false;
1682           Parser.Lex();
1683         }
1684         RegNum = (unsigned) RegLo;
1685         RegWidth = (RegHi - RegLo) + 1;
1686       }
1687     }
1688   } else if (getLexer().is(AsmToken::LBrac)) {
1689     // List of consecutive registers: [s0,s1,s2,s3]
1690     Parser.Lex();
1691     if (!ParseAMDGPURegister(RegKind, Reg, RegNum, RegWidth, nullptr))
1692       return false;
1693     if (RegWidth != 1)
1694       return false;
1695     RegisterKind RegKind1;
1696     unsigned Reg1, RegNum1, RegWidth1;
1697     do {
1698       if (getLexer().is(AsmToken::Comma)) {
1699         Parser.Lex();
1700       } else if (getLexer().is(AsmToken::RBrac)) {
1701         Parser.Lex();
1702         break;
1703       } else if (ParseAMDGPURegister(RegKind1, Reg1, RegNum1, RegWidth1, nullptr)) {
1704         if (RegWidth1 != 1) {
1705           return false;
1706         }
1707         if (RegKind1 != RegKind) {
1708           return false;
1709         }
1710         if (!AddNextRegisterToList(Reg, RegWidth, RegKind1, Reg1, RegNum1)) {
1711           return false;
1712         }
1713       } else {
1714         return false;
1715       }
1716     } while (true);
1717   } else {
1718     return false;
1719   }
1720   switch (RegKind) {
1721   case IS_SPECIAL:
1722     RegNum = 0;
1723     RegWidth = 1;
1724     break;
1725   case IS_VGPR:
1726   case IS_SGPR:
1727   case IS_TTMP:
1728   {
1729     unsigned Size = 1;
1730     if (RegKind == IS_SGPR || RegKind == IS_TTMP) {
1731       // SGPR and TTMP registers must be aligned. Max required alignment is 4 dwords.
1732       Size = std::min(RegWidth, 4u);
1733     }
1734     if (RegNum % Size != 0)
1735       return false;
1736     if (DwordRegIndex) { *DwordRegIndex = RegNum; }
1737     RegNum = RegNum / Size;
1738     int RCID = getRegClass(RegKind, RegWidth);
1739     if (RCID == -1)
1740       return false;
1741     const MCRegisterClass RC = TRI->getRegClass(RCID);
1742     if (RegNum >= RC.getNumRegs())
1743       return false;
1744     Reg = RC.getRegister(RegNum);
1745     break;
1746   }
1747 
1748   default:
1749     llvm_unreachable("unexpected register kind");
1750   }
1751 
1752   if (!subtargetHasRegister(*TRI, Reg))
1753     return false;
1754   return true;
1755 }
1756 
1757 std::unique_ptr<AMDGPUOperand> AMDGPUAsmParser::parseRegister() {
1758   const auto &Tok = Parser.getTok();
1759   SMLoc StartLoc = Tok.getLoc();
1760   SMLoc EndLoc = Tok.getEndLoc();
1761   RegisterKind RegKind;
1762   unsigned Reg, RegNum, RegWidth, DwordRegIndex;
1763 
1764   if (!ParseAMDGPURegister(RegKind, Reg, RegNum, RegWidth, &DwordRegIndex)) {
1765     return nullptr;
1766   }
1767   KernelScope.usesRegister(RegKind, DwordRegIndex, RegWidth);
1768   return AMDGPUOperand::CreateReg(this, Reg, StartLoc, EndLoc, false);
1769 }
1770 
1771 bool
1772 AMDGPUAsmParser::parseAbsoluteExpr(int64_t &Val, bool AbsMod) {
1773   if (AbsMod && getLexer().peekTok().is(AsmToken::Pipe) &&
1774       (getLexer().getKind() == AsmToken::Integer ||
1775        getLexer().getKind() == AsmToken::Real)) {
1776     // This is a workaround for handling operands like these:
1777     //     |1.0|
1778     //     |-1|
1779     // This syntax is not compatible with syntax of standard
1780     // MC expressions (due to the trailing '|').
1781 
1782     SMLoc EndLoc;
1783     const MCExpr *Expr;
1784 
1785     if (getParser().parsePrimaryExpr(Expr, EndLoc)) {
1786       return true;
1787     }
1788 
1789     return !Expr->evaluateAsAbsolute(Val);
1790   }
1791 
1792   return getParser().parseAbsoluteExpression(Val);
1793 }
1794 
1795 OperandMatchResultTy
1796 AMDGPUAsmParser::parseImm(OperandVector &Operands, bool AbsMod) {
1797   // TODO: add syntactic sugar for 1/(2*PI)
1798   bool Minus = false;
1799   if (getLexer().getKind() == AsmToken::Minus) {
1800     const AsmToken NextToken = getLexer().peekTok();
1801     if (!NextToken.is(AsmToken::Integer) &&
1802         !NextToken.is(AsmToken::Real)) {
1803         return MatchOperand_NoMatch;
1804     }
1805     Minus = true;
1806     Parser.Lex();
1807   }
1808 
1809   SMLoc S = Parser.getTok().getLoc();
1810   switch(getLexer().getKind()) {
1811   case AsmToken::Integer: {
1812     int64_t IntVal;
1813     if (parseAbsoluteExpr(IntVal, AbsMod))
1814       return MatchOperand_ParseFail;
1815     if (Minus)
1816       IntVal *= -1;
1817     Operands.push_back(AMDGPUOperand::CreateImm(this, IntVal, S));
1818     return MatchOperand_Success;
1819   }
1820   case AsmToken::Real: {
1821     int64_t IntVal;
1822     if (parseAbsoluteExpr(IntVal, AbsMod))
1823       return MatchOperand_ParseFail;
1824 
1825     APFloat F(BitsToDouble(IntVal));
1826     if (Minus)
1827       F.changeSign();
1828     Operands.push_back(
1829         AMDGPUOperand::CreateImm(this, F.bitcastToAPInt().getZExtValue(), S,
1830                                  AMDGPUOperand::ImmTyNone, true));
1831     return MatchOperand_Success;
1832   }
1833   default:
1834     return MatchOperand_NoMatch;
1835   }
1836 }
1837 
1838 OperandMatchResultTy
1839 AMDGPUAsmParser::parseReg(OperandVector &Operands) {
1840   if (auto R = parseRegister()) {
1841     assert(R->isReg());
1842     R->Reg.IsForcedVOP3 = isForcedVOP3();
1843     Operands.push_back(std::move(R));
1844     return MatchOperand_Success;
1845   }
1846   return MatchOperand_NoMatch;
1847 }
1848 
1849 OperandMatchResultTy
1850 AMDGPUAsmParser::parseRegOrImm(OperandVector &Operands, bool AbsMod) {
1851   auto res = parseImm(Operands, AbsMod);
1852   if (res != MatchOperand_NoMatch) {
1853     return res;
1854   }
1855 
1856   return parseReg(Operands);
1857 }
1858 
1859 OperandMatchResultTy
1860 AMDGPUAsmParser::parseRegOrImmWithFPInputMods(OperandVector &Operands,
1861                                               bool AllowImm) {
1862   bool Negate = false, Negate2 = false, Abs = false, Abs2 = false;
1863 
1864   if (getLexer().getKind()== AsmToken::Minus) {
1865     const AsmToken NextToken = getLexer().peekTok();
1866 
1867     // Disable ambiguous constructs like '--1' etc. Should use neg(-1) instead.
1868     if (NextToken.is(AsmToken::Minus)) {
1869       Error(Parser.getTok().getLoc(), "invalid syntax, expected 'neg' modifier");
1870       return MatchOperand_ParseFail;
1871     }
1872 
1873     // '-' followed by an integer literal N should be interpreted as integer
1874     // negation rather than a floating-point NEG modifier applied to N.
1875     // Beside being contr-intuitive, such use of floating-point NEG modifier
1876     // results in different meaning of integer literals used with VOP1/2/C
1877     // and VOP3, for example:
1878     //    v_exp_f32_e32 v5, -1 // VOP1: src0 = 0xFFFFFFFF
1879     //    v_exp_f32_e64 v5, -1 // VOP3: src0 = 0x80000001
1880     // Negative fp literals should be handled likewise for unifomtity
1881     if (!NextToken.is(AsmToken::Integer) && !NextToken.is(AsmToken::Real)) {
1882       Parser.Lex();
1883       Negate = true;
1884     }
1885   }
1886 
1887   if (getLexer().getKind() == AsmToken::Identifier &&
1888       Parser.getTok().getString() == "neg") {
1889     if (Negate) {
1890       Error(Parser.getTok().getLoc(), "expected register or immediate");
1891       return MatchOperand_ParseFail;
1892     }
1893     Parser.Lex();
1894     Negate2 = true;
1895     if (getLexer().isNot(AsmToken::LParen)) {
1896       Error(Parser.getTok().getLoc(), "expected left paren after neg");
1897       return MatchOperand_ParseFail;
1898     }
1899     Parser.Lex();
1900   }
1901 
1902   if (getLexer().getKind() == AsmToken::Identifier &&
1903       Parser.getTok().getString() == "abs") {
1904     Parser.Lex();
1905     Abs2 = true;
1906     if (getLexer().isNot(AsmToken::LParen)) {
1907       Error(Parser.getTok().getLoc(), "expected left paren after abs");
1908       return MatchOperand_ParseFail;
1909     }
1910     Parser.Lex();
1911   }
1912 
1913   if (getLexer().getKind() == AsmToken::Pipe) {
1914     if (Abs2) {
1915       Error(Parser.getTok().getLoc(), "expected register or immediate");
1916       return MatchOperand_ParseFail;
1917     }
1918     Parser.Lex();
1919     Abs = true;
1920   }
1921 
1922   OperandMatchResultTy Res;
1923   if (AllowImm) {
1924     Res = parseRegOrImm(Operands, Abs);
1925   } else {
1926     Res = parseReg(Operands);
1927   }
1928   if (Res != MatchOperand_Success) {
1929     return Res;
1930   }
1931 
1932   AMDGPUOperand::Modifiers Mods;
1933   if (Abs) {
1934     if (getLexer().getKind() != AsmToken::Pipe) {
1935       Error(Parser.getTok().getLoc(), "expected vertical bar");
1936       return MatchOperand_ParseFail;
1937     }
1938     Parser.Lex();
1939     Mods.Abs = true;
1940   }
1941   if (Abs2) {
1942     if (getLexer().isNot(AsmToken::RParen)) {
1943       Error(Parser.getTok().getLoc(), "expected closing parentheses");
1944       return MatchOperand_ParseFail;
1945     }
1946     Parser.Lex();
1947     Mods.Abs = true;
1948   }
1949 
1950   if (Negate) {
1951     Mods.Neg = true;
1952   } else if (Negate2) {
1953     if (getLexer().isNot(AsmToken::RParen)) {
1954       Error(Parser.getTok().getLoc(), "expected closing parentheses");
1955       return MatchOperand_ParseFail;
1956     }
1957     Parser.Lex();
1958     Mods.Neg = true;
1959   }
1960 
1961   if (Mods.hasFPModifiers()) {
1962     AMDGPUOperand &Op = static_cast<AMDGPUOperand &>(*Operands.back());
1963     Op.setModifiers(Mods);
1964   }
1965   return MatchOperand_Success;
1966 }
1967 
1968 OperandMatchResultTy
1969 AMDGPUAsmParser::parseRegOrImmWithIntInputMods(OperandVector &Operands,
1970                                                bool AllowImm) {
1971   bool Sext = false;
1972 
1973   if (getLexer().getKind() == AsmToken::Identifier &&
1974       Parser.getTok().getString() == "sext") {
1975     Parser.Lex();
1976     Sext = true;
1977     if (getLexer().isNot(AsmToken::LParen)) {
1978       Error(Parser.getTok().getLoc(), "expected left paren after sext");
1979       return MatchOperand_ParseFail;
1980     }
1981     Parser.Lex();
1982   }
1983 
1984   OperandMatchResultTy Res;
1985   if (AllowImm) {
1986     Res = parseRegOrImm(Operands);
1987   } else {
1988     Res = parseReg(Operands);
1989   }
1990   if (Res != MatchOperand_Success) {
1991     return Res;
1992   }
1993 
1994   AMDGPUOperand::Modifiers Mods;
1995   if (Sext) {
1996     if (getLexer().isNot(AsmToken::RParen)) {
1997       Error(Parser.getTok().getLoc(), "expected closing parentheses");
1998       return MatchOperand_ParseFail;
1999     }
2000     Parser.Lex();
2001     Mods.Sext = true;
2002   }
2003 
2004   if (Mods.hasIntModifiers()) {
2005     AMDGPUOperand &Op = static_cast<AMDGPUOperand &>(*Operands.back());
2006     Op.setModifiers(Mods);
2007   }
2008 
2009   return MatchOperand_Success;
2010 }
2011 
2012 OperandMatchResultTy
2013 AMDGPUAsmParser::parseRegWithFPInputMods(OperandVector &Operands) {
2014   return parseRegOrImmWithFPInputMods(Operands, false);
2015 }
2016 
2017 OperandMatchResultTy
2018 AMDGPUAsmParser::parseRegWithIntInputMods(OperandVector &Operands) {
2019   return parseRegOrImmWithIntInputMods(Operands, false);
2020 }
2021 
2022 OperandMatchResultTy AMDGPUAsmParser::parseVReg32OrOff(OperandVector &Operands) {
2023   std::unique_ptr<AMDGPUOperand> Reg = parseRegister();
2024   if (Reg) {
2025     Operands.push_back(std::move(Reg));
2026     return MatchOperand_Success;
2027   }
2028 
2029   const AsmToken &Tok = Parser.getTok();
2030   if (Tok.getString() == "off") {
2031     Operands.push_back(AMDGPUOperand::CreateImm(this, 0, Tok.getLoc(),
2032                                                 AMDGPUOperand::ImmTyOff, false));
2033     Parser.Lex();
2034     return MatchOperand_Success;
2035   }
2036 
2037   return MatchOperand_NoMatch;
2038 }
2039 
2040 unsigned AMDGPUAsmParser::checkTargetMatchPredicate(MCInst &Inst) {
2041   uint64_t TSFlags = MII.get(Inst.getOpcode()).TSFlags;
2042 
2043   if ((getForcedEncodingSize() == 32 && (TSFlags & SIInstrFlags::VOP3)) ||
2044       (getForcedEncodingSize() == 64 && !(TSFlags & SIInstrFlags::VOP3)) ||
2045       (isForcedDPP() && !(TSFlags & SIInstrFlags::DPP)) ||
2046       (isForcedSDWA() && !(TSFlags & SIInstrFlags::SDWA)) )
2047     return Match_InvalidOperand;
2048 
2049   if ((TSFlags & SIInstrFlags::VOP3) &&
2050       (TSFlags & SIInstrFlags::VOPAsmPrefer32Bit) &&
2051       getForcedEncodingSize() != 64)
2052     return Match_PreferE32;
2053 
2054   if (Inst.getOpcode() == AMDGPU::V_MAC_F32_sdwa_vi ||
2055       Inst.getOpcode() == AMDGPU::V_MAC_F16_sdwa_vi) {
2056     // v_mac_f32/16 allow only dst_sel == DWORD;
2057     auto OpNum =
2058         AMDGPU::getNamedOperandIdx(Inst.getOpcode(), AMDGPU::OpName::dst_sel);
2059     const auto &Op = Inst.getOperand(OpNum);
2060     if (!Op.isImm() || Op.getImm() != AMDGPU::SDWA::SdwaSel::DWORD) {
2061       return Match_InvalidOperand;
2062     }
2063   }
2064 
2065   if ((TSFlags & SIInstrFlags::FLAT) && !hasFlatOffsets()) {
2066     // FIXME: Produces error without correct column reported.
2067     auto OpNum =
2068         AMDGPU::getNamedOperandIdx(Inst.getOpcode(), AMDGPU::OpName::offset);
2069     const auto &Op = Inst.getOperand(OpNum);
2070     if (Op.getImm() != 0)
2071       return Match_InvalidOperand;
2072   }
2073 
2074   return Match_Success;
2075 }
2076 
2077 // What asm variants we should check
2078 ArrayRef<unsigned> AMDGPUAsmParser::getMatchedVariants() const {
2079   if (getForcedEncodingSize() == 32) {
2080     static const unsigned Variants[] = {AMDGPUAsmVariants::DEFAULT};
2081     return makeArrayRef(Variants);
2082   }
2083 
2084   if (isForcedVOP3()) {
2085     static const unsigned Variants[] = {AMDGPUAsmVariants::VOP3};
2086     return makeArrayRef(Variants);
2087   }
2088 
2089   if (isForcedSDWA()) {
2090     static const unsigned Variants[] = {AMDGPUAsmVariants::SDWA,
2091                                         AMDGPUAsmVariants::SDWA9};
2092     return makeArrayRef(Variants);
2093   }
2094 
2095   if (isForcedDPP()) {
2096     static const unsigned Variants[] = {AMDGPUAsmVariants::DPP};
2097     return makeArrayRef(Variants);
2098   }
2099 
2100   static const unsigned Variants[] = {
2101     AMDGPUAsmVariants::DEFAULT, AMDGPUAsmVariants::VOP3,
2102     AMDGPUAsmVariants::SDWA, AMDGPUAsmVariants::SDWA9, AMDGPUAsmVariants::DPP
2103   };
2104 
2105   return makeArrayRef(Variants);
2106 }
2107 
2108 unsigned AMDGPUAsmParser::findImplicitSGPRReadInVOP(const MCInst &Inst) const {
2109   const MCInstrDesc &Desc = MII.get(Inst.getOpcode());
2110   const unsigned Num = Desc.getNumImplicitUses();
2111   for (unsigned i = 0; i < Num; ++i) {
2112     unsigned Reg = Desc.ImplicitUses[i];
2113     switch (Reg) {
2114     case AMDGPU::FLAT_SCR:
2115     case AMDGPU::VCC:
2116     case AMDGPU::M0:
2117       return Reg;
2118     default:
2119       break;
2120     }
2121   }
2122   return AMDGPU::NoRegister;
2123 }
2124 
2125 // NB: This code is correct only when used to check constant
2126 // bus limitations because GFX7 support no f16 inline constants.
2127 // Note that there are no cases when a GFX7 opcode violates
2128 // constant bus limitations due to the use of an f16 constant.
2129 bool AMDGPUAsmParser::isInlineConstant(const MCInst &Inst,
2130                                        unsigned OpIdx) const {
2131   const MCInstrDesc &Desc = MII.get(Inst.getOpcode());
2132 
2133   if (!AMDGPU::isSISrcOperand(Desc, OpIdx)) {
2134     return false;
2135   }
2136 
2137   const MCOperand &MO = Inst.getOperand(OpIdx);
2138 
2139   int64_t Val = MO.getImm();
2140   auto OpSize = AMDGPU::getOperandSize(Desc, OpIdx);
2141 
2142   switch (OpSize) { // expected operand size
2143   case 8:
2144     return AMDGPU::isInlinableLiteral64(Val, hasInv2PiInlineImm());
2145   case 4:
2146     return AMDGPU::isInlinableLiteral32(Val, hasInv2PiInlineImm());
2147   case 2: {
2148     const unsigned OperandType = Desc.OpInfo[OpIdx].OperandType;
2149     if (OperandType == AMDGPU::OPERAND_REG_INLINE_C_V2INT16 ||
2150         OperandType == AMDGPU::OPERAND_REG_INLINE_C_V2FP16) {
2151       return AMDGPU::isInlinableLiteralV216(Val, hasInv2PiInlineImm());
2152     } else {
2153       return AMDGPU::isInlinableLiteral16(Val, hasInv2PiInlineImm());
2154     }
2155   }
2156   default:
2157     llvm_unreachable("invalid operand size");
2158   }
2159 }
2160 
2161 bool AMDGPUAsmParser::usesConstantBus(const MCInst &Inst, unsigned OpIdx) {
2162   const MCOperand &MO = Inst.getOperand(OpIdx);
2163   if (MO.isImm()) {
2164     return !isInlineConstant(Inst, OpIdx);
2165   }
2166   return !MO.isReg() ||
2167          isSGPR(mc2PseudoReg(MO.getReg()), getContext().getRegisterInfo());
2168 }
2169 
2170 bool AMDGPUAsmParser::validateConstantBusLimitations(const MCInst &Inst) {
2171   const unsigned Opcode = Inst.getOpcode();
2172   const MCInstrDesc &Desc = MII.get(Opcode);
2173   unsigned ConstantBusUseCount = 0;
2174 
2175   if (Desc.TSFlags &
2176       (SIInstrFlags::VOPC |
2177        SIInstrFlags::VOP1 | SIInstrFlags::VOP2 |
2178        SIInstrFlags::VOP3 | SIInstrFlags::VOP3P |
2179        SIInstrFlags::SDWA)) {
2180     // Check special imm operands (used by madmk, etc)
2181     if (AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::imm) != -1) {
2182       ++ConstantBusUseCount;
2183     }
2184 
2185     unsigned SGPRUsed = findImplicitSGPRReadInVOP(Inst);
2186     if (SGPRUsed != AMDGPU::NoRegister) {
2187       ++ConstantBusUseCount;
2188     }
2189 
2190     const int Src0Idx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::src0);
2191     const int Src1Idx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::src1);
2192     const int Src2Idx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::src2);
2193 
2194     const int OpIndices[] = { Src0Idx, Src1Idx, Src2Idx };
2195 
2196     for (int OpIdx : OpIndices) {
2197       if (OpIdx == -1) break;
2198 
2199       const MCOperand &MO = Inst.getOperand(OpIdx);
2200       if (usesConstantBus(Inst, OpIdx)) {
2201         if (MO.isReg()) {
2202           const unsigned Reg = mc2PseudoReg(MO.getReg());
2203           // Pairs of registers with a partial intersections like these
2204           //   s0, s[0:1]
2205           //   flat_scratch_lo, flat_scratch
2206           //   flat_scratch_lo, flat_scratch_hi
2207           // are theoretically valid but they are disabled anyway.
2208           // Note that this code mimics SIInstrInfo::verifyInstruction
2209           if (Reg != SGPRUsed) {
2210             ++ConstantBusUseCount;
2211           }
2212           SGPRUsed = Reg;
2213         } else { // Expression or a literal
2214           ++ConstantBusUseCount;
2215         }
2216       }
2217     }
2218   }
2219 
2220   return ConstantBusUseCount <= 1;
2221 }
2222 
2223 bool AMDGPUAsmParser::validateEarlyClobberLimitations(const MCInst &Inst) {
2224   const unsigned Opcode = Inst.getOpcode();
2225   const MCInstrDesc &Desc = MII.get(Opcode);
2226 
2227   const int DstIdx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::vdst);
2228   if (DstIdx == -1 ||
2229       Desc.getOperandConstraint(DstIdx, MCOI::EARLY_CLOBBER) == -1) {
2230     return true;
2231   }
2232 
2233   const MCRegisterInfo *TRI = getContext().getRegisterInfo();
2234 
2235   const int Src0Idx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::src0);
2236   const int Src1Idx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::src1);
2237   const int Src2Idx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::src2);
2238 
2239   assert(DstIdx != -1);
2240   const MCOperand &Dst = Inst.getOperand(DstIdx);
2241   assert(Dst.isReg());
2242   const unsigned DstReg = mc2PseudoReg(Dst.getReg());
2243 
2244   const int SrcIndices[] = { Src0Idx, Src1Idx, Src2Idx };
2245 
2246   for (int SrcIdx : SrcIndices) {
2247     if (SrcIdx == -1) break;
2248     const MCOperand &Src = Inst.getOperand(SrcIdx);
2249     if (Src.isReg()) {
2250       const unsigned SrcReg = mc2PseudoReg(Src.getReg());
2251       if (isRegIntersect(DstReg, SrcReg, TRI)) {
2252         return false;
2253       }
2254     }
2255   }
2256 
2257   return true;
2258 }
2259 
2260 bool AMDGPUAsmParser::validateIntClampSupported(const MCInst &Inst) {
2261 
2262   const unsigned Opc = Inst.getOpcode();
2263   const MCInstrDesc &Desc = MII.get(Opc);
2264 
2265   if ((Desc.TSFlags & SIInstrFlags::IntClamp) != 0 && !hasIntClamp()) {
2266     int ClampIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::clamp);
2267     assert(ClampIdx != -1);
2268     return Inst.getOperand(ClampIdx).getImm() == 0;
2269   }
2270 
2271   return true;
2272 }
2273 
2274 bool AMDGPUAsmParser::validateInstruction(const MCInst &Inst,
2275                                           const SMLoc &IDLoc) {
2276   if (!validateConstantBusLimitations(Inst)) {
2277     Error(IDLoc,
2278       "invalid operand (violates constant bus restrictions)");
2279     return false;
2280   }
2281   if (!validateEarlyClobberLimitations(Inst)) {
2282     Error(IDLoc,
2283       "destination must be different than all sources");
2284     return false;
2285   }
2286   if (!validateIntClampSupported(Inst)) {
2287     Error(IDLoc,
2288       "integer clamping is not supported on this GPU");
2289     return false;
2290   }
2291 
2292   return true;
2293 }
2294 
2295 static std::string AMDGPUMnemonicSpellCheck(StringRef S, uint64_t FBS,
2296                                             unsigned VariantID = 0);
2297 
2298 bool AMDGPUAsmParser::MatchAndEmitInstruction(SMLoc IDLoc, unsigned &Opcode,
2299                                               OperandVector &Operands,
2300                                               MCStreamer &Out,
2301                                               uint64_t &ErrorInfo,
2302                                               bool MatchingInlineAsm) {
2303   MCInst Inst;
2304   unsigned Result = Match_Success;
2305   for (auto Variant : getMatchedVariants()) {
2306     uint64_t EI;
2307     auto R = MatchInstructionImpl(Operands, Inst, EI, MatchingInlineAsm,
2308                                   Variant);
2309     // We order match statuses from least to most specific. We use most specific
2310     // status as resulting
2311     // Match_MnemonicFail < Match_InvalidOperand < Match_MissingFeature < Match_PreferE32
2312     if ((R == Match_Success) ||
2313         (R == Match_PreferE32) ||
2314         (R == Match_MissingFeature && Result != Match_PreferE32) ||
2315         (R == Match_InvalidOperand && Result != Match_MissingFeature
2316                                    && Result != Match_PreferE32) ||
2317         (R == Match_MnemonicFail   && Result != Match_InvalidOperand
2318                                    && Result != Match_MissingFeature
2319                                    && Result != Match_PreferE32)) {
2320       Result = R;
2321       ErrorInfo = EI;
2322     }
2323     if (R == Match_Success)
2324       break;
2325   }
2326 
2327   switch (Result) {
2328   default: break;
2329   case Match_Success:
2330     if (!validateInstruction(Inst, IDLoc)) {
2331       return true;
2332     }
2333     Inst.setLoc(IDLoc);
2334     Out.EmitInstruction(Inst, getSTI());
2335     return false;
2336 
2337   case Match_MissingFeature:
2338     return Error(IDLoc, "instruction not supported on this GPU");
2339 
2340   case Match_MnemonicFail: {
2341     uint64_t FBS = ComputeAvailableFeatures(getSTI().getFeatureBits());
2342     std::string Suggestion = AMDGPUMnemonicSpellCheck(
2343         ((AMDGPUOperand &)*Operands[0]).getToken(), FBS);
2344     return Error(IDLoc, "invalid instruction" + Suggestion,
2345                  ((AMDGPUOperand &)*Operands[0]).getLocRange());
2346   }
2347 
2348   case Match_InvalidOperand: {
2349     SMLoc ErrorLoc = IDLoc;
2350     if (ErrorInfo != ~0ULL) {
2351       if (ErrorInfo >= Operands.size()) {
2352         return Error(IDLoc, "too few operands for instruction");
2353       }
2354       ErrorLoc = ((AMDGPUOperand &)*Operands[ErrorInfo]).getStartLoc();
2355       if (ErrorLoc == SMLoc())
2356         ErrorLoc = IDLoc;
2357     }
2358     return Error(ErrorLoc, "invalid operand for instruction");
2359   }
2360 
2361   case Match_PreferE32:
2362     return Error(IDLoc, "internal error: instruction without _e64 suffix "
2363                         "should be encoded as e32");
2364   }
2365   llvm_unreachable("Implement any new match types added!");
2366 }
2367 
2368 bool AMDGPUAsmParser::ParseAsAbsoluteExpression(uint32_t &Ret) {
2369   int64_t Tmp = -1;
2370   if (getLexer().isNot(AsmToken::Integer) && getLexer().isNot(AsmToken::Identifier)) {
2371     return true;
2372   }
2373   if (getParser().parseAbsoluteExpression(Tmp)) {
2374     return true;
2375   }
2376   Ret = static_cast<uint32_t>(Tmp);
2377   return false;
2378 }
2379 
2380 bool AMDGPUAsmParser::ParseDirectiveMajorMinor(uint32_t &Major,
2381                                                uint32_t &Minor) {
2382   if (ParseAsAbsoluteExpression(Major))
2383     return TokError("invalid major version");
2384 
2385   if (getLexer().isNot(AsmToken::Comma))
2386     return TokError("minor version number required, comma expected");
2387   Lex();
2388 
2389   if (ParseAsAbsoluteExpression(Minor))
2390     return TokError("invalid minor version");
2391 
2392   return false;
2393 }
2394 
2395 bool AMDGPUAsmParser::ParseDirectiveHSACodeObjectVersion() {
2396   uint32_t Major;
2397   uint32_t Minor;
2398 
2399   if (ParseDirectiveMajorMinor(Major, Minor))
2400     return true;
2401 
2402   getTargetStreamer().EmitDirectiveHSACodeObjectVersion(Major, Minor);
2403   return false;
2404 }
2405 
2406 bool AMDGPUAsmParser::ParseDirectiveHSACodeObjectISA() {
2407   uint32_t Major;
2408   uint32_t Minor;
2409   uint32_t Stepping;
2410   StringRef VendorName;
2411   StringRef ArchName;
2412 
2413   // If this directive has no arguments, then use the ISA version for the
2414   // targeted GPU.
2415   if (getLexer().is(AsmToken::EndOfStatement)) {
2416     AMDGPU::IsaInfo::IsaVersion ISA =
2417         AMDGPU::IsaInfo::getIsaVersion(getFeatureBits());
2418     getTargetStreamer().EmitDirectiveHSACodeObjectISA(ISA.Major, ISA.Minor,
2419                                                       ISA.Stepping,
2420                                                       "AMD", "AMDGPU");
2421     return false;
2422   }
2423 
2424   if (ParseDirectiveMajorMinor(Major, Minor))
2425     return true;
2426 
2427   if (getLexer().isNot(AsmToken::Comma))
2428     return TokError("stepping version number required, comma expected");
2429   Lex();
2430 
2431   if (ParseAsAbsoluteExpression(Stepping))
2432     return TokError("invalid stepping version");
2433 
2434   if (getLexer().isNot(AsmToken::Comma))
2435     return TokError("vendor name required, comma expected");
2436   Lex();
2437 
2438   if (getLexer().isNot(AsmToken::String))
2439     return TokError("invalid vendor name");
2440 
2441   VendorName = getLexer().getTok().getStringContents();
2442   Lex();
2443 
2444   if (getLexer().isNot(AsmToken::Comma))
2445     return TokError("arch name required, comma expected");
2446   Lex();
2447 
2448   if (getLexer().isNot(AsmToken::String))
2449     return TokError("invalid arch name");
2450 
2451   ArchName = getLexer().getTok().getStringContents();
2452   Lex();
2453 
2454   getTargetStreamer().EmitDirectiveHSACodeObjectISA(Major, Minor, Stepping,
2455                                                     VendorName, ArchName);
2456   return false;
2457 }
2458 
2459 bool AMDGPUAsmParser::ParseAMDKernelCodeTValue(StringRef ID,
2460                                                amd_kernel_code_t &Header) {
2461   SmallString<40> ErrStr;
2462   raw_svector_ostream Err(ErrStr);
2463   if (!parseAmdKernelCodeField(ID, getParser(), Header, Err)) {
2464     return TokError(Err.str());
2465   }
2466   Lex();
2467   return false;
2468 }
2469 
2470 bool AMDGPUAsmParser::ParseDirectiveAMDKernelCodeT() {
2471   amd_kernel_code_t Header;
2472   AMDGPU::initDefaultAMDKernelCodeT(Header, getFeatureBits());
2473 
2474   while (true) {
2475     // Lex EndOfStatement.  This is in a while loop, because lexing a comment
2476     // will set the current token to EndOfStatement.
2477     while(getLexer().is(AsmToken::EndOfStatement))
2478       Lex();
2479 
2480     if (getLexer().isNot(AsmToken::Identifier))
2481       return TokError("expected value identifier or .end_amd_kernel_code_t");
2482 
2483     StringRef ID = getLexer().getTok().getIdentifier();
2484     Lex();
2485 
2486     if (ID == ".end_amd_kernel_code_t")
2487       break;
2488 
2489     if (ParseAMDKernelCodeTValue(ID, Header))
2490       return true;
2491   }
2492 
2493   getTargetStreamer().EmitAMDKernelCodeT(Header);
2494 
2495   return false;
2496 }
2497 
2498 bool AMDGPUAsmParser::ParseDirectiveAMDGPUHsaKernel() {
2499   if (getLexer().isNot(AsmToken::Identifier))
2500     return TokError("expected symbol name");
2501 
2502   StringRef KernelName = Parser.getTok().getString();
2503 
2504   getTargetStreamer().EmitAMDGPUSymbolType(KernelName,
2505                                            ELF::STT_AMDGPU_HSA_KERNEL);
2506   Lex();
2507   KernelScope.initialize(getContext());
2508   return false;
2509 }
2510 
2511 bool AMDGPUAsmParser::ParseDirectiveISAVersion() {
2512   if (getSTI().getTargetTriple().getArch() != Triple::amdgcn) {
2513     return Error(getParser().getTok().getLoc(),
2514                  ".amd_amdgpu_isa directive is not available on non-amdgcn "
2515                  "architectures");
2516   }
2517 
2518   auto ISAVersionStringFromASM = getLexer().getTok().getStringContents();
2519 
2520   std::string ISAVersionStringFromSTI;
2521   raw_string_ostream ISAVersionStreamFromSTI(ISAVersionStringFromSTI);
2522   IsaInfo::streamIsaVersion(&getSTI(), ISAVersionStreamFromSTI);
2523 
2524   if (ISAVersionStringFromASM != ISAVersionStreamFromSTI.str()) {
2525     return Error(getParser().getTok().getLoc(),
2526                  ".amd_amdgpu_isa directive does not match triple and/or mcpu "
2527                  "arguments specified through the command line");
2528   }
2529 
2530   getTargetStreamer().EmitISAVersion(ISAVersionStreamFromSTI.str());
2531   Lex();
2532 
2533   return false;
2534 }
2535 
2536 bool AMDGPUAsmParser::ParseDirectiveHSAMetadata() {
2537   if (getSTI().getTargetTriple().getOS() != Triple::AMDHSA) {
2538     return Error(getParser().getTok().getLoc(),
2539                  (Twine(HSAMD::AssemblerDirectiveBegin) + Twine(" directive is "
2540                  "not available on non-amdhsa OSes")).str());
2541   }
2542 
2543   std::string HSAMetadataString;
2544   raw_string_ostream YamlStream(HSAMetadataString);
2545 
2546   getLexer().setSkipSpace(false);
2547 
2548   bool FoundEnd = false;
2549   while (!getLexer().is(AsmToken::Eof)) {
2550     while (getLexer().is(AsmToken::Space)) {
2551       YamlStream << getLexer().getTok().getString();
2552       Lex();
2553     }
2554 
2555     if (getLexer().is(AsmToken::Identifier)) {
2556       StringRef ID = getLexer().getTok().getIdentifier();
2557       if (ID == AMDGPU::HSAMD::AssemblerDirectiveEnd) {
2558         Lex();
2559         FoundEnd = true;
2560         break;
2561       }
2562     }
2563 
2564     YamlStream << Parser.parseStringToEndOfStatement()
2565                << getContext().getAsmInfo()->getSeparatorString();
2566 
2567     Parser.eatToEndOfStatement();
2568   }
2569 
2570   getLexer().setSkipSpace(true);
2571 
2572   if (getLexer().is(AsmToken::Eof) && !FoundEnd) {
2573     return TokError(Twine("expected directive ") +
2574                     Twine(HSAMD::AssemblerDirectiveEnd) + Twine(" not found"));
2575   }
2576 
2577   YamlStream.flush();
2578 
2579   if (!getTargetStreamer().EmitHSAMetadata(HSAMetadataString))
2580     return Error(getParser().getTok().getLoc(), "invalid HSA metadata");
2581 
2582   return false;
2583 }
2584 
2585 bool AMDGPUAsmParser::ParseDirectivePALMetadata() {
2586   if (getSTI().getTargetTriple().getOS() != Triple::AMDPAL) {
2587     return Error(getParser().getTok().getLoc(),
2588                  (Twine(PALMD::AssemblerDirective) + Twine(" directive is "
2589                  "not available on non-amdpal OSes")).str());
2590   }
2591 
2592   PALMD::Metadata PALMetadata;
2593   for (;;) {
2594     uint32_t Value;
2595     if (ParseAsAbsoluteExpression(Value)) {
2596       return TokError(Twine("invalid value in ") +
2597                       Twine(PALMD::AssemblerDirective));
2598     }
2599     PALMetadata.push_back(Value);
2600     if (getLexer().isNot(AsmToken::Comma))
2601       break;
2602     Lex();
2603   }
2604   getTargetStreamer().EmitPALMetadata(PALMetadata);
2605   return false;
2606 }
2607 
2608 bool AMDGPUAsmParser::ParseDirective(AsmToken DirectiveID) {
2609   StringRef IDVal = DirectiveID.getString();
2610 
2611   if (IDVal == ".hsa_code_object_version")
2612     return ParseDirectiveHSACodeObjectVersion();
2613 
2614   if (IDVal == ".hsa_code_object_isa")
2615     return ParseDirectiveHSACodeObjectISA();
2616 
2617   if (IDVal == ".amd_kernel_code_t")
2618     return ParseDirectiveAMDKernelCodeT();
2619 
2620   if (IDVal == ".amdgpu_hsa_kernel")
2621     return ParseDirectiveAMDGPUHsaKernel();
2622 
2623   if (IDVal == ".amd_amdgpu_isa")
2624     return ParseDirectiveISAVersion();
2625 
2626   if (IDVal == AMDGPU::HSAMD::AssemblerDirectiveBegin)
2627     return ParseDirectiveHSAMetadata();
2628 
2629   if (IDVal == PALMD::AssemblerDirective)
2630     return ParseDirectivePALMetadata();
2631 
2632   return true;
2633 }
2634 
2635 bool AMDGPUAsmParser::subtargetHasRegister(const MCRegisterInfo &MRI,
2636                                            unsigned RegNo) const {
2637 
2638   for (MCRegAliasIterator R(AMDGPU::TTMP12_TTMP13_TTMP14_TTMP15, &MRI, true);
2639        R.isValid(); ++R) {
2640     if (*R == RegNo)
2641       return isGFX9();
2642   }
2643 
2644   switch (RegNo) {
2645   case AMDGPU::TBA:
2646   case AMDGPU::TBA_LO:
2647   case AMDGPU::TBA_HI:
2648   case AMDGPU::TMA:
2649   case AMDGPU::TMA_LO:
2650   case AMDGPU::TMA_HI:
2651     return !isGFX9();
2652   case AMDGPU::XNACK_MASK:
2653   case AMDGPU::XNACK_MASK_LO:
2654   case AMDGPU::XNACK_MASK_HI:
2655     return !isCI() && !isSI() && hasXNACK();
2656   default:
2657     break;
2658   }
2659 
2660   if (isCI())
2661     return true;
2662 
2663   if (isSI()) {
2664     // No flat_scr
2665     switch (RegNo) {
2666     case AMDGPU::FLAT_SCR:
2667     case AMDGPU::FLAT_SCR_LO:
2668     case AMDGPU::FLAT_SCR_HI:
2669       return false;
2670     default:
2671       return true;
2672     }
2673   }
2674 
2675   // VI only has 102 SGPRs, so make sure we aren't trying to use the 2 more that
2676   // SI/CI have.
2677   for (MCRegAliasIterator R(AMDGPU::SGPR102_SGPR103, &MRI, true);
2678        R.isValid(); ++R) {
2679     if (*R == RegNo)
2680       return false;
2681   }
2682 
2683   return true;
2684 }
2685 
2686 OperandMatchResultTy
2687 AMDGPUAsmParser::parseOperand(OperandVector &Operands, StringRef Mnemonic) {
2688   // Try to parse with a custom parser
2689   OperandMatchResultTy ResTy = MatchOperandParserImpl(Operands, Mnemonic);
2690 
2691   // If we successfully parsed the operand or if there as an error parsing,
2692   // we are done.
2693   //
2694   // If we are parsing after we reach EndOfStatement then this means we
2695   // are appending default values to the Operands list.  This is only done
2696   // by custom parser, so we shouldn't continue on to the generic parsing.
2697   if (ResTy == MatchOperand_Success || ResTy == MatchOperand_ParseFail ||
2698       getLexer().is(AsmToken::EndOfStatement))
2699     return ResTy;
2700 
2701   ResTy = parseRegOrImm(Operands);
2702 
2703   if (ResTy == MatchOperand_Success)
2704     return ResTy;
2705 
2706   const auto &Tok = Parser.getTok();
2707   SMLoc S = Tok.getLoc();
2708 
2709   const MCExpr *Expr = nullptr;
2710   if (!Parser.parseExpression(Expr)) {
2711     Operands.push_back(AMDGPUOperand::CreateExpr(this, Expr, S));
2712     return MatchOperand_Success;
2713   }
2714 
2715   // Possibly this is an instruction flag like 'gds'.
2716   if (Tok.getKind() == AsmToken::Identifier) {
2717     Operands.push_back(AMDGPUOperand::CreateToken(this, Tok.getString(), S));
2718     Parser.Lex();
2719     return MatchOperand_Success;
2720   }
2721 
2722   return MatchOperand_NoMatch;
2723 }
2724 
2725 StringRef AMDGPUAsmParser::parseMnemonicSuffix(StringRef Name) {
2726   // Clear any forced encodings from the previous instruction.
2727   setForcedEncodingSize(0);
2728   setForcedDPP(false);
2729   setForcedSDWA(false);
2730 
2731   if (Name.endswith("_e64")) {
2732     setForcedEncodingSize(64);
2733     return Name.substr(0, Name.size() - 4);
2734   } else if (Name.endswith("_e32")) {
2735     setForcedEncodingSize(32);
2736     return Name.substr(0, Name.size() - 4);
2737   } else if (Name.endswith("_dpp")) {
2738     setForcedDPP(true);
2739     return Name.substr(0, Name.size() - 4);
2740   } else if (Name.endswith("_sdwa")) {
2741     setForcedSDWA(true);
2742     return Name.substr(0, Name.size() - 5);
2743   }
2744   return Name;
2745 }
2746 
2747 bool AMDGPUAsmParser::ParseInstruction(ParseInstructionInfo &Info,
2748                                        StringRef Name,
2749                                        SMLoc NameLoc, OperandVector &Operands) {
2750   // Add the instruction mnemonic
2751   Name = parseMnemonicSuffix(Name);
2752   Operands.push_back(AMDGPUOperand::CreateToken(this, Name, NameLoc));
2753 
2754   while (!getLexer().is(AsmToken::EndOfStatement)) {
2755     OperandMatchResultTy Res = parseOperand(Operands, Name);
2756 
2757     // Eat the comma or space if there is one.
2758     if (getLexer().is(AsmToken::Comma))
2759       Parser.Lex();
2760 
2761     switch (Res) {
2762       case MatchOperand_Success: break;
2763       case MatchOperand_ParseFail:
2764         Error(getLexer().getLoc(), "failed parsing operand.");
2765         while (!getLexer().is(AsmToken::EndOfStatement)) {
2766           Parser.Lex();
2767         }
2768         return true;
2769       case MatchOperand_NoMatch:
2770         Error(getLexer().getLoc(), "not a valid operand.");
2771         while (!getLexer().is(AsmToken::EndOfStatement)) {
2772           Parser.Lex();
2773         }
2774         return true;
2775     }
2776   }
2777 
2778   return false;
2779 }
2780 
2781 //===----------------------------------------------------------------------===//
2782 // Utility functions
2783 //===----------------------------------------------------------------------===//
2784 
2785 OperandMatchResultTy
2786 AMDGPUAsmParser::parseIntWithPrefix(const char *Prefix, int64_t &Int) {
2787   switch(getLexer().getKind()) {
2788     default: return MatchOperand_NoMatch;
2789     case AsmToken::Identifier: {
2790       StringRef Name = Parser.getTok().getString();
2791       if (!Name.equals(Prefix)) {
2792         return MatchOperand_NoMatch;
2793       }
2794 
2795       Parser.Lex();
2796       if (getLexer().isNot(AsmToken::Colon))
2797         return MatchOperand_ParseFail;
2798 
2799       Parser.Lex();
2800 
2801       bool IsMinus = false;
2802       if (getLexer().getKind() == AsmToken::Minus) {
2803         Parser.Lex();
2804         IsMinus = true;
2805       }
2806 
2807       if (getLexer().isNot(AsmToken::Integer))
2808         return MatchOperand_ParseFail;
2809 
2810       if (getParser().parseAbsoluteExpression(Int))
2811         return MatchOperand_ParseFail;
2812 
2813       if (IsMinus)
2814         Int = -Int;
2815       break;
2816     }
2817   }
2818   return MatchOperand_Success;
2819 }
2820 
2821 OperandMatchResultTy
2822 AMDGPUAsmParser::parseIntWithPrefix(const char *Prefix, OperandVector &Operands,
2823                                     AMDGPUOperand::ImmTy ImmTy,
2824                                     bool (*ConvertResult)(int64_t&)) {
2825   SMLoc S = Parser.getTok().getLoc();
2826   int64_t Value = 0;
2827 
2828   OperandMatchResultTy Res = parseIntWithPrefix(Prefix, Value);
2829   if (Res != MatchOperand_Success)
2830     return Res;
2831 
2832   if (ConvertResult && !ConvertResult(Value)) {
2833     return MatchOperand_ParseFail;
2834   }
2835 
2836   Operands.push_back(AMDGPUOperand::CreateImm(this, Value, S, ImmTy));
2837   return MatchOperand_Success;
2838 }
2839 
2840 OperandMatchResultTy AMDGPUAsmParser::parseOperandArrayWithPrefix(
2841   const char *Prefix,
2842   OperandVector &Operands,
2843   AMDGPUOperand::ImmTy ImmTy,
2844   bool (*ConvertResult)(int64_t&)) {
2845   StringRef Name = Parser.getTok().getString();
2846   if (!Name.equals(Prefix))
2847     return MatchOperand_NoMatch;
2848 
2849   Parser.Lex();
2850   if (getLexer().isNot(AsmToken::Colon))
2851     return MatchOperand_ParseFail;
2852 
2853   Parser.Lex();
2854   if (getLexer().isNot(AsmToken::LBrac))
2855     return MatchOperand_ParseFail;
2856   Parser.Lex();
2857 
2858   unsigned Val = 0;
2859   SMLoc S = Parser.getTok().getLoc();
2860 
2861   // FIXME: How to verify the number of elements matches the number of src
2862   // operands?
2863   for (int I = 0; I < 4; ++I) {
2864     if (I != 0) {
2865       if (getLexer().is(AsmToken::RBrac))
2866         break;
2867 
2868       if (getLexer().isNot(AsmToken::Comma))
2869         return MatchOperand_ParseFail;
2870       Parser.Lex();
2871     }
2872 
2873     if (getLexer().isNot(AsmToken::Integer))
2874       return MatchOperand_ParseFail;
2875 
2876     int64_t Op;
2877     if (getParser().parseAbsoluteExpression(Op))
2878       return MatchOperand_ParseFail;
2879 
2880     if (Op != 0 && Op != 1)
2881       return MatchOperand_ParseFail;
2882     Val |= (Op << I);
2883   }
2884 
2885   Parser.Lex();
2886   Operands.push_back(AMDGPUOperand::CreateImm(this, Val, S, ImmTy));
2887   return MatchOperand_Success;
2888 }
2889 
2890 OperandMatchResultTy
2891 AMDGPUAsmParser::parseNamedBit(const char *Name, OperandVector &Operands,
2892                                AMDGPUOperand::ImmTy ImmTy) {
2893   int64_t Bit = 0;
2894   SMLoc S = Parser.getTok().getLoc();
2895 
2896   // We are at the end of the statement, and this is a default argument, so
2897   // use a default value.
2898   if (getLexer().isNot(AsmToken::EndOfStatement)) {
2899     switch(getLexer().getKind()) {
2900       case AsmToken::Identifier: {
2901         StringRef Tok = Parser.getTok().getString();
2902         if (Tok == Name) {
2903           Bit = 1;
2904           Parser.Lex();
2905         } else if (Tok.startswith("no") && Tok.endswith(Name)) {
2906           Bit = 0;
2907           Parser.Lex();
2908         } else {
2909           return MatchOperand_NoMatch;
2910         }
2911         break;
2912       }
2913       default:
2914         return MatchOperand_NoMatch;
2915     }
2916   }
2917 
2918   Operands.push_back(AMDGPUOperand::CreateImm(this, Bit, S, ImmTy));
2919   return MatchOperand_Success;
2920 }
2921 
2922 static void addOptionalImmOperand(
2923   MCInst& Inst, const OperandVector& Operands,
2924   AMDGPUAsmParser::OptionalImmIndexMap& OptionalIdx,
2925   AMDGPUOperand::ImmTy ImmT,
2926   int64_t Default = 0) {
2927   auto i = OptionalIdx.find(ImmT);
2928   if (i != OptionalIdx.end()) {
2929     unsigned Idx = i->second;
2930     ((AMDGPUOperand &)*Operands[Idx]).addImmOperands(Inst, 1);
2931   } else {
2932     Inst.addOperand(MCOperand::createImm(Default));
2933   }
2934 }
2935 
2936 OperandMatchResultTy
2937 AMDGPUAsmParser::parseStringWithPrefix(StringRef Prefix, StringRef &Value) {
2938   if (getLexer().isNot(AsmToken::Identifier)) {
2939     return MatchOperand_NoMatch;
2940   }
2941   StringRef Tok = Parser.getTok().getString();
2942   if (Tok != Prefix) {
2943     return MatchOperand_NoMatch;
2944   }
2945 
2946   Parser.Lex();
2947   if (getLexer().isNot(AsmToken::Colon)) {
2948     return MatchOperand_ParseFail;
2949   }
2950 
2951   Parser.Lex();
2952   if (getLexer().isNot(AsmToken::Identifier)) {
2953     return MatchOperand_ParseFail;
2954   }
2955 
2956   Value = Parser.getTok().getString();
2957   return MatchOperand_Success;
2958 }
2959 
2960 //===----------------------------------------------------------------------===//
2961 // ds
2962 //===----------------------------------------------------------------------===//
2963 
2964 void AMDGPUAsmParser::cvtDSOffset01(MCInst &Inst,
2965                                     const OperandVector &Operands) {
2966   OptionalImmIndexMap OptionalIdx;
2967 
2968   for (unsigned i = 1, e = Operands.size(); i != e; ++i) {
2969     AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[i]);
2970 
2971     // Add the register arguments
2972     if (Op.isReg()) {
2973       Op.addRegOperands(Inst, 1);
2974       continue;
2975     }
2976 
2977     // Handle optional arguments
2978     OptionalIdx[Op.getImmTy()] = i;
2979   }
2980 
2981   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyOffset0);
2982   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyOffset1);
2983   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyGDS);
2984 
2985   Inst.addOperand(MCOperand::createReg(AMDGPU::M0)); // m0
2986 }
2987 
2988 void AMDGPUAsmParser::cvtDSImpl(MCInst &Inst, const OperandVector &Operands,
2989                                 bool IsGdsHardcoded) {
2990   OptionalImmIndexMap OptionalIdx;
2991 
2992   for (unsigned i = 1, e = Operands.size(); i != e; ++i) {
2993     AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[i]);
2994 
2995     // Add the register arguments
2996     if (Op.isReg()) {
2997       Op.addRegOperands(Inst, 1);
2998       continue;
2999     }
3000 
3001     if (Op.isToken() && Op.getToken() == "gds") {
3002       IsGdsHardcoded = true;
3003       continue;
3004     }
3005 
3006     // Handle optional arguments
3007     OptionalIdx[Op.getImmTy()] = i;
3008   }
3009 
3010   AMDGPUOperand::ImmTy OffsetType =
3011     (Inst.getOpcode() == AMDGPU::DS_SWIZZLE_B32_si ||
3012      Inst.getOpcode() == AMDGPU::DS_SWIZZLE_B32_vi) ? AMDGPUOperand::ImmTySwizzle :
3013                                                       AMDGPUOperand::ImmTyOffset;
3014 
3015   addOptionalImmOperand(Inst, Operands, OptionalIdx, OffsetType);
3016 
3017   if (!IsGdsHardcoded) {
3018     addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyGDS);
3019   }
3020   Inst.addOperand(MCOperand::createReg(AMDGPU::M0)); // m0
3021 }
3022 
3023 void AMDGPUAsmParser::cvtExp(MCInst &Inst, const OperandVector &Operands) {
3024   OptionalImmIndexMap OptionalIdx;
3025 
3026   unsigned OperandIdx[4];
3027   unsigned EnMask = 0;
3028   int SrcIdx = 0;
3029 
3030   for (unsigned i = 1, e = Operands.size(); i != e; ++i) {
3031     AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[i]);
3032 
3033     // Add the register arguments
3034     if (Op.isReg()) {
3035       assert(SrcIdx < 4);
3036       OperandIdx[SrcIdx] = Inst.size();
3037       Op.addRegOperands(Inst, 1);
3038       ++SrcIdx;
3039       continue;
3040     }
3041 
3042     if (Op.isOff()) {
3043       assert(SrcIdx < 4);
3044       OperandIdx[SrcIdx] = Inst.size();
3045       Inst.addOperand(MCOperand::createReg(AMDGPU::NoRegister));
3046       ++SrcIdx;
3047       continue;
3048     }
3049 
3050     if (Op.isImm() && Op.getImmTy() == AMDGPUOperand::ImmTyExpTgt) {
3051       Op.addImmOperands(Inst, 1);
3052       continue;
3053     }
3054 
3055     if (Op.isToken() && Op.getToken() == "done")
3056       continue;
3057 
3058     // Handle optional arguments
3059     OptionalIdx[Op.getImmTy()] = i;
3060   }
3061 
3062   assert(SrcIdx == 4);
3063 
3064   bool Compr = false;
3065   if (OptionalIdx.find(AMDGPUOperand::ImmTyExpCompr) != OptionalIdx.end()) {
3066     Compr = true;
3067     Inst.getOperand(OperandIdx[1]) = Inst.getOperand(OperandIdx[2]);
3068     Inst.getOperand(OperandIdx[2]).setReg(AMDGPU::NoRegister);
3069     Inst.getOperand(OperandIdx[3]).setReg(AMDGPU::NoRegister);
3070   }
3071 
3072   for (auto i = 0; i < SrcIdx; ++i) {
3073     if (Inst.getOperand(OperandIdx[i]).getReg() != AMDGPU::NoRegister) {
3074       EnMask |= Compr? (0x3 << i * 2) : (0x1 << i);
3075     }
3076   }
3077 
3078   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyExpVM);
3079   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyExpCompr);
3080 
3081   Inst.addOperand(MCOperand::createImm(EnMask));
3082 }
3083 
3084 //===----------------------------------------------------------------------===//
3085 // s_waitcnt
3086 //===----------------------------------------------------------------------===//
3087 
3088 static bool
3089 encodeCnt(
3090   const AMDGPU::IsaInfo::IsaVersion ISA,
3091   int64_t &IntVal,
3092   int64_t CntVal,
3093   bool Saturate,
3094   unsigned (*encode)(const IsaInfo::IsaVersion &Version, unsigned, unsigned),
3095   unsigned (*decode)(const IsaInfo::IsaVersion &Version, unsigned))
3096 {
3097   bool Failed = false;
3098 
3099   IntVal = encode(ISA, IntVal, CntVal);
3100   if (CntVal != decode(ISA, IntVal)) {
3101     if (Saturate) {
3102       IntVal = encode(ISA, IntVal, -1);
3103     } else {
3104       Failed = true;
3105     }
3106   }
3107   return Failed;
3108 }
3109 
3110 bool AMDGPUAsmParser::parseCnt(int64_t &IntVal) {
3111   StringRef CntName = Parser.getTok().getString();
3112   int64_t CntVal;
3113 
3114   Parser.Lex();
3115   if (getLexer().isNot(AsmToken::LParen))
3116     return true;
3117 
3118   Parser.Lex();
3119   if (getLexer().isNot(AsmToken::Integer))
3120     return true;
3121 
3122   SMLoc ValLoc = Parser.getTok().getLoc();
3123   if (getParser().parseAbsoluteExpression(CntVal))
3124     return true;
3125 
3126   AMDGPU::IsaInfo::IsaVersion ISA =
3127       AMDGPU::IsaInfo::getIsaVersion(getFeatureBits());
3128 
3129   bool Failed = true;
3130   bool Sat = CntName.endswith("_sat");
3131 
3132   if (CntName == "vmcnt" || CntName == "vmcnt_sat") {
3133     Failed = encodeCnt(ISA, IntVal, CntVal, Sat, encodeVmcnt, decodeVmcnt);
3134   } else if (CntName == "expcnt" || CntName == "expcnt_sat") {
3135     Failed = encodeCnt(ISA, IntVal, CntVal, Sat, encodeExpcnt, decodeExpcnt);
3136   } else if (CntName == "lgkmcnt" || CntName == "lgkmcnt_sat") {
3137     Failed = encodeCnt(ISA, IntVal, CntVal, Sat, encodeLgkmcnt, decodeLgkmcnt);
3138   }
3139 
3140   if (Failed) {
3141     Error(ValLoc, "too large value for " + CntName);
3142     return true;
3143   }
3144 
3145   if (getLexer().isNot(AsmToken::RParen)) {
3146     return true;
3147   }
3148 
3149   Parser.Lex();
3150   if (getLexer().is(AsmToken::Amp) || getLexer().is(AsmToken::Comma)) {
3151     const AsmToken NextToken = getLexer().peekTok();
3152     if (NextToken.is(AsmToken::Identifier)) {
3153       Parser.Lex();
3154     }
3155   }
3156 
3157   return false;
3158 }
3159 
3160 OperandMatchResultTy
3161 AMDGPUAsmParser::parseSWaitCntOps(OperandVector &Operands) {
3162   AMDGPU::IsaInfo::IsaVersion ISA =
3163       AMDGPU::IsaInfo::getIsaVersion(getFeatureBits());
3164   int64_t Waitcnt = getWaitcntBitMask(ISA);
3165   SMLoc S = Parser.getTok().getLoc();
3166 
3167   switch(getLexer().getKind()) {
3168     default: return MatchOperand_ParseFail;
3169     case AsmToken::Integer:
3170       // The operand can be an integer value.
3171       if (getParser().parseAbsoluteExpression(Waitcnt))
3172         return MatchOperand_ParseFail;
3173       break;
3174 
3175     case AsmToken::Identifier:
3176       do {
3177         if (parseCnt(Waitcnt))
3178           return MatchOperand_ParseFail;
3179       } while(getLexer().isNot(AsmToken::EndOfStatement));
3180       break;
3181   }
3182   Operands.push_back(AMDGPUOperand::CreateImm(this, Waitcnt, S));
3183   return MatchOperand_Success;
3184 }
3185 
3186 bool AMDGPUAsmParser::parseHwregConstruct(OperandInfoTy &HwReg, int64_t &Offset,
3187                                           int64_t &Width) {
3188   using namespace llvm::AMDGPU::Hwreg;
3189 
3190   if (Parser.getTok().getString() != "hwreg")
3191     return true;
3192   Parser.Lex();
3193 
3194   if (getLexer().isNot(AsmToken::LParen))
3195     return true;
3196   Parser.Lex();
3197 
3198   if (getLexer().is(AsmToken::Identifier)) {
3199     HwReg.IsSymbolic = true;
3200     HwReg.Id = ID_UNKNOWN_;
3201     const StringRef tok = Parser.getTok().getString();
3202     int Last = ID_SYMBOLIC_LAST_;
3203     if (isSI() || isCI() || isVI())
3204       Last = ID_SYMBOLIC_FIRST_GFX9_;
3205     for (int i = ID_SYMBOLIC_FIRST_; i < Last; ++i) {
3206       if (tok == IdSymbolic[i]) {
3207         HwReg.Id = i;
3208         break;
3209       }
3210     }
3211     Parser.Lex();
3212   } else {
3213     HwReg.IsSymbolic = false;
3214     if (getLexer().isNot(AsmToken::Integer))
3215       return true;
3216     if (getParser().parseAbsoluteExpression(HwReg.Id))
3217       return true;
3218   }
3219 
3220   if (getLexer().is(AsmToken::RParen)) {
3221     Parser.Lex();
3222     return false;
3223   }
3224 
3225   // optional params
3226   if (getLexer().isNot(AsmToken::Comma))
3227     return true;
3228   Parser.Lex();
3229 
3230   if (getLexer().isNot(AsmToken::Integer))
3231     return true;
3232   if (getParser().parseAbsoluteExpression(Offset))
3233     return true;
3234 
3235   if (getLexer().isNot(AsmToken::Comma))
3236     return true;
3237   Parser.Lex();
3238 
3239   if (getLexer().isNot(AsmToken::Integer))
3240     return true;
3241   if (getParser().parseAbsoluteExpression(Width))
3242     return true;
3243 
3244   if (getLexer().isNot(AsmToken::RParen))
3245     return true;
3246   Parser.Lex();
3247 
3248   return false;
3249 }
3250 
3251 OperandMatchResultTy AMDGPUAsmParser::parseHwreg(OperandVector &Operands) {
3252   using namespace llvm::AMDGPU::Hwreg;
3253 
3254   int64_t Imm16Val = 0;
3255   SMLoc S = Parser.getTok().getLoc();
3256 
3257   switch(getLexer().getKind()) {
3258     default: return MatchOperand_NoMatch;
3259     case AsmToken::Integer:
3260       // The operand can be an integer value.
3261       if (getParser().parseAbsoluteExpression(Imm16Val))
3262         return MatchOperand_NoMatch;
3263       if (Imm16Val < 0 || !isUInt<16>(Imm16Val)) {
3264         Error(S, "invalid immediate: only 16-bit values are legal");
3265         // Do not return error code, but create an imm operand anyway and proceed
3266         // to the next operand, if any. That avoids unneccessary error messages.
3267       }
3268       break;
3269 
3270     case AsmToken::Identifier: {
3271         OperandInfoTy HwReg(ID_UNKNOWN_);
3272         int64_t Offset = OFFSET_DEFAULT_;
3273         int64_t Width = WIDTH_M1_DEFAULT_ + 1;
3274         if (parseHwregConstruct(HwReg, Offset, Width))
3275           return MatchOperand_ParseFail;
3276         if (HwReg.Id < 0 || !isUInt<ID_WIDTH_>(HwReg.Id)) {
3277           if (HwReg.IsSymbolic)
3278             Error(S, "invalid symbolic name of hardware register");
3279           else
3280             Error(S, "invalid code of hardware register: only 6-bit values are legal");
3281         }
3282         if (Offset < 0 || !isUInt<OFFSET_WIDTH_>(Offset))
3283           Error(S, "invalid bit offset: only 5-bit values are legal");
3284         if ((Width-1) < 0 || !isUInt<WIDTH_M1_WIDTH_>(Width-1))
3285           Error(S, "invalid bitfield width: only values from 1 to 32 are legal");
3286         Imm16Val = (HwReg.Id << ID_SHIFT_) | (Offset << OFFSET_SHIFT_) | ((Width-1) << WIDTH_M1_SHIFT_);
3287       }
3288       break;
3289   }
3290   Operands.push_back(AMDGPUOperand::CreateImm(this, Imm16Val, S, AMDGPUOperand::ImmTyHwreg));
3291   return MatchOperand_Success;
3292 }
3293 
3294 bool AMDGPUOperand::isSWaitCnt() const {
3295   return isImm();
3296 }
3297 
3298 bool AMDGPUOperand::isHwreg() const {
3299   return isImmTy(ImmTyHwreg);
3300 }
3301 
3302 bool AMDGPUAsmParser::parseSendMsgConstruct(OperandInfoTy &Msg, OperandInfoTy &Operation, int64_t &StreamId) {
3303   using namespace llvm::AMDGPU::SendMsg;
3304 
3305   if (Parser.getTok().getString() != "sendmsg")
3306     return true;
3307   Parser.Lex();
3308 
3309   if (getLexer().isNot(AsmToken::LParen))
3310     return true;
3311   Parser.Lex();
3312 
3313   if (getLexer().is(AsmToken::Identifier)) {
3314     Msg.IsSymbolic = true;
3315     Msg.Id = ID_UNKNOWN_;
3316     const std::string tok = Parser.getTok().getString();
3317     for (int i = ID_GAPS_FIRST_; i < ID_GAPS_LAST_; ++i) {
3318       switch(i) {
3319         default: continue; // Omit gaps.
3320         case ID_INTERRUPT: case ID_GS: case ID_GS_DONE:  case ID_SYSMSG: break;
3321       }
3322       if (tok == IdSymbolic[i]) {
3323         Msg.Id = i;
3324         break;
3325       }
3326     }
3327     Parser.Lex();
3328   } else {
3329     Msg.IsSymbolic = false;
3330     if (getLexer().isNot(AsmToken::Integer))
3331       return true;
3332     if (getParser().parseAbsoluteExpression(Msg.Id))
3333       return true;
3334     if (getLexer().is(AsmToken::Integer))
3335       if (getParser().parseAbsoluteExpression(Msg.Id))
3336         Msg.Id = ID_UNKNOWN_;
3337   }
3338   if (Msg.Id == ID_UNKNOWN_) // Don't know how to parse the rest.
3339     return false;
3340 
3341   if (!(Msg.Id == ID_GS || Msg.Id == ID_GS_DONE || Msg.Id == ID_SYSMSG)) {
3342     if (getLexer().isNot(AsmToken::RParen))
3343       return true;
3344     Parser.Lex();
3345     return false;
3346   }
3347 
3348   if (getLexer().isNot(AsmToken::Comma))
3349     return true;
3350   Parser.Lex();
3351 
3352   assert(Msg.Id == ID_GS || Msg.Id == ID_GS_DONE || Msg.Id == ID_SYSMSG);
3353   Operation.Id = ID_UNKNOWN_;
3354   if (getLexer().is(AsmToken::Identifier)) {
3355     Operation.IsSymbolic = true;
3356     const char* const *S = (Msg.Id == ID_SYSMSG) ? OpSysSymbolic : OpGsSymbolic;
3357     const int F = (Msg.Id == ID_SYSMSG) ? OP_SYS_FIRST_ : OP_GS_FIRST_;
3358     const int L = (Msg.Id == ID_SYSMSG) ? OP_SYS_LAST_ : OP_GS_LAST_;
3359     const StringRef Tok = Parser.getTok().getString();
3360     for (int i = F; i < L; ++i) {
3361       if (Tok == S[i]) {
3362         Operation.Id = i;
3363         break;
3364       }
3365     }
3366     Parser.Lex();
3367   } else {
3368     Operation.IsSymbolic = false;
3369     if (getLexer().isNot(AsmToken::Integer))
3370       return true;
3371     if (getParser().parseAbsoluteExpression(Operation.Id))
3372       return true;
3373   }
3374 
3375   if ((Msg.Id == ID_GS || Msg.Id == ID_GS_DONE) && Operation.Id != OP_GS_NOP) {
3376     // Stream id is optional.
3377     if (getLexer().is(AsmToken::RParen)) {
3378       Parser.Lex();
3379       return false;
3380     }
3381 
3382     if (getLexer().isNot(AsmToken::Comma))
3383       return true;
3384     Parser.Lex();
3385 
3386     if (getLexer().isNot(AsmToken::Integer))
3387       return true;
3388     if (getParser().parseAbsoluteExpression(StreamId))
3389       return true;
3390   }
3391 
3392   if (getLexer().isNot(AsmToken::RParen))
3393     return true;
3394   Parser.Lex();
3395   return false;
3396 }
3397 
3398 OperandMatchResultTy AMDGPUAsmParser::parseInterpSlot(OperandVector &Operands) {
3399   if (getLexer().getKind() != AsmToken::Identifier)
3400     return MatchOperand_NoMatch;
3401 
3402   StringRef Str = Parser.getTok().getString();
3403   int Slot = StringSwitch<int>(Str)
3404     .Case("p10", 0)
3405     .Case("p20", 1)
3406     .Case("p0", 2)
3407     .Default(-1);
3408 
3409   SMLoc S = Parser.getTok().getLoc();
3410   if (Slot == -1)
3411     return MatchOperand_ParseFail;
3412 
3413   Parser.Lex();
3414   Operands.push_back(AMDGPUOperand::CreateImm(this, Slot, S,
3415                                               AMDGPUOperand::ImmTyInterpSlot));
3416   return MatchOperand_Success;
3417 }
3418 
3419 OperandMatchResultTy AMDGPUAsmParser::parseInterpAttr(OperandVector &Operands) {
3420   if (getLexer().getKind() != AsmToken::Identifier)
3421     return MatchOperand_NoMatch;
3422 
3423   StringRef Str = Parser.getTok().getString();
3424   if (!Str.startswith("attr"))
3425     return MatchOperand_NoMatch;
3426 
3427   StringRef Chan = Str.take_back(2);
3428   int AttrChan = StringSwitch<int>(Chan)
3429     .Case(".x", 0)
3430     .Case(".y", 1)
3431     .Case(".z", 2)
3432     .Case(".w", 3)
3433     .Default(-1);
3434   if (AttrChan == -1)
3435     return MatchOperand_ParseFail;
3436 
3437   Str = Str.drop_back(2).drop_front(4);
3438 
3439   uint8_t Attr;
3440   if (Str.getAsInteger(10, Attr))
3441     return MatchOperand_ParseFail;
3442 
3443   SMLoc S = Parser.getTok().getLoc();
3444   Parser.Lex();
3445   if (Attr > 63) {
3446     Error(S, "out of bounds attr");
3447     return MatchOperand_Success;
3448   }
3449 
3450   SMLoc SChan = SMLoc::getFromPointer(Chan.data());
3451 
3452   Operands.push_back(AMDGPUOperand::CreateImm(this, Attr, S,
3453                                               AMDGPUOperand::ImmTyInterpAttr));
3454   Operands.push_back(AMDGPUOperand::CreateImm(this, AttrChan, SChan,
3455                                               AMDGPUOperand::ImmTyAttrChan));
3456   return MatchOperand_Success;
3457 }
3458 
3459 void AMDGPUAsmParser::errorExpTgt() {
3460   Error(Parser.getTok().getLoc(), "invalid exp target");
3461 }
3462 
3463 OperandMatchResultTy AMDGPUAsmParser::parseExpTgtImpl(StringRef Str,
3464                                                       uint8_t &Val) {
3465   if (Str == "null") {
3466     Val = 9;
3467     return MatchOperand_Success;
3468   }
3469 
3470   if (Str.startswith("mrt")) {
3471     Str = Str.drop_front(3);
3472     if (Str == "z") { // == mrtz
3473       Val = 8;
3474       return MatchOperand_Success;
3475     }
3476 
3477     if (Str.getAsInteger(10, Val))
3478       return MatchOperand_ParseFail;
3479 
3480     if (Val > 7)
3481       errorExpTgt();
3482 
3483     return MatchOperand_Success;
3484   }
3485 
3486   if (Str.startswith("pos")) {
3487     Str = Str.drop_front(3);
3488     if (Str.getAsInteger(10, Val))
3489       return MatchOperand_ParseFail;
3490 
3491     if (Val > 3)
3492       errorExpTgt();
3493 
3494     Val += 12;
3495     return MatchOperand_Success;
3496   }
3497 
3498   if (Str.startswith("param")) {
3499     Str = Str.drop_front(5);
3500     if (Str.getAsInteger(10, Val))
3501       return MatchOperand_ParseFail;
3502 
3503     if (Val >= 32)
3504       errorExpTgt();
3505 
3506     Val += 32;
3507     return MatchOperand_Success;
3508   }
3509 
3510   if (Str.startswith("invalid_target_")) {
3511     Str = Str.drop_front(15);
3512     if (Str.getAsInteger(10, Val))
3513       return MatchOperand_ParseFail;
3514 
3515     errorExpTgt();
3516     return MatchOperand_Success;
3517   }
3518 
3519   return MatchOperand_NoMatch;
3520 }
3521 
3522 OperandMatchResultTy AMDGPUAsmParser::parseExpTgt(OperandVector &Operands) {
3523   uint8_t Val;
3524   StringRef Str = Parser.getTok().getString();
3525 
3526   auto Res = parseExpTgtImpl(Str, Val);
3527   if (Res != MatchOperand_Success)
3528     return Res;
3529 
3530   SMLoc S = Parser.getTok().getLoc();
3531   Parser.Lex();
3532 
3533   Operands.push_back(AMDGPUOperand::CreateImm(this, Val, S,
3534                                               AMDGPUOperand::ImmTyExpTgt));
3535   return MatchOperand_Success;
3536 }
3537 
3538 OperandMatchResultTy
3539 AMDGPUAsmParser::parseSendMsgOp(OperandVector &Operands) {
3540   using namespace llvm::AMDGPU::SendMsg;
3541 
3542   int64_t Imm16Val = 0;
3543   SMLoc S = Parser.getTok().getLoc();
3544 
3545   switch(getLexer().getKind()) {
3546   default:
3547     return MatchOperand_NoMatch;
3548   case AsmToken::Integer:
3549     // The operand can be an integer value.
3550     if (getParser().parseAbsoluteExpression(Imm16Val))
3551       return MatchOperand_NoMatch;
3552     if (Imm16Val < 0 || !isUInt<16>(Imm16Val)) {
3553       Error(S, "invalid immediate: only 16-bit values are legal");
3554       // Do not return error code, but create an imm operand anyway and proceed
3555       // to the next operand, if any. That avoids unneccessary error messages.
3556     }
3557     break;
3558   case AsmToken::Identifier: {
3559       OperandInfoTy Msg(ID_UNKNOWN_);
3560       OperandInfoTy Operation(OP_UNKNOWN_);
3561       int64_t StreamId = STREAM_ID_DEFAULT_;
3562       if (parseSendMsgConstruct(Msg, Operation, StreamId))
3563         return MatchOperand_ParseFail;
3564       do {
3565         // Validate and encode message ID.
3566         if (! ((ID_INTERRUPT <= Msg.Id && Msg.Id <= ID_GS_DONE)
3567                 || Msg.Id == ID_SYSMSG)) {
3568           if (Msg.IsSymbolic)
3569             Error(S, "invalid/unsupported symbolic name of message");
3570           else
3571             Error(S, "invalid/unsupported code of message");
3572           break;
3573         }
3574         Imm16Val = (Msg.Id << ID_SHIFT_);
3575         // Validate and encode operation ID.
3576         if (Msg.Id == ID_GS || Msg.Id == ID_GS_DONE) {
3577           if (! (OP_GS_FIRST_ <= Operation.Id && Operation.Id < OP_GS_LAST_)) {
3578             if (Operation.IsSymbolic)
3579               Error(S, "invalid symbolic name of GS_OP");
3580             else
3581               Error(S, "invalid code of GS_OP: only 2-bit values are legal");
3582             break;
3583           }
3584           if (Operation.Id == OP_GS_NOP
3585               && Msg.Id != ID_GS_DONE) {
3586             Error(S, "invalid GS_OP: NOP is for GS_DONE only");
3587             break;
3588           }
3589           Imm16Val |= (Operation.Id << OP_SHIFT_);
3590         }
3591         if (Msg.Id == ID_SYSMSG) {
3592           if (! (OP_SYS_FIRST_ <= Operation.Id && Operation.Id < OP_SYS_LAST_)) {
3593             if (Operation.IsSymbolic)
3594               Error(S, "invalid/unsupported symbolic name of SYSMSG_OP");
3595             else
3596               Error(S, "invalid/unsupported code of SYSMSG_OP");
3597             break;
3598           }
3599           Imm16Val |= (Operation.Id << OP_SHIFT_);
3600         }
3601         // Validate and encode stream ID.
3602         if ((Msg.Id == ID_GS || Msg.Id == ID_GS_DONE) && Operation.Id != OP_GS_NOP) {
3603           if (! (STREAM_ID_FIRST_ <= StreamId && StreamId < STREAM_ID_LAST_)) {
3604             Error(S, "invalid stream id: only 2-bit values are legal");
3605             break;
3606           }
3607           Imm16Val |= (StreamId << STREAM_ID_SHIFT_);
3608         }
3609       } while (false);
3610     }
3611     break;
3612   }
3613   Operands.push_back(AMDGPUOperand::CreateImm(this, Imm16Val, S, AMDGPUOperand::ImmTySendMsg));
3614   return MatchOperand_Success;
3615 }
3616 
3617 bool AMDGPUOperand::isSendMsg() const {
3618   return isImmTy(ImmTySendMsg);
3619 }
3620 
3621 //===----------------------------------------------------------------------===//
3622 // parser helpers
3623 //===----------------------------------------------------------------------===//
3624 
3625 bool
3626 AMDGPUAsmParser::trySkipId(const StringRef Id) {
3627   if (getLexer().getKind() == AsmToken::Identifier &&
3628       Parser.getTok().getString() == Id) {
3629     Parser.Lex();
3630     return true;
3631   }
3632   return false;
3633 }
3634 
3635 bool
3636 AMDGPUAsmParser::trySkipToken(const AsmToken::TokenKind Kind) {
3637   if (getLexer().getKind() == Kind) {
3638     Parser.Lex();
3639     return true;
3640   }
3641   return false;
3642 }
3643 
3644 bool
3645 AMDGPUAsmParser::skipToken(const AsmToken::TokenKind Kind,
3646                            const StringRef ErrMsg) {
3647   if (!trySkipToken(Kind)) {
3648     Error(Parser.getTok().getLoc(), ErrMsg);
3649     return false;
3650   }
3651   return true;
3652 }
3653 
3654 bool
3655 AMDGPUAsmParser::parseExpr(int64_t &Imm) {
3656   return !getParser().parseAbsoluteExpression(Imm);
3657 }
3658 
3659 bool
3660 AMDGPUAsmParser::parseString(StringRef &Val, const StringRef ErrMsg) {
3661   SMLoc S = Parser.getTok().getLoc();
3662   if (getLexer().getKind() == AsmToken::String) {
3663     Val = Parser.getTok().getStringContents();
3664     Parser.Lex();
3665     return true;
3666   } else {
3667     Error(S, ErrMsg);
3668     return false;
3669   }
3670 }
3671 
3672 //===----------------------------------------------------------------------===//
3673 // swizzle
3674 //===----------------------------------------------------------------------===//
3675 
3676 LLVM_READNONE
3677 static unsigned
3678 encodeBitmaskPerm(const unsigned AndMask,
3679                   const unsigned OrMask,
3680                   const unsigned XorMask) {
3681   using namespace llvm::AMDGPU::Swizzle;
3682 
3683   return BITMASK_PERM_ENC |
3684          (AndMask << BITMASK_AND_SHIFT) |
3685          (OrMask  << BITMASK_OR_SHIFT)  |
3686          (XorMask << BITMASK_XOR_SHIFT);
3687 }
3688 
3689 bool
3690 AMDGPUAsmParser::parseSwizzleOperands(const unsigned OpNum, int64_t* Op,
3691                                       const unsigned MinVal,
3692                                       const unsigned MaxVal,
3693                                       const StringRef ErrMsg) {
3694   for (unsigned i = 0; i < OpNum; ++i) {
3695     if (!skipToken(AsmToken::Comma, "expected a comma")){
3696       return false;
3697     }
3698     SMLoc ExprLoc = Parser.getTok().getLoc();
3699     if (!parseExpr(Op[i])) {
3700       return false;
3701     }
3702     if (Op[i] < MinVal || Op[i] > MaxVal) {
3703       Error(ExprLoc, ErrMsg);
3704       return false;
3705     }
3706   }
3707 
3708   return true;
3709 }
3710 
3711 bool
3712 AMDGPUAsmParser::parseSwizzleQuadPerm(int64_t &Imm) {
3713   using namespace llvm::AMDGPU::Swizzle;
3714 
3715   int64_t Lane[LANE_NUM];
3716   if (parseSwizzleOperands(LANE_NUM, Lane, 0, LANE_MAX,
3717                            "expected a 2-bit lane id")) {
3718     Imm = QUAD_PERM_ENC;
3719     for (auto i = 0; i < LANE_NUM; ++i) {
3720       Imm |= Lane[i] << (LANE_SHIFT * i);
3721     }
3722     return true;
3723   }
3724   return false;
3725 }
3726 
3727 bool
3728 AMDGPUAsmParser::parseSwizzleBroadcast(int64_t &Imm) {
3729   using namespace llvm::AMDGPU::Swizzle;
3730 
3731   SMLoc S = Parser.getTok().getLoc();
3732   int64_t GroupSize;
3733   int64_t LaneIdx;
3734 
3735   if (!parseSwizzleOperands(1, &GroupSize,
3736                             2, 32,
3737                             "group size must be in the interval [2,32]")) {
3738     return false;
3739   }
3740   if (!isPowerOf2_64(GroupSize)) {
3741     Error(S, "group size must be a power of two");
3742     return false;
3743   }
3744   if (parseSwizzleOperands(1, &LaneIdx,
3745                            0, GroupSize - 1,
3746                            "lane id must be in the interval [0,group size - 1]")) {
3747     Imm = encodeBitmaskPerm(BITMASK_MAX - GroupSize + 1, LaneIdx, 0);
3748     return true;
3749   }
3750   return false;
3751 }
3752 
3753 bool
3754 AMDGPUAsmParser::parseSwizzleReverse(int64_t &Imm) {
3755   using namespace llvm::AMDGPU::Swizzle;
3756 
3757   SMLoc S = Parser.getTok().getLoc();
3758   int64_t GroupSize;
3759 
3760   if (!parseSwizzleOperands(1, &GroupSize,
3761       2, 32, "group size must be in the interval [2,32]")) {
3762     return false;
3763   }
3764   if (!isPowerOf2_64(GroupSize)) {
3765     Error(S, "group size must be a power of two");
3766     return false;
3767   }
3768 
3769   Imm = encodeBitmaskPerm(BITMASK_MAX, 0, GroupSize - 1);
3770   return true;
3771 }
3772 
3773 bool
3774 AMDGPUAsmParser::parseSwizzleSwap(int64_t &Imm) {
3775   using namespace llvm::AMDGPU::Swizzle;
3776 
3777   SMLoc S = Parser.getTok().getLoc();
3778   int64_t GroupSize;
3779 
3780   if (!parseSwizzleOperands(1, &GroupSize,
3781       1, 16, "group size must be in the interval [1,16]")) {
3782     return false;
3783   }
3784   if (!isPowerOf2_64(GroupSize)) {
3785     Error(S, "group size must be a power of two");
3786     return false;
3787   }
3788 
3789   Imm = encodeBitmaskPerm(BITMASK_MAX, 0, GroupSize);
3790   return true;
3791 }
3792 
3793 bool
3794 AMDGPUAsmParser::parseSwizzleBitmaskPerm(int64_t &Imm) {
3795   using namespace llvm::AMDGPU::Swizzle;
3796 
3797   if (!skipToken(AsmToken::Comma, "expected a comma")) {
3798     return false;
3799   }
3800 
3801   StringRef Ctl;
3802   SMLoc StrLoc = Parser.getTok().getLoc();
3803   if (!parseString(Ctl)) {
3804     return false;
3805   }
3806   if (Ctl.size() != BITMASK_WIDTH) {
3807     Error(StrLoc, "expected a 5-character mask");
3808     return false;
3809   }
3810 
3811   unsigned AndMask = 0;
3812   unsigned OrMask = 0;
3813   unsigned XorMask = 0;
3814 
3815   for (size_t i = 0; i < Ctl.size(); ++i) {
3816     unsigned Mask = 1 << (BITMASK_WIDTH - 1 - i);
3817     switch(Ctl[i]) {
3818     default:
3819       Error(StrLoc, "invalid mask");
3820       return false;
3821     case '0':
3822       break;
3823     case '1':
3824       OrMask |= Mask;
3825       break;
3826     case 'p':
3827       AndMask |= Mask;
3828       break;
3829     case 'i':
3830       AndMask |= Mask;
3831       XorMask |= Mask;
3832       break;
3833     }
3834   }
3835 
3836   Imm = encodeBitmaskPerm(AndMask, OrMask, XorMask);
3837   return true;
3838 }
3839 
3840 bool
3841 AMDGPUAsmParser::parseSwizzleOffset(int64_t &Imm) {
3842 
3843   SMLoc OffsetLoc = Parser.getTok().getLoc();
3844 
3845   if (!parseExpr(Imm)) {
3846     return false;
3847   }
3848   if (!isUInt<16>(Imm)) {
3849     Error(OffsetLoc, "expected a 16-bit offset");
3850     return false;
3851   }
3852   return true;
3853 }
3854 
3855 bool
3856 AMDGPUAsmParser::parseSwizzleMacro(int64_t &Imm) {
3857   using namespace llvm::AMDGPU::Swizzle;
3858 
3859   if (skipToken(AsmToken::LParen, "expected a left parentheses")) {
3860 
3861     SMLoc ModeLoc = Parser.getTok().getLoc();
3862     bool Ok = false;
3863 
3864     if (trySkipId(IdSymbolic[ID_QUAD_PERM])) {
3865       Ok = parseSwizzleQuadPerm(Imm);
3866     } else if (trySkipId(IdSymbolic[ID_BITMASK_PERM])) {
3867       Ok = parseSwizzleBitmaskPerm(Imm);
3868     } else if (trySkipId(IdSymbolic[ID_BROADCAST])) {
3869       Ok = parseSwizzleBroadcast(Imm);
3870     } else if (trySkipId(IdSymbolic[ID_SWAP])) {
3871       Ok = parseSwizzleSwap(Imm);
3872     } else if (trySkipId(IdSymbolic[ID_REVERSE])) {
3873       Ok = parseSwizzleReverse(Imm);
3874     } else {
3875       Error(ModeLoc, "expected a swizzle mode");
3876     }
3877 
3878     return Ok && skipToken(AsmToken::RParen, "expected a closing parentheses");
3879   }
3880 
3881   return false;
3882 }
3883 
3884 OperandMatchResultTy
3885 AMDGPUAsmParser::parseSwizzleOp(OperandVector &Operands) {
3886   SMLoc S = Parser.getTok().getLoc();
3887   int64_t Imm = 0;
3888 
3889   if (trySkipId("offset")) {
3890 
3891     bool Ok = false;
3892     if (skipToken(AsmToken::Colon, "expected a colon")) {
3893       if (trySkipId("swizzle")) {
3894         Ok = parseSwizzleMacro(Imm);
3895       } else {
3896         Ok = parseSwizzleOffset(Imm);
3897       }
3898     }
3899 
3900     Operands.push_back(AMDGPUOperand::CreateImm(this, Imm, S, AMDGPUOperand::ImmTySwizzle));
3901 
3902     return Ok? MatchOperand_Success : MatchOperand_ParseFail;
3903   } else {
3904     // Swizzle "offset" operand is optional.
3905     // If it is omitted, try parsing other optional operands.
3906     return parseOptionalOpr(Operands);
3907   }
3908 }
3909 
3910 bool
3911 AMDGPUOperand::isSwizzle() const {
3912   return isImmTy(ImmTySwizzle);
3913 }
3914 
3915 //===----------------------------------------------------------------------===//
3916 // sopp branch targets
3917 //===----------------------------------------------------------------------===//
3918 
3919 OperandMatchResultTy
3920 AMDGPUAsmParser::parseSOppBrTarget(OperandVector &Operands) {
3921   SMLoc S = Parser.getTok().getLoc();
3922 
3923   switch (getLexer().getKind()) {
3924     default: return MatchOperand_ParseFail;
3925     case AsmToken::Integer: {
3926       int64_t Imm;
3927       if (getParser().parseAbsoluteExpression(Imm))
3928         return MatchOperand_ParseFail;
3929       Operands.push_back(AMDGPUOperand::CreateImm(this, Imm, S));
3930       return MatchOperand_Success;
3931     }
3932 
3933     case AsmToken::Identifier:
3934       Operands.push_back(AMDGPUOperand::CreateExpr(this,
3935           MCSymbolRefExpr::create(getContext().getOrCreateSymbol(
3936                                   Parser.getTok().getString()), getContext()), S));
3937       Parser.Lex();
3938       return MatchOperand_Success;
3939   }
3940 }
3941 
3942 //===----------------------------------------------------------------------===//
3943 // mubuf
3944 //===----------------------------------------------------------------------===//
3945 
3946 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultGLC() const {
3947   return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTyGLC);
3948 }
3949 
3950 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultSLC() const {
3951   return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTySLC);
3952 }
3953 
3954 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultTFE() const {
3955   return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTyTFE);
3956 }
3957 
3958 void AMDGPUAsmParser::cvtMubufImpl(MCInst &Inst,
3959                                const OperandVector &Operands,
3960                                bool IsAtomic, bool IsAtomicReturn) {
3961   OptionalImmIndexMap OptionalIdx;
3962   assert(IsAtomicReturn ? IsAtomic : true);
3963 
3964   for (unsigned i = 1, e = Operands.size(); i != e; ++i) {
3965     AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[i]);
3966 
3967     // Add the register arguments
3968     if (Op.isReg()) {
3969       Op.addRegOperands(Inst, 1);
3970       continue;
3971     }
3972 
3973     // Handle the case where soffset is an immediate
3974     if (Op.isImm() && Op.getImmTy() == AMDGPUOperand::ImmTyNone) {
3975       Op.addImmOperands(Inst, 1);
3976       continue;
3977     }
3978 
3979     // Handle tokens like 'offen' which are sometimes hard-coded into the
3980     // asm string.  There are no MCInst operands for these.
3981     if (Op.isToken()) {
3982       continue;
3983     }
3984     assert(Op.isImm());
3985 
3986     // Handle optional arguments
3987     OptionalIdx[Op.getImmTy()] = i;
3988   }
3989 
3990   // Copy $vdata_in operand and insert as $vdata for MUBUF_Atomic RTN insns.
3991   if (IsAtomicReturn) {
3992     MCInst::iterator I = Inst.begin(); // $vdata_in is always at the beginning.
3993     Inst.insert(I, *I);
3994   }
3995 
3996   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyOffset);
3997   if (!IsAtomic) { // glc is hard-coded.
3998     addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyGLC);
3999   }
4000   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySLC);
4001   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyTFE);
4002 }
4003 
4004 void AMDGPUAsmParser::cvtMtbuf(MCInst &Inst, const OperandVector &Operands) {
4005   OptionalImmIndexMap OptionalIdx;
4006 
4007   for (unsigned i = 1, e = Operands.size(); i != e; ++i) {
4008     AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[i]);
4009 
4010     // Add the register arguments
4011     if (Op.isReg()) {
4012       Op.addRegOperands(Inst, 1);
4013       continue;
4014     }
4015 
4016     // Handle the case where soffset is an immediate
4017     if (Op.isImm() && Op.getImmTy() == AMDGPUOperand::ImmTyNone) {
4018       Op.addImmOperands(Inst, 1);
4019       continue;
4020     }
4021 
4022     // Handle tokens like 'offen' which are sometimes hard-coded into the
4023     // asm string.  There are no MCInst operands for these.
4024     if (Op.isToken()) {
4025       continue;
4026     }
4027     assert(Op.isImm());
4028 
4029     // Handle optional arguments
4030     OptionalIdx[Op.getImmTy()] = i;
4031   }
4032 
4033   addOptionalImmOperand(Inst, Operands, OptionalIdx,
4034                         AMDGPUOperand::ImmTyOffset);
4035   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyDFMT);
4036   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyNFMT);
4037   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyGLC);
4038   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySLC);
4039   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyTFE);
4040 }
4041 
4042 //===----------------------------------------------------------------------===//
4043 // mimg
4044 //===----------------------------------------------------------------------===//
4045 
4046 void AMDGPUAsmParser::cvtMIMG(MCInst &Inst, const OperandVector &Operands,
4047                               bool IsAtomic) {
4048   unsigned I = 1;
4049   const MCInstrDesc &Desc = MII.get(Inst.getOpcode());
4050   for (unsigned J = 0; J < Desc.getNumDefs(); ++J) {
4051     ((AMDGPUOperand &)*Operands[I++]).addRegOperands(Inst, 1);
4052   }
4053 
4054   if (IsAtomic) {
4055     // Add src, same as dst
4056     ((AMDGPUOperand &)*Operands[I]).addRegOperands(Inst, 1);
4057   }
4058 
4059   OptionalImmIndexMap OptionalIdx;
4060 
4061   for (unsigned E = Operands.size(); I != E; ++I) {
4062     AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[I]);
4063 
4064     // Add the register arguments
4065     if (Op.isRegOrImm()) {
4066       Op.addRegOrImmOperands(Inst, 1);
4067       continue;
4068     } else if (Op.isImmModifier()) {
4069       OptionalIdx[Op.getImmTy()] = I;
4070     } else {
4071       llvm_unreachable("unexpected operand type");
4072     }
4073   }
4074 
4075   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyDMask);
4076   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyUNorm);
4077   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyGLC);
4078   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyDA);
4079   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyR128);
4080   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyTFE);
4081   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyLWE);
4082   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySLC);
4083 }
4084 
4085 void AMDGPUAsmParser::cvtMIMGAtomic(MCInst &Inst, const OperandVector &Operands) {
4086   cvtMIMG(Inst, Operands, true);
4087 }
4088 
4089 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultDMask() const {
4090   return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTyDMask);
4091 }
4092 
4093 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultUNorm() const {
4094   return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTyUNorm);
4095 }
4096 
4097 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultDA() const {
4098   return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTyDA);
4099 }
4100 
4101 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultR128() const {
4102   return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTyR128);
4103 }
4104 
4105 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultLWE() const {
4106   return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTyLWE);
4107 }
4108 
4109 //===----------------------------------------------------------------------===//
4110 // smrd
4111 //===----------------------------------------------------------------------===//
4112 
4113 bool AMDGPUOperand::isSMRDOffset8() const {
4114   return isImm() && isUInt<8>(getImm());
4115 }
4116 
4117 bool AMDGPUOperand::isSMRDOffset20() const {
4118   return isImm() && isUInt<20>(getImm());
4119 }
4120 
4121 bool AMDGPUOperand::isSMRDLiteralOffset() const {
4122   // 32-bit literals are only supported on CI and we only want to use them
4123   // when the offset is > 8-bits.
4124   return isImm() && !isUInt<8>(getImm()) && isUInt<32>(getImm());
4125 }
4126 
4127 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultSMRDOffset8() const {
4128   return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTyOffset);
4129 }
4130 
4131 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultSMRDOffset20() const {
4132   return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTyOffset);
4133 }
4134 
4135 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultSMRDLiteralOffset() const {
4136   return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTyOffset);
4137 }
4138 
4139 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultOffsetU12() const {
4140   return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTyOffset);
4141 }
4142 
4143 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultOffsetS13() const {
4144   return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTyOffset);
4145 }
4146 
4147 //===----------------------------------------------------------------------===//
4148 // vop3
4149 //===----------------------------------------------------------------------===//
4150 
4151 static bool ConvertOmodMul(int64_t &Mul) {
4152   if (Mul != 1 && Mul != 2 && Mul != 4)
4153     return false;
4154 
4155   Mul >>= 1;
4156   return true;
4157 }
4158 
4159 static bool ConvertOmodDiv(int64_t &Div) {
4160   if (Div == 1) {
4161     Div = 0;
4162     return true;
4163   }
4164 
4165   if (Div == 2) {
4166     Div = 3;
4167     return true;
4168   }
4169 
4170   return false;
4171 }
4172 
4173 static bool ConvertBoundCtrl(int64_t &BoundCtrl) {
4174   if (BoundCtrl == 0) {
4175     BoundCtrl = 1;
4176     return true;
4177   }
4178 
4179   if (BoundCtrl == -1) {
4180     BoundCtrl = 0;
4181     return true;
4182   }
4183 
4184   return false;
4185 }
4186 
4187 // Note: the order in this table matches the order of operands in AsmString.
4188 static const OptionalOperand AMDGPUOptionalOperandTable[] = {
4189   {"offen",   AMDGPUOperand::ImmTyOffen, true, nullptr},
4190   {"idxen",   AMDGPUOperand::ImmTyIdxen, true, nullptr},
4191   {"addr64",  AMDGPUOperand::ImmTyAddr64, true, nullptr},
4192   {"offset0", AMDGPUOperand::ImmTyOffset0, false, nullptr},
4193   {"offset1", AMDGPUOperand::ImmTyOffset1, false, nullptr},
4194   {"gds",     AMDGPUOperand::ImmTyGDS, true, nullptr},
4195   {"offset",  AMDGPUOperand::ImmTyOffset, false, nullptr},
4196   {"inst_offset", AMDGPUOperand::ImmTyInstOffset, false, nullptr},
4197   {"dfmt",    AMDGPUOperand::ImmTyDFMT, false, nullptr},
4198   {"nfmt",    AMDGPUOperand::ImmTyNFMT, false, nullptr},
4199   {"glc",     AMDGPUOperand::ImmTyGLC, true, nullptr},
4200   {"slc",     AMDGPUOperand::ImmTySLC, true, nullptr},
4201   {"tfe",     AMDGPUOperand::ImmTyTFE, true, nullptr},
4202   {"high",    AMDGPUOperand::ImmTyHigh, true, nullptr},
4203   {"clamp",   AMDGPUOperand::ImmTyClampSI, true, nullptr},
4204   {"omod",    AMDGPUOperand::ImmTyOModSI, false, ConvertOmodMul},
4205   {"unorm",   AMDGPUOperand::ImmTyUNorm, true, nullptr},
4206   {"da",      AMDGPUOperand::ImmTyDA,    true, nullptr},
4207   {"r128",    AMDGPUOperand::ImmTyR128,  true, nullptr},
4208   {"lwe",     AMDGPUOperand::ImmTyLWE,   true, nullptr},
4209   {"dmask",   AMDGPUOperand::ImmTyDMask, false, nullptr},
4210   {"row_mask",   AMDGPUOperand::ImmTyDppRowMask, false, nullptr},
4211   {"bank_mask",  AMDGPUOperand::ImmTyDppBankMask, false, nullptr},
4212   {"bound_ctrl", AMDGPUOperand::ImmTyDppBoundCtrl, false, ConvertBoundCtrl},
4213   {"dst_sel",    AMDGPUOperand::ImmTySdwaDstSel, false, nullptr},
4214   {"src0_sel",   AMDGPUOperand::ImmTySdwaSrc0Sel, false, nullptr},
4215   {"src1_sel",   AMDGPUOperand::ImmTySdwaSrc1Sel, false, nullptr},
4216   {"dst_unused", AMDGPUOperand::ImmTySdwaDstUnused, false, nullptr},
4217   {"compr", AMDGPUOperand::ImmTyExpCompr, true, nullptr },
4218   {"vm", AMDGPUOperand::ImmTyExpVM, true, nullptr},
4219   {"op_sel", AMDGPUOperand::ImmTyOpSel, false, nullptr},
4220   {"op_sel_hi", AMDGPUOperand::ImmTyOpSelHi, false, nullptr},
4221   {"neg_lo", AMDGPUOperand::ImmTyNegLo, false, nullptr},
4222   {"neg_hi", AMDGPUOperand::ImmTyNegHi, false, nullptr}
4223 };
4224 
4225 OperandMatchResultTy AMDGPUAsmParser::parseOptionalOperand(OperandVector &Operands) {
4226   unsigned size = Operands.size();
4227   assert(size > 0);
4228 
4229   OperandMatchResultTy res = parseOptionalOpr(Operands);
4230 
4231   // This is a hack to enable hardcoded mandatory operands which follow
4232   // optional operands.
4233   //
4234   // Current design assumes that all operands after the first optional operand
4235   // are also optional. However implementation of some instructions violates
4236   // this rule (see e.g. flat/global atomic which have hardcoded 'glc' operands).
4237   //
4238   // To alleviate this problem, we have to (implicitly) parse extra operands
4239   // to make sure autogenerated parser of custom operands never hit hardcoded
4240   // mandatory operands.
4241 
4242   if (size == 1 || ((AMDGPUOperand &)*Operands[size - 1]).isRegKind()) {
4243 
4244     // We have parsed the first optional operand.
4245     // Parse as many operands as necessary to skip all mandatory operands.
4246 
4247     for (unsigned i = 0; i < MAX_OPR_LOOKAHEAD; ++i) {
4248       if (res != MatchOperand_Success ||
4249           getLexer().is(AsmToken::EndOfStatement)) break;
4250       if (getLexer().is(AsmToken::Comma)) Parser.Lex();
4251       res = parseOptionalOpr(Operands);
4252     }
4253   }
4254 
4255   return res;
4256 }
4257 
4258 OperandMatchResultTy AMDGPUAsmParser::parseOptionalOpr(OperandVector &Operands) {
4259   OperandMatchResultTy res;
4260   for (const OptionalOperand &Op : AMDGPUOptionalOperandTable) {
4261     // try to parse any optional operand here
4262     if (Op.IsBit) {
4263       res = parseNamedBit(Op.Name, Operands, Op.Type);
4264     } else if (Op.Type == AMDGPUOperand::ImmTyOModSI) {
4265       res = parseOModOperand(Operands);
4266     } else if (Op.Type == AMDGPUOperand::ImmTySdwaDstSel ||
4267                Op.Type == AMDGPUOperand::ImmTySdwaSrc0Sel ||
4268                Op.Type == AMDGPUOperand::ImmTySdwaSrc1Sel) {
4269       res = parseSDWASel(Operands, Op.Name, Op.Type);
4270     } else if (Op.Type == AMDGPUOperand::ImmTySdwaDstUnused) {
4271       res = parseSDWADstUnused(Operands);
4272     } else if (Op.Type == AMDGPUOperand::ImmTyOpSel ||
4273                Op.Type == AMDGPUOperand::ImmTyOpSelHi ||
4274                Op.Type == AMDGPUOperand::ImmTyNegLo ||
4275                Op.Type == AMDGPUOperand::ImmTyNegHi) {
4276       res = parseOperandArrayWithPrefix(Op.Name, Operands, Op.Type,
4277                                         Op.ConvertResult);
4278     } else {
4279       res = parseIntWithPrefix(Op.Name, Operands, Op.Type, Op.ConvertResult);
4280     }
4281     if (res != MatchOperand_NoMatch) {
4282       return res;
4283     }
4284   }
4285   return MatchOperand_NoMatch;
4286 }
4287 
4288 OperandMatchResultTy AMDGPUAsmParser::parseOModOperand(OperandVector &Operands) {
4289   StringRef Name = Parser.getTok().getString();
4290   if (Name == "mul") {
4291     return parseIntWithPrefix("mul", Operands,
4292                               AMDGPUOperand::ImmTyOModSI, ConvertOmodMul);
4293   }
4294 
4295   if (Name == "div") {
4296     return parseIntWithPrefix("div", Operands,
4297                               AMDGPUOperand::ImmTyOModSI, ConvertOmodDiv);
4298   }
4299 
4300   return MatchOperand_NoMatch;
4301 }
4302 
4303 void AMDGPUAsmParser::cvtVOP3OpSel(MCInst &Inst, const OperandVector &Operands) {
4304   cvtVOP3P(Inst, Operands);
4305 
4306   int Opc = Inst.getOpcode();
4307 
4308   int SrcNum;
4309   const int Ops[] = { AMDGPU::OpName::src0,
4310                       AMDGPU::OpName::src1,
4311                       AMDGPU::OpName::src2 };
4312   for (SrcNum = 0;
4313        SrcNum < 3 && AMDGPU::getNamedOperandIdx(Opc, Ops[SrcNum]) != -1;
4314        ++SrcNum);
4315   assert(SrcNum > 0);
4316 
4317   int OpSelIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::op_sel);
4318   unsigned OpSel = Inst.getOperand(OpSelIdx).getImm();
4319 
4320   if ((OpSel & (1 << SrcNum)) != 0) {
4321     int ModIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src0_modifiers);
4322     uint32_t ModVal = Inst.getOperand(ModIdx).getImm();
4323     Inst.getOperand(ModIdx).setImm(ModVal | SISrcMods::DST_OP_SEL);
4324   }
4325 }
4326 
4327 static bool isRegOrImmWithInputMods(const MCInstrDesc &Desc, unsigned OpNum) {
4328       // 1. This operand is input modifiers
4329   return Desc.OpInfo[OpNum].OperandType == AMDGPU::OPERAND_INPUT_MODS
4330       // 2. This is not last operand
4331       && Desc.NumOperands > (OpNum + 1)
4332       // 3. Next operand is register class
4333       && Desc.OpInfo[OpNum + 1].RegClass != -1
4334       // 4. Next register is not tied to any other operand
4335       && Desc.getOperandConstraint(OpNum + 1, MCOI::OperandConstraint::TIED_TO) == -1;
4336 }
4337 
4338 void AMDGPUAsmParser::cvtVOP3Interp(MCInst &Inst, const OperandVector &Operands)
4339 {
4340   OptionalImmIndexMap OptionalIdx;
4341   unsigned Opc = Inst.getOpcode();
4342 
4343   unsigned I = 1;
4344   const MCInstrDesc &Desc = MII.get(Inst.getOpcode());
4345   for (unsigned J = 0; J < Desc.getNumDefs(); ++J) {
4346     ((AMDGPUOperand &)*Operands[I++]).addRegOperands(Inst, 1);
4347   }
4348 
4349   for (unsigned E = Operands.size(); I != E; ++I) {
4350     AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[I]);
4351     if (isRegOrImmWithInputMods(Desc, Inst.getNumOperands())) {
4352       Op.addRegOrImmWithFPInputModsOperands(Inst, 2);
4353     } else if (Op.isInterpSlot() ||
4354                Op.isInterpAttr() ||
4355                Op.isAttrChan()) {
4356       Inst.addOperand(MCOperand::createImm(Op.Imm.Val));
4357     } else if (Op.isImmModifier()) {
4358       OptionalIdx[Op.getImmTy()] = I;
4359     } else {
4360       llvm_unreachable("unhandled operand type");
4361     }
4362   }
4363 
4364   if (AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::high) != -1) {
4365     addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyHigh);
4366   }
4367 
4368   if (AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::clamp) != -1) {
4369     addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyClampSI);
4370   }
4371 
4372   if (AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::omod) != -1) {
4373     addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyOModSI);
4374   }
4375 }
4376 
4377 void AMDGPUAsmParser::cvtVOP3(MCInst &Inst, const OperandVector &Operands,
4378                               OptionalImmIndexMap &OptionalIdx) {
4379   unsigned Opc = Inst.getOpcode();
4380 
4381   unsigned I = 1;
4382   const MCInstrDesc &Desc = MII.get(Inst.getOpcode());
4383   for (unsigned J = 0; J < Desc.getNumDefs(); ++J) {
4384     ((AMDGPUOperand &)*Operands[I++]).addRegOperands(Inst, 1);
4385   }
4386 
4387   if (AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src0_modifiers) != -1) {
4388     // This instruction has src modifiers
4389     for (unsigned E = Operands.size(); I != E; ++I) {
4390       AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[I]);
4391       if (isRegOrImmWithInputMods(Desc, Inst.getNumOperands())) {
4392         Op.addRegOrImmWithFPInputModsOperands(Inst, 2);
4393       } else if (Op.isImmModifier()) {
4394         OptionalIdx[Op.getImmTy()] = I;
4395       } else if (Op.isRegOrImm()) {
4396         Op.addRegOrImmOperands(Inst, 1);
4397       } else {
4398         llvm_unreachable("unhandled operand type");
4399       }
4400     }
4401   } else {
4402     // No src modifiers
4403     for (unsigned E = Operands.size(); I != E; ++I) {
4404       AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[I]);
4405       if (Op.isMod()) {
4406         OptionalIdx[Op.getImmTy()] = I;
4407       } else {
4408         Op.addRegOrImmOperands(Inst, 1);
4409       }
4410     }
4411   }
4412 
4413   if (AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::clamp) != -1) {
4414     addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyClampSI);
4415   }
4416 
4417   if (AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::omod) != -1) {
4418     addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyOModSI);
4419   }
4420 
4421   // special case v_mac_{f16, f32}:
4422   // it has src2 register operand that is tied to dst operand
4423   // we don't allow modifiers for this operand in assembler so src2_modifiers
4424   // should be 0
4425   if (Opc == AMDGPU::V_MAC_F32_e64_si || Opc == AMDGPU::V_MAC_F32_e64_vi ||
4426       Opc == AMDGPU::V_MAC_F16_e64_vi) {
4427     auto it = Inst.begin();
4428     std::advance(it, AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src2_modifiers));
4429     it = Inst.insert(it, MCOperand::createImm(0)); // no modifiers for src2
4430     ++it;
4431     Inst.insert(it, Inst.getOperand(0)); // src2 = dst
4432   }
4433 }
4434 
4435 void AMDGPUAsmParser::cvtVOP3(MCInst &Inst, const OperandVector &Operands) {
4436   OptionalImmIndexMap OptionalIdx;
4437   cvtVOP3(Inst, Operands, OptionalIdx);
4438 }
4439 
4440 void AMDGPUAsmParser::cvtVOP3P(MCInst &Inst,
4441                                const OperandVector &Operands) {
4442   OptionalImmIndexMap OptIdx;
4443   const int Opc = Inst.getOpcode();
4444   const MCInstrDesc &Desc = MII.get(Opc);
4445 
4446   const bool IsPacked = (Desc.TSFlags & SIInstrFlags::IsPacked) != 0;
4447 
4448   cvtVOP3(Inst, Operands, OptIdx);
4449 
4450   if (AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::vdst_in) != -1) {
4451     assert(!IsPacked);
4452     Inst.addOperand(Inst.getOperand(0));
4453   }
4454 
4455   // FIXME: This is messy. Parse the modifiers as if it was a normal VOP3
4456   // instruction, and then figure out where to actually put the modifiers
4457 
4458   addOptionalImmOperand(Inst, Operands, OptIdx, AMDGPUOperand::ImmTyOpSel);
4459 
4460   int OpSelHiIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::op_sel_hi);
4461   if (OpSelHiIdx != -1) {
4462     int DefaultVal = IsPacked ? -1 : 0;
4463     addOptionalImmOperand(Inst, Operands, OptIdx, AMDGPUOperand::ImmTyOpSelHi,
4464                           DefaultVal);
4465   }
4466 
4467   int NegLoIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::neg_lo);
4468   if (NegLoIdx != -1) {
4469     assert(IsPacked);
4470     addOptionalImmOperand(Inst, Operands, OptIdx, AMDGPUOperand::ImmTyNegLo);
4471     addOptionalImmOperand(Inst, Operands, OptIdx, AMDGPUOperand::ImmTyNegHi);
4472   }
4473 
4474   const int Ops[] = { AMDGPU::OpName::src0,
4475                       AMDGPU::OpName::src1,
4476                       AMDGPU::OpName::src2 };
4477   const int ModOps[] = { AMDGPU::OpName::src0_modifiers,
4478                          AMDGPU::OpName::src1_modifiers,
4479                          AMDGPU::OpName::src2_modifiers };
4480 
4481   int OpSelIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::op_sel);
4482 
4483   unsigned OpSel = Inst.getOperand(OpSelIdx).getImm();
4484   unsigned OpSelHi = 0;
4485   unsigned NegLo = 0;
4486   unsigned NegHi = 0;
4487 
4488   if (OpSelHiIdx != -1) {
4489     OpSelHi = Inst.getOperand(OpSelHiIdx).getImm();
4490   }
4491 
4492   if (NegLoIdx != -1) {
4493     int NegHiIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::neg_hi);
4494     NegLo = Inst.getOperand(NegLoIdx).getImm();
4495     NegHi = Inst.getOperand(NegHiIdx).getImm();
4496   }
4497 
4498   for (int J = 0; J < 3; ++J) {
4499     int OpIdx = AMDGPU::getNamedOperandIdx(Opc, Ops[J]);
4500     if (OpIdx == -1)
4501       break;
4502 
4503     uint32_t ModVal = 0;
4504 
4505     if ((OpSel & (1 << J)) != 0)
4506       ModVal |= SISrcMods::OP_SEL_0;
4507 
4508     if ((OpSelHi & (1 << J)) != 0)
4509       ModVal |= SISrcMods::OP_SEL_1;
4510 
4511     if ((NegLo & (1 << J)) != 0)
4512       ModVal |= SISrcMods::NEG;
4513 
4514     if ((NegHi & (1 << J)) != 0)
4515       ModVal |= SISrcMods::NEG_HI;
4516 
4517     int ModIdx = AMDGPU::getNamedOperandIdx(Opc, ModOps[J]);
4518 
4519     Inst.getOperand(ModIdx).setImm(Inst.getOperand(ModIdx).getImm() | ModVal);
4520   }
4521 }
4522 
4523 //===----------------------------------------------------------------------===//
4524 // dpp
4525 //===----------------------------------------------------------------------===//
4526 
4527 bool AMDGPUOperand::isDPPCtrl() const {
4528   bool result = isImm() && getImmTy() == ImmTyDppCtrl && isUInt<9>(getImm());
4529   if (result) {
4530     int64_t Imm = getImm();
4531     return ((Imm >= 0x000) && (Imm <= 0x0ff)) ||
4532            ((Imm >= 0x101) && (Imm <= 0x10f)) ||
4533            ((Imm >= 0x111) && (Imm <= 0x11f)) ||
4534            ((Imm >= 0x121) && (Imm <= 0x12f)) ||
4535            (Imm == 0x130) ||
4536            (Imm == 0x134) ||
4537            (Imm == 0x138) ||
4538            (Imm == 0x13c) ||
4539            (Imm == 0x140) ||
4540            (Imm == 0x141) ||
4541            (Imm == 0x142) ||
4542            (Imm == 0x143);
4543   }
4544   return false;
4545 }
4546 
4547 bool AMDGPUOperand::isGPRIdxMode() const {
4548   return isImm() && isUInt<4>(getImm());
4549 }
4550 
4551 bool AMDGPUOperand::isS16Imm() const {
4552   return isImm() && (isInt<16>(getImm()) || isUInt<16>(getImm()));
4553 }
4554 
4555 bool AMDGPUOperand::isU16Imm() const {
4556   return isImm() && isUInt<16>(getImm());
4557 }
4558 
4559 OperandMatchResultTy
4560 AMDGPUAsmParser::parseDPPCtrl(OperandVector &Operands) {
4561   SMLoc S = Parser.getTok().getLoc();
4562   StringRef Prefix;
4563   int64_t Int;
4564 
4565   if (getLexer().getKind() == AsmToken::Identifier) {
4566     Prefix = Parser.getTok().getString();
4567   } else {
4568     return MatchOperand_NoMatch;
4569   }
4570 
4571   if (Prefix == "row_mirror") {
4572     Int = 0x140;
4573     Parser.Lex();
4574   } else if (Prefix == "row_half_mirror") {
4575     Int = 0x141;
4576     Parser.Lex();
4577   } else {
4578     // Check to prevent parseDPPCtrlOps from eating invalid tokens
4579     if (Prefix != "quad_perm"
4580         && Prefix != "row_shl"
4581         && Prefix != "row_shr"
4582         && Prefix != "row_ror"
4583         && Prefix != "wave_shl"
4584         && Prefix != "wave_rol"
4585         && Prefix != "wave_shr"
4586         && Prefix != "wave_ror"
4587         && Prefix != "row_bcast") {
4588       return MatchOperand_NoMatch;
4589     }
4590 
4591     Parser.Lex();
4592     if (getLexer().isNot(AsmToken::Colon))
4593       return MatchOperand_ParseFail;
4594 
4595     if (Prefix == "quad_perm") {
4596       // quad_perm:[%d,%d,%d,%d]
4597       Parser.Lex();
4598       if (getLexer().isNot(AsmToken::LBrac))
4599         return MatchOperand_ParseFail;
4600       Parser.Lex();
4601 
4602       if (getParser().parseAbsoluteExpression(Int) || !(0 <= Int && Int <=3))
4603         return MatchOperand_ParseFail;
4604 
4605       for (int i = 0; i < 3; ++i) {
4606         if (getLexer().isNot(AsmToken::Comma))
4607           return MatchOperand_ParseFail;
4608         Parser.Lex();
4609 
4610         int64_t Temp;
4611         if (getParser().parseAbsoluteExpression(Temp) || !(0 <= Temp && Temp <=3))
4612           return MatchOperand_ParseFail;
4613         const int shift = i*2 + 2;
4614         Int += (Temp << shift);
4615       }
4616 
4617       if (getLexer().isNot(AsmToken::RBrac))
4618         return MatchOperand_ParseFail;
4619       Parser.Lex();
4620     } else {
4621       // sel:%d
4622       Parser.Lex();
4623       if (getParser().parseAbsoluteExpression(Int))
4624         return MatchOperand_ParseFail;
4625 
4626       if (Prefix == "row_shl" && 1 <= Int && Int <= 15) {
4627         Int |= 0x100;
4628       } else if (Prefix == "row_shr" && 1 <= Int && Int <= 15) {
4629         Int |= 0x110;
4630       } else if (Prefix == "row_ror" && 1 <= Int && Int <= 15) {
4631         Int |= 0x120;
4632       } else if (Prefix == "wave_shl" && 1 == Int) {
4633         Int = 0x130;
4634       } else if (Prefix == "wave_rol" && 1 == Int) {
4635         Int = 0x134;
4636       } else if (Prefix == "wave_shr" && 1 == Int) {
4637         Int = 0x138;
4638       } else if (Prefix == "wave_ror" && 1 == Int) {
4639         Int = 0x13C;
4640       } else if (Prefix == "row_bcast") {
4641         if (Int == 15) {
4642           Int = 0x142;
4643         } else if (Int == 31) {
4644           Int = 0x143;
4645         } else {
4646           return MatchOperand_ParseFail;
4647         }
4648       } else {
4649         return MatchOperand_ParseFail;
4650       }
4651     }
4652   }
4653 
4654   Operands.push_back(AMDGPUOperand::CreateImm(this, Int, S, AMDGPUOperand::ImmTyDppCtrl));
4655   return MatchOperand_Success;
4656 }
4657 
4658 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultRowMask() const {
4659   return AMDGPUOperand::CreateImm(this, 0xf, SMLoc(), AMDGPUOperand::ImmTyDppRowMask);
4660 }
4661 
4662 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultBankMask() const {
4663   return AMDGPUOperand::CreateImm(this, 0xf, SMLoc(), AMDGPUOperand::ImmTyDppBankMask);
4664 }
4665 
4666 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultBoundCtrl() const {
4667   return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTyDppBoundCtrl);
4668 }
4669 
4670 void AMDGPUAsmParser::cvtDPP(MCInst &Inst, const OperandVector &Operands) {
4671   OptionalImmIndexMap OptionalIdx;
4672 
4673   unsigned I = 1;
4674   const MCInstrDesc &Desc = MII.get(Inst.getOpcode());
4675   for (unsigned J = 0; J < Desc.getNumDefs(); ++J) {
4676     ((AMDGPUOperand &)*Operands[I++]).addRegOperands(Inst, 1);
4677   }
4678 
4679   // All DPP instructions with at least one source operand have a fake "old"
4680   // source at the beginning that's tied to the dst operand. Handle it here.
4681   if (Desc.getNumOperands() >= 2)
4682     Inst.addOperand(Inst.getOperand(0));
4683 
4684   for (unsigned E = Operands.size(); I != E; ++I) {
4685     AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[I]);
4686     // Add the register arguments
4687     if (Op.isReg() && Op.Reg.RegNo == AMDGPU::VCC) {
4688       // VOP2b (v_add_u32, v_sub_u32 ...) dpp use "vcc" token.
4689       // Skip it.
4690       continue;
4691     } if (isRegOrImmWithInputMods(Desc, Inst.getNumOperands())) {
4692       Op.addRegWithFPInputModsOperands(Inst, 2);
4693     } else if (Op.isDPPCtrl()) {
4694       Op.addImmOperands(Inst, 1);
4695     } else if (Op.isImm()) {
4696       // Handle optional arguments
4697       OptionalIdx[Op.getImmTy()] = I;
4698     } else {
4699       llvm_unreachable("Invalid operand type");
4700     }
4701   }
4702 
4703   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyDppRowMask, 0xf);
4704   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyDppBankMask, 0xf);
4705   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyDppBoundCtrl);
4706 }
4707 
4708 //===----------------------------------------------------------------------===//
4709 // sdwa
4710 //===----------------------------------------------------------------------===//
4711 
4712 OperandMatchResultTy
4713 AMDGPUAsmParser::parseSDWASel(OperandVector &Operands, StringRef Prefix,
4714                               AMDGPUOperand::ImmTy Type) {
4715   using namespace llvm::AMDGPU::SDWA;
4716 
4717   SMLoc S = Parser.getTok().getLoc();
4718   StringRef Value;
4719   OperandMatchResultTy res;
4720 
4721   res = parseStringWithPrefix(Prefix, Value);
4722   if (res != MatchOperand_Success) {
4723     return res;
4724   }
4725 
4726   int64_t Int;
4727   Int = StringSwitch<int64_t>(Value)
4728         .Case("BYTE_0", SdwaSel::BYTE_0)
4729         .Case("BYTE_1", SdwaSel::BYTE_1)
4730         .Case("BYTE_2", SdwaSel::BYTE_2)
4731         .Case("BYTE_3", SdwaSel::BYTE_3)
4732         .Case("WORD_0", SdwaSel::WORD_0)
4733         .Case("WORD_1", SdwaSel::WORD_1)
4734         .Case("DWORD", SdwaSel::DWORD)
4735         .Default(0xffffffff);
4736   Parser.Lex(); // eat last token
4737 
4738   if (Int == 0xffffffff) {
4739     return MatchOperand_ParseFail;
4740   }
4741 
4742   Operands.push_back(AMDGPUOperand::CreateImm(this, Int, S, Type));
4743   return MatchOperand_Success;
4744 }
4745 
4746 OperandMatchResultTy
4747 AMDGPUAsmParser::parseSDWADstUnused(OperandVector &Operands) {
4748   using namespace llvm::AMDGPU::SDWA;
4749 
4750   SMLoc S = Parser.getTok().getLoc();
4751   StringRef Value;
4752   OperandMatchResultTy res;
4753 
4754   res = parseStringWithPrefix("dst_unused", Value);
4755   if (res != MatchOperand_Success) {
4756     return res;
4757   }
4758 
4759   int64_t Int;
4760   Int = StringSwitch<int64_t>(Value)
4761         .Case("UNUSED_PAD", DstUnused::UNUSED_PAD)
4762         .Case("UNUSED_SEXT", DstUnused::UNUSED_SEXT)
4763         .Case("UNUSED_PRESERVE", DstUnused::UNUSED_PRESERVE)
4764         .Default(0xffffffff);
4765   Parser.Lex(); // eat last token
4766 
4767   if (Int == 0xffffffff) {
4768     return MatchOperand_ParseFail;
4769   }
4770 
4771   Operands.push_back(AMDGPUOperand::CreateImm(this, Int, S, AMDGPUOperand::ImmTySdwaDstUnused));
4772   return MatchOperand_Success;
4773 }
4774 
4775 void AMDGPUAsmParser::cvtSdwaVOP1(MCInst &Inst, const OperandVector &Operands) {
4776   cvtSDWA(Inst, Operands, SIInstrFlags::VOP1);
4777 }
4778 
4779 void AMDGPUAsmParser::cvtSdwaVOP2(MCInst &Inst, const OperandVector &Operands) {
4780   cvtSDWA(Inst, Operands, SIInstrFlags::VOP2);
4781 }
4782 
4783 void AMDGPUAsmParser::cvtSdwaVOP2b(MCInst &Inst, const OperandVector &Operands) {
4784   cvtSDWA(Inst, Operands, SIInstrFlags::VOP2, true);
4785 }
4786 
4787 void AMDGPUAsmParser::cvtSdwaVOPC(MCInst &Inst, const OperandVector &Operands) {
4788   cvtSDWA(Inst, Operands, SIInstrFlags::VOPC, isVI());
4789 }
4790 
4791 void AMDGPUAsmParser::cvtSDWA(MCInst &Inst, const OperandVector &Operands,
4792                               uint64_t BasicInstType, bool skipVcc) {
4793   using namespace llvm::AMDGPU::SDWA;
4794 
4795   OptionalImmIndexMap OptionalIdx;
4796   bool skippedVcc = false;
4797 
4798   unsigned I = 1;
4799   const MCInstrDesc &Desc = MII.get(Inst.getOpcode());
4800   for (unsigned J = 0; J < Desc.getNumDefs(); ++J) {
4801     ((AMDGPUOperand &)*Operands[I++]).addRegOperands(Inst, 1);
4802   }
4803 
4804   for (unsigned E = Operands.size(); I != E; ++I) {
4805     AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[I]);
4806     if (skipVcc && !skippedVcc && Op.isReg() && Op.Reg.RegNo == AMDGPU::VCC) {
4807       // VOP2b (v_add_u32, v_sub_u32 ...) sdwa use "vcc" token as dst.
4808       // Skip it if it's 2nd (e.g. v_add_i32_sdwa v1, vcc, v2, v3)
4809       // or 4th (v_addc_u32_sdwa v1, vcc, v2, v3, vcc) operand.
4810       // Skip VCC only if we didn't skip it on previous iteration.
4811       if (BasicInstType == SIInstrFlags::VOP2 &&
4812           (Inst.getNumOperands() == 1 || Inst.getNumOperands() == 5)) {
4813         skippedVcc = true;
4814         continue;
4815       } else if (BasicInstType == SIInstrFlags::VOPC &&
4816                  Inst.getNumOperands() == 0) {
4817         skippedVcc = true;
4818         continue;
4819       }
4820     }
4821     if (isRegOrImmWithInputMods(Desc, Inst.getNumOperands())) {
4822       Op.addRegOrImmWithInputModsOperands(Inst, 2);
4823     } else if (Op.isImm()) {
4824       // Handle optional arguments
4825       OptionalIdx[Op.getImmTy()] = I;
4826     } else {
4827       llvm_unreachable("Invalid operand type");
4828     }
4829     skippedVcc = false;
4830   }
4831 
4832   if (Inst.getOpcode() != AMDGPU::V_NOP_sdwa_gfx9 &&
4833       Inst.getOpcode() != AMDGPU::V_NOP_sdwa_vi) {
4834     // v_nop_sdwa_sdwa_vi/gfx9 has no optional sdwa arguments
4835     switch (BasicInstType) {
4836     case SIInstrFlags::VOP1:
4837       addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyClampSI, 0);
4838       if (AMDGPU::getNamedOperandIdx(Inst.getOpcode(), AMDGPU::OpName::omod) != -1) {
4839         addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyOModSI, 0);
4840       }
4841       addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySdwaDstSel, SdwaSel::DWORD);
4842       addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySdwaDstUnused, DstUnused::UNUSED_PRESERVE);
4843       addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySdwaSrc0Sel, SdwaSel::DWORD);
4844       break;
4845 
4846     case SIInstrFlags::VOP2:
4847       addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyClampSI, 0);
4848       if (AMDGPU::getNamedOperandIdx(Inst.getOpcode(), AMDGPU::OpName::omod) != -1) {
4849         addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyOModSI, 0);
4850       }
4851       addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySdwaDstSel, SdwaSel::DWORD);
4852       addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySdwaDstUnused, DstUnused::UNUSED_PRESERVE);
4853       addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySdwaSrc0Sel, SdwaSel::DWORD);
4854       addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySdwaSrc1Sel, SdwaSel::DWORD);
4855       break;
4856 
4857     case SIInstrFlags::VOPC:
4858       addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyClampSI, 0);
4859       addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySdwaSrc0Sel, SdwaSel::DWORD);
4860       addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySdwaSrc1Sel, SdwaSel::DWORD);
4861       break;
4862 
4863     default:
4864       llvm_unreachable("Invalid instruction type. Only VOP1, VOP2 and VOPC allowed");
4865     }
4866   }
4867 
4868   // special case v_mac_{f16, f32}:
4869   // it has src2 register operand that is tied to dst operand
4870   if (Inst.getOpcode() == AMDGPU::V_MAC_F32_sdwa_vi ||
4871       Inst.getOpcode() == AMDGPU::V_MAC_F16_sdwa_vi)  {
4872     auto it = Inst.begin();
4873     std::advance(
4874       it, AMDGPU::getNamedOperandIdx(Inst.getOpcode(), AMDGPU::OpName::src2));
4875     Inst.insert(it, Inst.getOperand(0)); // src2 = dst
4876   }
4877 }
4878 
4879 /// Force static initialization.
4880 extern "C" void LLVMInitializeAMDGPUAsmParser() {
4881   RegisterMCAsmParser<AMDGPUAsmParser> A(getTheAMDGPUTarget());
4882   RegisterMCAsmParser<AMDGPUAsmParser> B(getTheGCNTarget());
4883 }
4884 
4885 #define GET_REGISTER_MATCHER
4886 #define GET_MATCHER_IMPLEMENTATION
4887 #define GET_MNEMONIC_SPELL_CHECKER
4888 #include "AMDGPUGenAsmMatcher.inc"
4889 
4890 // This fuction should be defined after auto-generated include so that we have
4891 // MatchClassKind enum defined
4892 unsigned AMDGPUAsmParser::validateTargetOperandClass(MCParsedAsmOperand &Op,
4893                                                      unsigned Kind) {
4894   // Tokens like "glc" would be parsed as immediate operands in ParseOperand().
4895   // But MatchInstructionImpl() expects to meet token and fails to validate
4896   // operand. This method checks if we are given immediate operand but expect to
4897   // get corresponding token.
4898   AMDGPUOperand &Operand = (AMDGPUOperand&)Op;
4899   switch (Kind) {
4900   case MCK_addr64:
4901     return Operand.isAddr64() ? Match_Success : Match_InvalidOperand;
4902   case MCK_gds:
4903     return Operand.isGDS() ? Match_Success : Match_InvalidOperand;
4904   case MCK_glc:
4905     return Operand.isGLC() ? Match_Success : Match_InvalidOperand;
4906   case MCK_idxen:
4907     return Operand.isIdxen() ? Match_Success : Match_InvalidOperand;
4908   case MCK_offen:
4909     return Operand.isOffen() ? Match_Success : Match_InvalidOperand;
4910   case MCK_SSrcB32:
4911     // When operands have expression values, they will return true for isToken,
4912     // because it is not possible to distinguish between a token and an
4913     // expression at parse time. MatchInstructionImpl() will always try to
4914     // match an operand as a token, when isToken returns true, and when the
4915     // name of the expression is not a valid token, the match will fail,
4916     // so we need to handle it here.
4917     return Operand.isSSrcB32() ? Match_Success : Match_InvalidOperand;
4918   case MCK_SSrcF32:
4919     return Operand.isSSrcF32() ? Match_Success : Match_InvalidOperand;
4920   case MCK_SoppBrTarget:
4921     return Operand.isSoppBrTarget() ? Match_Success : Match_InvalidOperand;
4922   case MCK_VReg32OrOff:
4923     return Operand.isVReg32OrOff() ? Match_Success : Match_InvalidOperand;
4924   case MCK_InterpSlot:
4925     return Operand.isInterpSlot() ? Match_Success : Match_InvalidOperand;
4926   case MCK_Attr:
4927     return Operand.isInterpAttr() ? Match_Success : Match_InvalidOperand;
4928   case MCK_AttrChan:
4929     return Operand.isAttrChan() ? Match_Success : Match_InvalidOperand;
4930   default:
4931     return Match_InvalidOperand;
4932   }
4933 }
4934