1 //===-- AMDGPUAsmParser.cpp - Parse SI asm to MCInst instructions ---------===//
2 //
3 //                     The LLVM Compiler Infrastructure
4 //
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
7 //
8 //===----------------------------------------------------------------------===//
9 
10 #include "AMDKernelCodeT.h"
11 #include "MCTargetDesc/AMDGPUMCTargetDesc.h"
12 #include "MCTargetDesc/AMDGPUTargetStreamer.h"
13 #include "SIDefines.h"
14 #include "Utils/AMDGPUBaseInfo.h"
15 #include "Utils/AMDKernelCodeTUtils.h"
16 #include "Utils/AMDGPUAsmUtils.h"
17 #include "llvm/ADT/APFloat.h"
18 #include "llvm/ADT/APInt.h"
19 #include "llvm/ADT/ArrayRef.h"
20 #include "llvm/ADT/SmallBitVector.h"
21 #include "llvm/ADT/SmallString.h"
22 #include "llvm/ADT/STLExtras.h"
23 #include "llvm/ADT/StringRef.h"
24 #include "llvm/ADT/StringSwitch.h"
25 #include "llvm/ADT/Twine.h"
26 #include "llvm/CodeGen/MachineValueType.h"
27 #include "llvm/MC/MCAsmInfo.h"
28 #include "llvm/MC/MCContext.h"
29 #include "llvm/MC/MCExpr.h"
30 #include "llvm/MC/MCInst.h"
31 #include "llvm/MC/MCInstrDesc.h"
32 #include "llvm/MC/MCInstrInfo.h"
33 #include "llvm/MC/MCParser/MCAsmLexer.h"
34 #include "llvm/MC/MCParser/MCAsmParser.h"
35 #include "llvm/MC/MCParser/MCAsmParserExtension.h"
36 #include "llvm/MC/MCParser/MCParsedAsmOperand.h"
37 #include "llvm/MC/MCParser/MCTargetAsmParser.h"
38 #include "llvm/MC/MCRegisterInfo.h"
39 #include "llvm/MC/MCStreamer.h"
40 #include "llvm/MC/MCSubtargetInfo.h"
41 #include "llvm/MC/MCSymbol.h"
42 #include "llvm/Support/Casting.h"
43 #include "llvm/Support/ELF.h"
44 #include "llvm/Support/ErrorHandling.h"
45 #include "llvm/Support/MathExtras.h"
46 #include "llvm/Support/raw_ostream.h"
47 #include "llvm/Support/SMLoc.h"
48 #include "llvm/Support/TargetRegistry.h"
49 #include <algorithm>
50 #include <cassert>
51 #include <cstdint>
52 #include <cstring>
53 #include <iterator>
54 #include <map>
55 #include <memory>
56 #include <string>
57 
58 using namespace llvm;
59 using namespace llvm::AMDGPU;
60 
61 namespace {
62 
63 class AMDGPUAsmParser;
64 
65 enum RegisterKind { IS_UNKNOWN, IS_VGPR, IS_SGPR, IS_TTMP, IS_SPECIAL };
66 
67 //===----------------------------------------------------------------------===//
68 // Operand
69 //===----------------------------------------------------------------------===//
70 
71 class AMDGPUOperand : public MCParsedAsmOperand {
72   enum KindTy {
73     Token,
74     Immediate,
75     Register,
76     Expression
77   } Kind;
78 
79   SMLoc StartLoc, EndLoc;
80   const AMDGPUAsmParser *AsmParser;
81 
82 public:
83   AMDGPUOperand(KindTy Kind_, const AMDGPUAsmParser *AsmParser_)
84     : MCParsedAsmOperand(), Kind(Kind_), AsmParser(AsmParser_) {}
85 
86   typedef std::unique_ptr<AMDGPUOperand> Ptr;
87 
88   struct Modifiers {
89     bool Abs = false;
90     bool Neg = false;
91     bool Sext = false;
92 
93     bool hasFPModifiers() const { return Abs || Neg; }
94     bool hasIntModifiers() const { return Sext; }
95     bool hasModifiers() const { return hasFPModifiers() || hasIntModifiers(); }
96 
97     int64_t getFPModifiersOperand() const {
98       int64_t Operand = 0;
99       Operand |= Abs ? SISrcMods::ABS : 0;
100       Operand |= Neg ? SISrcMods::NEG : 0;
101       return Operand;
102     }
103 
104     int64_t getIntModifiersOperand() const {
105       int64_t Operand = 0;
106       Operand |= Sext ? SISrcMods::SEXT : 0;
107       return Operand;
108     }
109 
110     int64_t getModifiersOperand() const {
111       assert(!(hasFPModifiers() && hasIntModifiers())
112            && "fp and int modifiers should not be used simultaneously");
113       if (hasFPModifiers()) {
114         return getFPModifiersOperand();
115       } else if (hasIntModifiers()) {
116         return getIntModifiersOperand();
117       } else {
118         return 0;
119       }
120     }
121 
122     friend raw_ostream &operator <<(raw_ostream &OS, AMDGPUOperand::Modifiers Mods);
123   };
124 
125   enum ImmTy {
126     ImmTyNone,
127     ImmTyGDS,
128     ImmTyOffen,
129     ImmTyIdxen,
130     ImmTyAddr64,
131     ImmTyOffset,
132     ImmTyOffset0,
133     ImmTyOffset1,
134     ImmTyGLC,
135     ImmTySLC,
136     ImmTyTFE,
137     ImmTyClampSI,
138     ImmTyOModSI,
139     ImmTyDppCtrl,
140     ImmTyDppRowMask,
141     ImmTyDppBankMask,
142     ImmTyDppBoundCtrl,
143     ImmTySdwaDstSel,
144     ImmTySdwaSrc0Sel,
145     ImmTySdwaSrc1Sel,
146     ImmTySdwaDstUnused,
147     ImmTyDMask,
148     ImmTyUNorm,
149     ImmTyDA,
150     ImmTyR128,
151     ImmTyLWE,
152     ImmTyExpTgt,
153     ImmTyExpCompr,
154     ImmTyExpVM,
155     ImmTyHwreg,
156     ImmTyOff,
157     ImmTySendMsg,
158     ImmTyInterpSlot,
159     ImmTyInterpAttr,
160     ImmTyAttrChan,
161     ImmTyOpSel,
162     ImmTyOpSelHi,
163     ImmTyNegLo,
164     ImmTyNegHi
165   };
166 
167   struct TokOp {
168     const char *Data;
169     unsigned Length;
170   };
171 
172   struct ImmOp {
173     int64_t Val;
174     ImmTy Type;
175     bool IsFPImm;
176     Modifiers Mods;
177   };
178 
179   struct RegOp {
180     unsigned RegNo;
181     bool IsForcedVOP3;
182     Modifiers Mods;
183   };
184 
185   union {
186     TokOp Tok;
187     ImmOp Imm;
188     RegOp Reg;
189     const MCExpr *Expr;
190   };
191 
192   bool isToken() const override {
193     if (Kind == Token)
194       return true;
195 
196     if (Kind != Expression || !Expr)
197       return false;
198 
199     // When parsing operands, we can't always tell if something was meant to be
200     // a token, like 'gds', or an expression that references a global variable.
201     // In this case, we assume the string is an expression, and if we need to
202     // interpret is a token, then we treat the symbol name as the token.
203     return isa<MCSymbolRefExpr>(Expr);
204   }
205 
206   bool isImm() const override {
207     return Kind == Immediate;
208   }
209 
210   bool isInlinableImm(MVT type) const;
211   bool isLiteralImm(MVT type) const;
212 
213   bool isRegKind() const {
214     return Kind == Register;
215   }
216 
217   bool isReg() const override {
218     return isRegKind() && !hasModifiers();
219   }
220 
221   bool isRegOrImmWithInputMods(MVT type) const {
222     return isRegKind() || isInlinableImm(type);
223   }
224 
225   bool isRegOrImmWithInt16InputMods() const {
226     return isRegOrImmWithInputMods(MVT::i16);
227   }
228 
229   bool isRegOrImmWithInt32InputMods() const {
230     return isRegOrImmWithInputMods(MVT::i32);
231   }
232 
233   bool isRegOrImmWithInt64InputMods() const {
234     return isRegOrImmWithInputMods(MVT::i64);
235   }
236 
237   bool isRegOrImmWithFP16InputMods() const {
238     return isRegOrImmWithInputMods(MVT::f16);
239   }
240 
241   bool isRegOrImmWithFP32InputMods() const {
242     return isRegOrImmWithInputMods(MVT::f32);
243   }
244 
245   bool isRegOrImmWithFP64InputMods() const {
246     return isRegOrImmWithInputMods(MVT::f64);
247   }
248 
249   bool isVReg() const {
250     return isRegClass(AMDGPU::VGPR_32RegClassID) ||
251            isRegClass(AMDGPU::VReg_64RegClassID) ||
252            isRegClass(AMDGPU::VReg_96RegClassID) ||
253            isRegClass(AMDGPU::VReg_128RegClassID) ||
254            isRegClass(AMDGPU::VReg_256RegClassID) ||
255            isRegClass(AMDGPU::VReg_512RegClassID);
256   }
257 
258   bool isVReg32OrOff() const {
259     return isOff() || isRegClass(AMDGPU::VGPR_32RegClassID);
260   }
261 
262   bool isImmTy(ImmTy ImmT) const {
263     return isImm() && Imm.Type == ImmT;
264   }
265 
266   bool isImmModifier() const {
267     return isImm() && Imm.Type != ImmTyNone;
268   }
269 
270   bool isClampSI() const { return isImmTy(ImmTyClampSI); }
271   bool isOModSI() const { return isImmTy(ImmTyOModSI); }
272   bool isDMask() const { return isImmTy(ImmTyDMask); }
273   bool isUNorm() const { return isImmTy(ImmTyUNorm); }
274   bool isDA() const { return isImmTy(ImmTyDA); }
275   bool isR128() const { return isImmTy(ImmTyUNorm); }
276   bool isLWE() const { return isImmTy(ImmTyLWE); }
277   bool isOff() const { return isImmTy(ImmTyOff); }
278   bool isExpTgt() const { return isImmTy(ImmTyExpTgt); }
279   bool isExpVM() const { return isImmTy(ImmTyExpVM); }
280   bool isExpCompr() const { return isImmTy(ImmTyExpCompr); }
281   bool isOffen() const { return isImmTy(ImmTyOffen); }
282   bool isIdxen() const { return isImmTy(ImmTyIdxen); }
283   bool isAddr64() const { return isImmTy(ImmTyAddr64); }
284   bool isOffset() const { return isImmTy(ImmTyOffset) && isUInt<16>(getImm()); }
285   bool isOffset0() const { return isImmTy(ImmTyOffset0) && isUInt<16>(getImm()); }
286   bool isOffset1() const { return isImmTy(ImmTyOffset1) && isUInt<8>(getImm()); }
287   bool isGDS() const { return isImmTy(ImmTyGDS); }
288   bool isGLC() const { return isImmTy(ImmTyGLC); }
289   bool isSLC() const { return isImmTy(ImmTySLC); }
290   bool isTFE() const { return isImmTy(ImmTyTFE); }
291   bool isBankMask() const { return isImmTy(ImmTyDppBankMask); }
292   bool isRowMask() const { return isImmTy(ImmTyDppRowMask); }
293   bool isBoundCtrl() const { return isImmTy(ImmTyDppBoundCtrl); }
294   bool isSDWADstSel() const { return isImmTy(ImmTySdwaDstSel); }
295   bool isSDWASrc0Sel() const { return isImmTy(ImmTySdwaSrc0Sel); }
296   bool isSDWASrc1Sel() const { return isImmTy(ImmTySdwaSrc1Sel); }
297   bool isSDWADstUnused() const { return isImmTy(ImmTySdwaDstUnused); }
298   bool isInterpSlot() const { return isImmTy(ImmTyInterpSlot); }
299   bool isInterpAttr() const { return isImmTy(ImmTyInterpAttr); }
300   bool isAttrChan() const { return isImmTy(ImmTyAttrChan); }
301   bool isOpSel() const { return isImmTy(ImmTyOpSel); }
302   bool isOpSelHi() const { return isImmTy(ImmTyOpSelHi); }
303   bool isNegLo() const { return isImmTy(ImmTyNegLo); }
304   bool isNegHi() const { return isImmTy(ImmTyNegHi); }
305 
306   bool isMod() const {
307     return isClampSI() || isOModSI();
308   }
309 
310   bool isRegOrImm() const {
311     return isReg() || isImm();
312   }
313 
314   bool isRegClass(unsigned RCID) const;
315 
316   bool isRegOrInlineNoMods(unsigned RCID, MVT type) const {
317     return (isRegClass(RCID) || isInlinableImm(type)) && !hasModifiers();
318   }
319 
320   bool isSCSrcB16() const {
321     return isRegOrInlineNoMods(AMDGPU::SReg_32RegClassID, MVT::i16);
322   }
323 
324   bool isSCSrcV2B16() const {
325     return isSCSrcB16();
326   }
327 
328   bool isSCSrcB32() const {
329     return isRegOrInlineNoMods(AMDGPU::SReg_32RegClassID, MVT::i32);
330   }
331 
332   bool isSCSrcB64() const {
333     return isRegOrInlineNoMods(AMDGPU::SReg_64RegClassID, MVT::i64);
334   }
335 
336   bool isSCSrcF16() const {
337     return isRegOrInlineNoMods(AMDGPU::SReg_32RegClassID, MVT::f16);
338   }
339 
340   bool isSCSrcV2F16() const {
341     return isSCSrcF16();
342   }
343 
344   bool isSCSrcF32() const {
345     return isRegOrInlineNoMods(AMDGPU::SReg_32RegClassID, MVT::f32);
346   }
347 
348   bool isSCSrcF64() const {
349     return isRegOrInlineNoMods(AMDGPU::SReg_64RegClassID, MVT::f64);
350   }
351 
352   bool isSSrcB32() const {
353     return isSCSrcB32() || isLiteralImm(MVT::i32) || isExpr();
354   }
355 
356   bool isSSrcB16() const {
357     return isSCSrcB16() || isLiteralImm(MVT::i16);
358   }
359 
360   bool isSSrcV2B16() const {
361     llvm_unreachable("cannot happen");
362     return isSSrcB16();
363   }
364 
365   bool isSSrcB64() const {
366     // TODO: Find out how SALU supports extension of 32-bit literals to 64 bits.
367     // See isVSrc64().
368     return isSCSrcB64() || isLiteralImm(MVT::i64);
369   }
370 
371   bool isSSrcF32() const {
372     return isSCSrcB32() || isLiteralImm(MVT::f32) || isExpr();
373   }
374 
375   bool isSSrcF64() const {
376     return isSCSrcB64() || isLiteralImm(MVT::f64);
377   }
378 
379   bool isSSrcF16() const {
380     return isSCSrcB16() || isLiteralImm(MVT::f16);
381   }
382 
383   bool isSSrcV2F16() const {
384     llvm_unreachable("cannot happen");
385     return isSSrcF16();
386   }
387 
388   bool isVCSrcB32() const {
389     return isRegOrInlineNoMods(AMDGPU::VS_32RegClassID, MVT::i32);
390   }
391 
392   bool isVCSrcB64() const {
393     return isRegOrInlineNoMods(AMDGPU::VS_64RegClassID, MVT::i64);
394   }
395 
396   bool isVCSrcB16() const {
397     return isRegOrInlineNoMods(AMDGPU::VS_32RegClassID, MVT::i16);
398   }
399 
400   bool isVCSrcV2B16() const {
401     return isVCSrcB16();
402   }
403 
404   bool isVCSrcF32() const {
405     return isRegOrInlineNoMods(AMDGPU::VS_32RegClassID, MVT::f32);
406   }
407 
408   bool isVCSrcF64() const {
409     return isRegOrInlineNoMods(AMDGPU::VS_64RegClassID, MVT::f64);
410   }
411 
412   bool isVCSrcF16() const {
413     return isRegOrInlineNoMods(AMDGPU::VS_32RegClassID, MVT::f16);
414   }
415 
416   bool isVCSrcV2F16() const {
417     return isVCSrcF16();
418   }
419 
420   bool isVSrcB32() const {
421     return isVCSrcF32() || isLiteralImm(MVT::i32);
422   }
423 
424   bool isVSrcB64() const {
425     return isVCSrcF64() || isLiteralImm(MVT::i64);
426   }
427 
428   bool isVSrcB16() const {
429     return isVCSrcF16() || isLiteralImm(MVT::i16);
430   }
431 
432   bool isVSrcV2B16() const {
433     llvm_unreachable("cannot happen");
434     return isVSrcB16();
435   }
436 
437   bool isVSrcF32() const {
438     return isVCSrcF32() || isLiteralImm(MVT::f32);
439   }
440 
441   bool isVSrcF64() const {
442     return isVCSrcF64() || isLiteralImm(MVT::f64);
443   }
444 
445   bool isVSrcF16() const {
446     return isVCSrcF16() || isLiteralImm(MVT::f16);
447   }
448 
449   bool isVSrcV2F16() const {
450     llvm_unreachable("cannot happen");
451     return isVSrcF16();
452   }
453 
454   bool isKImmFP32() const {
455     return isLiteralImm(MVT::f32);
456   }
457 
458   bool isKImmFP16() const {
459     return isLiteralImm(MVT::f16);
460   }
461 
462   bool isMem() const override {
463     return false;
464   }
465 
466   bool isExpr() const {
467     return Kind == Expression;
468   }
469 
470   bool isSoppBrTarget() const {
471     return isExpr() || isImm();
472   }
473 
474   bool isSWaitCnt() const;
475   bool isHwreg() const;
476   bool isSendMsg() const;
477   bool isSMRDOffset8() const;
478   bool isSMRDOffset20() const;
479   bool isSMRDLiteralOffset() const;
480   bool isDPPCtrl() const;
481   bool isGPRIdxMode() const;
482 
483   StringRef getExpressionAsToken() const {
484     assert(isExpr());
485     const MCSymbolRefExpr *S = cast<MCSymbolRefExpr>(Expr);
486     return S->getSymbol().getName();
487   }
488 
489   StringRef getToken() const {
490     assert(isToken());
491 
492     if (Kind == Expression)
493       return getExpressionAsToken();
494 
495     return StringRef(Tok.Data, Tok.Length);
496   }
497 
498   int64_t getImm() const {
499     assert(isImm());
500     return Imm.Val;
501   }
502 
503   ImmTy getImmTy() const {
504     assert(isImm());
505     return Imm.Type;
506   }
507 
508   unsigned getReg() const override {
509     return Reg.RegNo;
510   }
511 
512   SMLoc getStartLoc() const override {
513     return StartLoc;
514   }
515 
516   SMLoc getEndLoc() const override {
517     return EndLoc;
518   }
519 
520   Modifiers getModifiers() const {
521     assert(isRegKind() || isImmTy(ImmTyNone));
522     return isRegKind() ? Reg.Mods : Imm.Mods;
523   }
524 
525   void setModifiers(Modifiers Mods) {
526     assert(isRegKind() || isImmTy(ImmTyNone));
527     if (isRegKind())
528       Reg.Mods = Mods;
529     else
530       Imm.Mods = Mods;
531   }
532 
533   bool hasModifiers() const {
534     return getModifiers().hasModifiers();
535   }
536 
537   bool hasFPModifiers() const {
538     return getModifiers().hasFPModifiers();
539   }
540 
541   bool hasIntModifiers() const {
542     return getModifiers().hasIntModifiers();
543   }
544 
545   void addImmOperands(MCInst &Inst, unsigned N, bool ApplyModifiers = true) const;
546 
547   void addLiteralImmOperand(MCInst &Inst, int64_t Val) const;
548 
549   template <unsigned Bitwidth>
550   void addKImmFPOperands(MCInst &Inst, unsigned N) const;
551 
552   void addKImmFP16Operands(MCInst &Inst, unsigned N) const {
553     addKImmFPOperands<16>(Inst, N);
554   }
555 
556   void addKImmFP32Operands(MCInst &Inst, unsigned N) const {
557     addKImmFPOperands<32>(Inst, N);
558   }
559 
560   void addRegOperands(MCInst &Inst, unsigned N) const;
561 
562   void addRegOrImmOperands(MCInst &Inst, unsigned N) const {
563     if (isRegKind())
564       addRegOperands(Inst, N);
565     else if (isExpr())
566       Inst.addOperand(MCOperand::createExpr(Expr));
567     else
568       addImmOperands(Inst, N);
569   }
570 
571   void addRegOrImmWithInputModsOperands(MCInst &Inst, unsigned N) const {
572     Modifiers Mods = getModifiers();
573     Inst.addOperand(MCOperand::createImm(Mods.getModifiersOperand()));
574     if (isRegKind()) {
575       addRegOperands(Inst, N);
576     } else {
577       addImmOperands(Inst, N, false);
578     }
579   }
580 
581   void addRegOrImmWithFPInputModsOperands(MCInst &Inst, unsigned N) const {
582     assert(!hasIntModifiers());
583     addRegOrImmWithInputModsOperands(Inst, N);
584   }
585 
586   void addRegOrImmWithIntInputModsOperands(MCInst &Inst, unsigned N) const {
587     assert(!hasFPModifiers());
588     addRegOrImmWithInputModsOperands(Inst, N);
589   }
590 
591   void addRegWithInputModsOperands(MCInst &Inst, unsigned N) const {
592     Modifiers Mods = getModifiers();
593     Inst.addOperand(MCOperand::createImm(Mods.getModifiersOperand()));
594     assert(isRegKind());
595     addRegOperands(Inst, N);
596   }
597 
598   void addRegWithFPInputModsOperands(MCInst &Inst, unsigned N) const {
599     assert(!hasIntModifiers());
600     addRegWithInputModsOperands(Inst, N);
601   }
602 
603   void addRegWithIntInputModsOperands(MCInst &Inst, unsigned N) const {
604     assert(!hasFPModifiers());
605     addRegWithInputModsOperands(Inst, N);
606   }
607 
608   void addSoppBrTargetOperands(MCInst &Inst, unsigned N) const {
609     if (isImm())
610       addImmOperands(Inst, N);
611     else {
612       assert(isExpr());
613       Inst.addOperand(MCOperand::createExpr(Expr));
614     }
615   }
616 
617   static void printImmTy(raw_ostream& OS, ImmTy Type) {
618     switch (Type) {
619     case ImmTyNone: OS << "None"; break;
620     case ImmTyGDS: OS << "GDS"; break;
621     case ImmTyOffen: OS << "Offen"; break;
622     case ImmTyIdxen: OS << "Idxen"; break;
623     case ImmTyAddr64: OS << "Addr64"; break;
624     case ImmTyOffset: OS << "Offset"; break;
625     case ImmTyOffset0: OS << "Offset0"; break;
626     case ImmTyOffset1: OS << "Offset1"; break;
627     case ImmTyGLC: OS << "GLC"; break;
628     case ImmTySLC: OS << "SLC"; break;
629     case ImmTyTFE: OS << "TFE"; break;
630     case ImmTyClampSI: OS << "ClampSI"; break;
631     case ImmTyOModSI: OS << "OModSI"; break;
632     case ImmTyDppCtrl: OS << "DppCtrl"; break;
633     case ImmTyDppRowMask: OS << "DppRowMask"; break;
634     case ImmTyDppBankMask: OS << "DppBankMask"; break;
635     case ImmTyDppBoundCtrl: OS << "DppBoundCtrl"; break;
636     case ImmTySdwaDstSel: OS << "SdwaDstSel"; break;
637     case ImmTySdwaSrc0Sel: OS << "SdwaSrc0Sel"; break;
638     case ImmTySdwaSrc1Sel: OS << "SdwaSrc1Sel"; break;
639     case ImmTySdwaDstUnused: OS << "SdwaDstUnused"; break;
640     case ImmTyDMask: OS << "DMask"; break;
641     case ImmTyUNorm: OS << "UNorm"; break;
642     case ImmTyDA: OS << "DA"; break;
643     case ImmTyR128: OS << "R128"; break;
644     case ImmTyLWE: OS << "LWE"; break;
645     case ImmTyOff: OS << "Off"; break;
646     case ImmTyExpTgt: OS << "ExpTgt"; break;
647     case ImmTyExpCompr: OS << "ExpCompr"; break;
648     case ImmTyExpVM: OS << "ExpVM"; break;
649     case ImmTyHwreg: OS << "Hwreg"; break;
650     case ImmTySendMsg: OS << "SendMsg"; break;
651     case ImmTyInterpSlot: OS << "InterpSlot"; break;
652     case ImmTyInterpAttr: OS << "InterpAttr"; break;
653     case ImmTyAttrChan: OS << "AttrChan"; break;
654     case ImmTyOpSel: OS << "OpSel"; break;
655     case ImmTyOpSelHi: OS << "OpSelHi"; break;
656     case ImmTyNegLo: OS << "NegLo"; break;
657     case ImmTyNegHi: OS << "NegHi"; break;
658     }
659   }
660 
661   void print(raw_ostream &OS) const override {
662     switch (Kind) {
663     case Register:
664       OS << "<register " << getReg() << " mods: " << Reg.Mods << '>';
665       break;
666     case Immediate:
667       OS << '<' << getImm();
668       if (getImmTy() != ImmTyNone) {
669         OS << " type: "; printImmTy(OS, getImmTy());
670       }
671       OS << " mods: " << Imm.Mods << '>';
672       break;
673     case Token:
674       OS << '\'' << getToken() << '\'';
675       break;
676     case Expression:
677       OS << "<expr " << *Expr << '>';
678       break;
679     }
680   }
681 
682   static AMDGPUOperand::Ptr CreateImm(const AMDGPUAsmParser *AsmParser,
683                                       int64_t Val, SMLoc Loc,
684                                       ImmTy Type = ImmTyNone,
685                                       bool IsFPImm = false) {
686     auto Op = llvm::make_unique<AMDGPUOperand>(Immediate, AsmParser);
687     Op->Imm.Val = Val;
688     Op->Imm.IsFPImm = IsFPImm;
689     Op->Imm.Type = Type;
690     Op->Imm.Mods = Modifiers();
691     Op->StartLoc = Loc;
692     Op->EndLoc = Loc;
693     return Op;
694   }
695 
696   static AMDGPUOperand::Ptr CreateToken(const AMDGPUAsmParser *AsmParser,
697                                         StringRef Str, SMLoc Loc,
698                                         bool HasExplicitEncodingSize = true) {
699     auto Res = llvm::make_unique<AMDGPUOperand>(Token, AsmParser);
700     Res->Tok.Data = Str.data();
701     Res->Tok.Length = Str.size();
702     Res->StartLoc = Loc;
703     Res->EndLoc = Loc;
704     return Res;
705   }
706 
707   static AMDGPUOperand::Ptr CreateReg(const AMDGPUAsmParser *AsmParser,
708                                       unsigned RegNo, SMLoc S,
709                                       SMLoc E,
710                                       bool ForceVOP3) {
711     auto Op = llvm::make_unique<AMDGPUOperand>(Register, AsmParser);
712     Op->Reg.RegNo = RegNo;
713     Op->Reg.Mods = Modifiers();
714     Op->Reg.IsForcedVOP3 = ForceVOP3;
715     Op->StartLoc = S;
716     Op->EndLoc = E;
717     return Op;
718   }
719 
720   static AMDGPUOperand::Ptr CreateExpr(const AMDGPUAsmParser *AsmParser,
721                                        const class MCExpr *Expr, SMLoc S) {
722     auto Op = llvm::make_unique<AMDGPUOperand>(Expression, AsmParser);
723     Op->Expr = Expr;
724     Op->StartLoc = S;
725     Op->EndLoc = S;
726     return Op;
727   }
728 };
729 
730 raw_ostream &operator <<(raw_ostream &OS, AMDGPUOperand::Modifiers Mods) {
731   OS << "abs:" << Mods.Abs << " neg: " << Mods.Neg << " sext:" << Mods.Sext;
732   return OS;
733 }
734 
735 //===----------------------------------------------------------------------===//
736 // AsmParser
737 //===----------------------------------------------------------------------===//
738 
739 // Holds info related to the current kernel, e.g. count of SGPRs used.
740 // Kernel scope begins at .amdgpu_hsa_kernel directive, ends at next
741 // .amdgpu_hsa_kernel or at EOF.
742 class KernelScopeInfo {
743   int SgprIndexUnusedMin = -1;
744   int VgprIndexUnusedMin = -1;
745   MCContext *Ctx = nullptr;
746 
747   void usesSgprAt(int i) {
748     if (i >= SgprIndexUnusedMin) {
749       SgprIndexUnusedMin = ++i;
750       if (Ctx) {
751         MCSymbol * const Sym = Ctx->getOrCreateSymbol(Twine(".kernel.sgpr_count"));
752         Sym->setVariableValue(MCConstantExpr::create(SgprIndexUnusedMin, *Ctx));
753       }
754     }
755   }
756 
757   void usesVgprAt(int i) {
758     if (i >= VgprIndexUnusedMin) {
759       VgprIndexUnusedMin = ++i;
760       if (Ctx) {
761         MCSymbol * const Sym = Ctx->getOrCreateSymbol(Twine(".kernel.vgpr_count"));
762         Sym->setVariableValue(MCConstantExpr::create(VgprIndexUnusedMin, *Ctx));
763       }
764     }
765   }
766 
767 public:
768   KernelScopeInfo() = default;
769 
770   void initialize(MCContext &Context) {
771     Ctx = &Context;
772     usesSgprAt(SgprIndexUnusedMin = -1);
773     usesVgprAt(VgprIndexUnusedMin = -1);
774   }
775 
776   void usesRegister(RegisterKind RegKind, unsigned DwordRegIndex, unsigned RegWidth) {
777     switch (RegKind) {
778       case IS_SGPR: usesSgprAt(DwordRegIndex + RegWidth - 1); break;
779       case IS_VGPR: usesVgprAt(DwordRegIndex + RegWidth - 1); break;
780       default: break;
781     }
782   }
783 };
784 
785 class AMDGPUAsmParser : public MCTargetAsmParser {
786   const MCInstrInfo &MII;
787   MCAsmParser &Parser;
788 
789   unsigned ForcedEncodingSize = 0;
790   bool ForcedDPP = false;
791   bool ForcedSDWA = false;
792   KernelScopeInfo KernelScope;
793 
794   /// @name Auto-generated Match Functions
795   /// {
796 
797 #define GET_ASSEMBLER_HEADER
798 #include "AMDGPUGenAsmMatcher.inc"
799 
800   /// }
801 
802 private:
803   bool ParseAsAbsoluteExpression(uint32_t &Ret);
804   bool ParseDirectiveMajorMinor(uint32_t &Major, uint32_t &Minor);
805   bool ParseDirectiveHSACodeObjectVersion();
806   bool ParseDirectiveHSACodeObjectISA();
807   bool ParseDirectiveRuntimeMetadata();
808   bool ParseAMDKernelCodeTValue(StringRef ID, amd_kernel_code_t &Header);
809   bool ParseDirectiveAMDKernelCodeT();
810   bool ParseSectionDirectiveHSAText();
811   bool subtargetHasRegister(const MCRegisterInfo &MRI, unsigned RegNo) const;
812   bool ParseDirectiveAMDGPUHsaKernel();
813   bool ParseDirectiveAMDGPUHsaModuleGlobal();
814   bool ParseDirectiveAMDGPUHsaProgramGlobal();
815   bool ParseSectionDirectiveHSADataGlobalAgent();
816   bool ParseSectionDirectiveHSADataGlobalProgram();
817   bool ParseSectionDirectiveHSARodataReadonlyAgent();
818   bool AddNextRegisterToList(unsigned& Reg, unsigned& RegWidth,
819                              RegisterKind RegKind, unsigned Reg1,
820                              unsigned RegNum);
821   bool ParseAMDGPURegister(RegisterKind& RegKind, unsigned& Reg,
822                            unsigned& RegNum, unsigned& RegWidth,
823                            unsigned *DwordRegIndex);
824   void cvtMubufImpl(MCInst &Inst, const OperandVector &Operands,
825                     bool IsAtomic, bool IsAtomicReturn);
826   void cvtDSImpl(MCInst &Inst, const OperandVector &Operands,
827                  bool IsGdsHardcoded);
828 
829 public:
830   enum AMDGPUMatchResultTy {
831     Match_PreferE32 = FIRST_TARGET_MATCH_RESULT_TY
832   };
833 
834   typedef std::map<AMDGPUOperand::ImmTy, unsigned> OptionalImmIndexMap;
835 
836   AMDGPUAsmParser(const MCSubtargetInfo &STI, MCAsmParser &_Parser,
837                const MCInstrInfo &MII,
838                const MCTargetOptions &Options)
839       : MCTargetAsmParser(Options, STI), MII(MII), Parser(_Parser) {
840     MCAsmParserExtension::Initialize(Parser);
841 
842     if (getFeatureBits().none()) {
843       // Set default features.
844       copySTI().ToggleFeature("SOUTHERN_ISLANDS");
845     }
846 
847     setAvailableFeatures(ComputeAvailableFeatures(getFeatureBits()));
848 
849     {
850       // TODO: make those pre-defined variables read-only.
851       // Currently there is none suitable machinery in the core llvm-mc for this.
852       // MCSymbol::isRedefinable is intended for another purpose, and
853       // AsmParser::parseDirectiveSet() cannot be specialized for specific target.
854       AMDGPU::IsaInfo::IsaVersion ISA =
855           AMDGPU::IsaInfo::getIsaVersion(getFeatureBits());
856       MCContext &Ctx = getContext();
857       MCSymbol *Sym =
858           Ctx.getOrCreateSymbol(Twine(".option.machine_version_major"));
859       Sym->setVariableValue(MCConstantExpr::create(ISA.Major, Ctx));
860       Sym = Ctx.getOrCreateSymbol(Twine(".option.machine_version_minor"));
861       Sym->setVariableValue(MCConstantExpr::create(ISA.Minor, Ctx));
862       Sym = Ctx.getOrCreateSymbol(Twine(".option.machine_version_stepping"));
863       Sym->setVariableValue(MCConstantExpr::create(ISA.Stepping, Ctx));
864     }
865     KernelScope.initialize(getContext());
866   }
867 
868   bool isSI() const {
869     return AMDGPU::isSI(getSTI());
870   }
871 
872   bool isCI() const {
873     return AMDGPU::isCI(getSTI());
874   }
875 
876   bool isVI() const {
877     return AMDGPU::isVI(getSTI());
878   }
879 
880   bool hasInv2PiInlineImm() const {
881     return getFeatureBits()[AMDGPU::FeatureInv2PiInlineImm];
882   }
883 
884   bool hasSGPR102_SGPR103() const {
885     return !isVI();
886   }
887 
888   AMDGPUTargetStreamer &getTargetStreamer() {
889     MCTargetStreamer &TS = *getParser().getStreamer().getTargetStreamer();
890     return static_cast<AMDGPUTargetStreamer &>(TS);
891   }
892 
893   const MCRegisterInfo *getMRI() const {
894     // We need this const_cast because for some reason getContext() is not const
895     // in MCAsmParser.
896     return const_cast<AMDGPUAsmParser*>(this)->getContext().getRegisterInfo();
897   }
898 
899   const MCInstrInfo *getMII() const {
900     return &MII;
901   }
902 
903   const FeatureBitset &getFeatureBits() const {
904     return getSTI().getFeatureBits();
905   }
906 
907   void setForcedEncodingSize(unsigned Size) { ForcedEncodingSize = Size; }
908   void setForcedDPP(bool ForceDPP_) { ForcedDPP = ForceDPP_; }
909   void setForcedSDWA(bool ForceSDWA_) { ForcedSDWA = ForceSDWA_; }
910 
911   unsigned getForcedEncodingSize() const { return ForcedEncodingSize; }
912   bool isForcedVOP3() const { return ForcedEncodingSize == 64; }
913   bool isForcedDPP() const { return ForcedDPP; }
914   bool isForcedSDWA() const { return ForcedSDWA; }
915   ArrayRef<unsigned> getMatchedVariants() const;
916 
917   std::unique_ptr<AMDGPUOperand> parseRegister();
918   bool ParseRegister(unsigned &RegNo, SMLoc &StartLoc, SMLoc &EndLoc) override;
919   unsigned checkTargetMatchPredicate(MCInst &Inst) override;
920   unsigned validateTargetOperandClass(MCParsedAsmOperand &Op,
921                                       unsigned Kind) override;
922   bool MatchAndEmitInstruction(SMLoc IDLoc, unsigned &Opcode,
923                                OperandVector &Operands, MCStreamer &Out,
924                                uint64_t &ErrorInfo,
925                                bool MatchingInlineAsm) override;
926   bool ParseDirective(AsmToken DirectiveID) override;
927   OperandMatchResultTy parseOperand(OperandVector &Operands, StringRef Mnemonic);
928   StringRef parseMnemonicSuffix(StringRef Name);
929   bool ParseInstruction(ParseInstructionInfo &Info, StringRef Name,
930                         SMLoc NameLoc, OperandVector &Operands) override;
931   //bool ProcessInstruction(MCInst &Inst);
932 
933   OperandMatchResultTy parseIntWithPrefix(const char *Prefix, int64_t &Int);
934 
935   OperandMatchResultTy
936   parseIntWithPrefix(const char *Prefix, OperandVector &Operands,
937                      AMDGPUOperand::ImmTy ImmTy = AMDGPUOperand::ImmTyNone,
938                      bool (*ConvertResult)(int64_t &) = nullptr);
939 
940   OperandMatchResultTy parseOperandArrayWithPrefix(
941     const char *Prefix,
942     OperandVector &Operands,
943     AMDGPUOperand::ImmTy ImmTy = AMDGPUOperand::ImmTyNone,
944     bool (*ConvertResult)(int64_t&) = nullptr);
945 
946   OperandMatchResultTy
947   parseNamedBit(const char *Name, OperandVector &Operands,
948                 AMDGPUOperand::ImmTy ImmTy = AMDGPUOperand::ImmTyNone);
949   OperandMatchResultTy parseStringWithPrefix(StringRef Prefix,
950                                              StringRef &Value);
951 
952   OperandMatchResultTy parseImm(OperandVector &Operands);
953   OperandMatchResultTy parseReg(OperandVector &Operands);
954   OperandMatchResultTy parseRegOrImm(OperandVector &Operands);
955   OperandMatchResultTy parseRegOrImmWithFPInputMods(OperandVector &Operands, bool AllowImm = true);
956   OperandMatchResultTy parseRegOrImmWithIntInputMods(OperandVector &Operands, bool AllowImm = true);
957   OperandMatchResultTy parseRegWithFPInputMods(OperandVector &Operands);
958   OperandMatchResultTy parseRegWithIntInputMods(OperandVector &Operands);
959   OperandMatchResultTy parseVReg32OrOff(OperandVector &Operands);
960 
961   void cvtDSOffset01(MCInst &Inst, const OperandVector &Operands);
962   void cvtDS(MCInst &Inst, const OperandVector &Operands) { cvtDSImpl(Inst, Operands, false); }
963   void cvtDSGds(MCInst &Inst, const OperandVector &Operands) { cvtDSImpl(Inst, Operands, true); }
964   void cvtExp(MCInst &Inst, const OperandVector &Operands);
965 
966   bool parseCnt(int64_t &IntVal);
967   OperandMatchResultTy parseSWaitCntOps(OperandVector &Operands);
968   OperandMatchResultTy parseHwreg(OperandVector &Operands);
969 
970 private:
971   struct OperandInfoTy {
972     int64_t Id;
973     bool IsSymbolic;
974     OperandInfoTy(int64_t Id_) : Id(Id_), IsSymbolic(false) { }
975   };
976 
977   bool parseSendMsgConstruct(OperandInfoTy &Msg, OperandInfoTy &Operation, int64_t &StreamId);
978   bool parseHwregConstruct(OperandInfoTy &HwReg, int64_t &Offset, int64_t &Width);
979 
980   void errorExpTgt();
981   OperandMatchResultTy parseExpTgtImpl(StringRef Str, uint8_t &Val);
982 
983   bool validateOperandLimitations(const MCInst &Inst);
984   bool usesConstantBus(const MCInst &Inst, unsigned OpIdx);
985   bool isInlineConstant(const MCInst &Inst, unsigned OpIdx) const;
986   unsigned findImplicitSGPRReadInVOP(const MCInst &Inst) const;
987   bool isSGPR(unsigned Reg);
988 
989 public:
990   OperandMatchResultTy parseOptionalOperand(OperandVector &Operands);
991 
992   OperandMatchResultTy parseExpTgt(OperandVector &Operands);
993   OperandMatchResultTy parseSendMsgOp(OperandVector &Operands);
994   OperandMatchResultTy parseInterpSlot(OperandVector &Operands);
995   OperandMatchResultTy parseInterpAttr(OperandVector &Operands);
996   OperandMatchResultTy parseSOppBrTarget(OperandVector &Operands);
997 
998   void cvtMubuf(MCInst &Inst, const OperandVector &Operands) { cvtMubufImpl(Inst, Operands, false, false); }
999   void cvtMubufAtomic(MCInst &Inst, const OperandVector &Operands) { cvtMubufImpl(Inst, Operands, true, false); }
1000   void cvtMubufAtomicReturn(MCInst &Inst, const OperandVector &Operands) { cvtMubufImpl(Inst, Operands, true, true); }
1001   AMDGPUOperand::Ptr defaultGLC() const;
1002   AMDGPUOperand::Ptr defaultSLC() const;
1003   AMDGPUOperand::Ptr defaultTFE() const;
1004 
1005   AMDGPUOperand::Ptr defaultDMask() const;
1006   AMDGPUOperand::Ptr defaultUNorm() const;
1007   AMDGPUOperand::Ptr defaultDA() const;
1008   AMDGPUOperand::Ptr defaultR128() const;
1009   AMDGPUOperand::Ptr defaultLWE() const;
1010   AMDGPUOperand::Ptr defaultSMRDOffset8() const;
1011   AMDGPUOperand::Ptr defaultSMRDOffset20() const;
1012   AMDGPUOperand::Ptr defaultSMRDLiteralOffset() const;
1013 
1014   OperandMatchResultTy parseOModOperand(OperandVector &Operands);
1015 
1016   void cvtId(MCInst &Inst, const OperandVector &Operands);
1017   void cvtVOP3_2_mod(MCInst &Inst, const OperandVector &Operands);
1018 
1019   void cvtVOP3Impl(MCInst &Inst,
1020                    const OperandVector &Operands,
1021                    OptionalImmIndexMap &OptionalIdx);
1022   void cvtVOP3(MCInst &Inst, const OperandVector &Operands);
1023   void cvtVOP3P(MCInst &Inst, const OperandVector &Operands);
1024 
1025   void cvtMIMG(MCInst &Inst, const OperandVector &Operands);
1026   void cvtMIMGAtomic(MCInst &Inst, const OperandVector &Operands);
1027 
1028   OperandMatchResultTy parseDPPCtrl(OperandVector &Operands);
1029   AMDGPUOperand::Ptr defaultRowMask() const;
1030   AMDGPUOperand::Ptr defaultBankMask() const;
1031   AMDGPUOperand::Ptr defaultBoundCtrl() const;
1032   void cvtDPP(MCInst &Inst, const OperandVector &Operands);
1033 
1034   OperandMatchResultTy parseSDWASel(OperandVector &Operands, StringRef Prefix,
1035                                     AMDGPUOperand::ImmTy Type);
1036   OperandMatchResultTy parseSDWADstUnused(OperandVector &Operands);
1037   void cvtSdwaVOP1(MCInst &Inst, const OperandVector &Operands);
1038   void cvtSdwaVOP2(MCInst &Inst, const OperandVector &Operands);
1039   void cvtSdwaVOPC(MCInst &Inst, const OperandVector &Operands);
1040   void cvtSDWA(MCInst &Inst, const OperandVector &Operands,
1041                uint64_t BasicInstType);
1042 };
1043 
1044 struct OptionalOperand {
1045   const char *Name;
1046   AMDGPUOperand::ImmTy Type;
1047   bool IsBit;
1048   bool (*ConvertResult)(int64_t&);
1049 };
1050 
1051 } // end anonymous namespace
1052 
1053 // May be called with integer type with equivalent bitwidth.
1054 static const fltSemantics *getFltSemantics(unsigned Size) {
1055   switch (Size) {
1056   case 4:
1057     return &APFloat::IEEEsingle();
1058   case 8:
1059     return &APFloat::IEEEdouble();
1060   case 2:
1061     return &APFloat::IEEEhalf();
1062   default:
1063     llvm_unreachable("unsupported fp type");
1064   }
1065 }
1066 
1067 static const fltSemantics *getFltSemantics(MVT VT) {
1068   return getFltSemantics(VT.getSizeInBits() / 8);
1069 }
1070 
1071 static const fltSemantics *getOpFltSemantics(uint8_t OperandType) {
1072   switch (OperandType) {
1073   case AMDGPU::OPERAND_REG_IMM_INT32:
1074   case AMDGPU::OPERAND_REG_IMM_FP32:
1075   case AMDGPU::OPERAND_REG_INLINE_C_INT32:
1076   case AMDGPU::OPERAND_REG_INLINE_C_FP32:
1077     return &APFloat::IEEEsingle();
1078   case AMDGPU::OPERAND_REG_IMM_INT64:
1079   case AMDGPU::OPERAND_REG_IMM_FP64:
1080   case AMDGPU::OPERAND_REG_INLINE_C_INT64:
1081   case AMDGPU::OPERAND_REG_INLINE_C_FP64:
1082     return &APFloat::IEEEdouble();
1083   case AMDGPU::OPERAND_REG_IMM_INT16:
1084   case AMDGPU::OPERAND_REG_IMM_FP16:
1085   case AMDGPU::OPERAND_REG_INLINE_C_INT16:
1086   case AMDGPU::OPERAND_REG_INLINE_C_FP16:
1087   case AMDGPU::OPERAND_REG_INLINE_C_V2INT16:
1088   case AMDGPU::OPERAND_REG_INLINE_C_V2FP16:
1089     return &APFloat::IEEEhalf();
1090   default:
1091     llvm_unreachable("unsupported fp type");
1092   }
1093 }
1094 
1095 //===----------------------------------------------------------------------===//
1096 // Operand
1097 //===----------------------------------------------------------------------===//
1098 
1099 static bool canLosslesslyConvertToFPType(APFloat &FPLiteral, MVT VT) {
1100   bool Lost;
1101 
1102   // Convert literal to single precision
1103   APFloat::opStatus Status = FPLiteral.convert(*getFltSemantics(VT),
1104                                                APFloat::rmNearestTiesToEven,
1105                                                &Lost);
1106   // We allow precision lost but not overflow or underflow
1107   if (Status != APFloat::opOK &&
1108       Lost &&
1109       ((Status & APFloat::opOverflow)  != 0 ||
1110        (Status & APFloat::opUnderflow) != 0)) {
1111     return false;
1112   }
1113 
1114   return true;
1115 }
1116 
1117 bool AMDGPUOperand::isInlinableImm(MVT type) const {
1118   if (!isImmTy(ImmTyNone)) {
1119     // Only plain immediates are inlinable (e.g. "clamp" attribute is not)
1120     return false;
1121   }
1122   // TODO: We should avoid using host float here. It would be better to
1123   // check the float bit values which is what a few other places do.
1124   // We've had bot failures before due to weird NaN support on mips hosts.
1125 
1126   APInt Literal(64, Imm.Val);
1127 
1128   if (Imm.IsFPImm) { // We got fp literal token
1129     if (type == MVT::f64 || type == MVT::i64) { // Expected 64-bit operand
1130       return AMDGPU::isInlinableLiteral64(Imm.Val,
1131                                           AsmParser->hasInv2PiInlineImm());
1132     }
1133 
1134     APFloat FPLiteral(APFloat::IEEEdouble(), APInt(64, Imm.Val));
1135     if (!canLosslesslyConvertToFPType(FPLiteral, type))
1136       return false;
1137 
1138     if (type.getScalarSizeInBits() == 16) {
1139       return AMDGPU::isInlinableLiteral16(
1140         static_cast<int16_t>(FPLiteral.bitcastToAPInt().getZExtValue()),
1141         AsmParser->hasInv2PiInlineImm());
1142     }
1143 
1144     // Check if single precision literal is inlinable
1145     return AMDGPU::isInlinableLiteral32(
1146       static_cast<int32_t>(FPLiteral.bitcastToAPInt().getZExtValue()),
1147       AsmParser->hasInv2PiInlineImm());
1148   }
1149 
1150   // We got int literal token.
1151   if (type == MVT::f64 || type == MVT::i64) { // Expected 64-bit operand
1152     return AMDGPU::isInlinableLiteral64(Imm.Val,
1153                                         AsmParser->hasInv2PiInlineImm());
1154   }
1155 
1156   if (type.getScalarSizeInBits() == 16) {
1157     return AMDGPU::isInlinableLiteral16(
1158       static_cast<int16_t>(Literal.getLoBits(16).getSExtValue()),
1159       AsmParser->hasInv2PiInlineImm());
1160   }
1161 
1162   return AMDGPU::isInlinableLiteral32(
1163     static_cast<int32_t>(Literal.getLoBits(32).getZExtValue()),
1164     AsmParser->hasInv2PiInlineImm());
1165 }
1166 
1167 bool AMDGPUOperand::isLiteralImm(MVT type) const {
1168   // Check that this imediate can be added as literal
1169   if (!isImmTy(ImmTyNone)) {
1170     return false;
1171   }
1172 
1173   if (!Imm.IsFPImm) {
1174     // We got int literal token.
1175 
1176     unsigned Size = type.getSizeInBits();
1177     if (Size == 64)
1178       Size = 32;
1179 
1180     // FIXME: 64-bit operands can zero extend, sign extend, or pad zeroes for FP
1181     // types.
1182     return isUIntN(Size, Imm.Val) || isIntN(Size, Imm.Val);
1183   }
1184 
1185   // We got fp literal token
1186   if (type == MVT::f64) { // Expected 64-bit fp operand
1187     // We would set low 64-bits of literal to zeroes but we accept this literals
1188     return true;
1189   }
1190 
1191   if (type == MVT::i64) { // Expected 64-bit int operand
1192     // We don't allow fp literals in 64-bit integer instructions. It is
1193     // unclear how we should encode them.
1194     return false;
1195   }
1196 
1197   APFloat FPLiteral(APFloat::IEEEdouble(), APInt(64, Imm.Val));
1198   return canLosslesslyConvertToFPType(FPLiteral, type);
1199 }
1200 
1201 bool AMDGPUOperand::isRegClass(unsigned RCID) const {
1202   return isRegKind() && AsmParser->getMRI()->getRegClass(RCID).contains(getReg());
1203 }
1204 
1205 void AMDGPUOperand::addImmOperands(MCInst &Inst, unsigned N, bool ApplyModifiers) const {
1206   int64_t Val = Imm.Val;
1207   if (isImmTy(ImmTyNone) && ApplyModifiers && Imm.Mods.hasFPModifiers() && Imm.Mods.Neg) {
1208     // Apply modifiers to immediate value. Only negate can get here
1209     if (Imm.IsFPImm) {
1210       APFloat F(BitsToDouble(Val));
1211       F.changeSign();
1212       Val = F.bitcastToAPInt().getZExtValue();
1213     } else {
1214       Val = -Val;
1215     }
1216   }
1217 
1218   if (AMDGPU::isSISrcOperand(AsmParser->getMII()->get(Inst.getOpcode()),
1219                              Inst.getNumOperands())) {
1220     addLiteralImmOperand(Inst, Val);
1221   } else {
1222     Inst.addOperand(MCOperand::createImm(Val));
1223   }
1224 }
1225 
1226 void AMDGPUOperand::addLiteralImmOperand(MCInst &Inst, int64_t Val) const {
1227   const auto& InstDesc = AsmParser->getMII()->get(Inst.getOpcode());
1228   auto OpNum = Inst.getNumOperands();
1229   // Check that this operand accepts literals
1230   assert(AMDGPU::isSISrcOperand(InstDesc, OpNum));
1231 
1232   APInt Literal(64, Val);
1233   uint8_t OpTy = InstDesc.OpInfo[OpNum].OperandType;
1234 
1235   if (Imm.IsFPImm) { // We got fp literal token
1236     switch (OpTy) {
1237     case AMDGPU::OPERAND_REG_IMM_INT64:
1238     case AMDGPU::OPERAND_REG_IMM_FP64:
1239     case AMDGPU::OPERAND_REG_INLINE_C_INT64:
1240     case AMDGPU::OPERAND_REG_INLINE_C_FP64: {
1241       if (AMDGPU::isInlinableLiteral64(Literal.getZExtValue(),
1242                                        AsmParser->hasInv2PiInlineImm())) {
1243         Inst.addOperand(MCOperand::createImm(Literal.getZExtValue()));
1244         return;
1245       }
1246 
1247       // Non-inlineable
1248       if (AMDGPU::isSISrcFPOperand(InstDesc, OpNum)) { // Expected 64-bit fp operand
1249         // For fp operands we check if low 32 bits are zeros
1250         if (Literal.getLoBits(32) != 0) {
1251           const_cast<AMDGPUAsmParser *>(AsmParser)->Warning(Inst.getLoc(),
1252           "Can't encode literal as exact 64-bit floating-point operand. "
1253           "Low 32-bits will be set to zero");
1254         }
1255 
1256         Inst.addOperand(MCOperand::createImm(Literal.lshr(32).getZExtValue()));
1257         return;
1258       }
1259 
1260       // We don't allow fp literals in 64-bit integer instructions. It is
1261       // unclear how we should encode them. This case should be checked earlier
1262       // in predicate methods (isLiteralImm())
1263       llvm_unreachable("fp literal in 64-bit integer instruction.");
1264     }
1265     case AMDGPU::OPERAND_REG_IMM_INT32:
1266     case AMDGPU::OPERAND_REG_IMM_FP32:
1267     case AMDGPU::OPERAND_REG_INLINE_C_INT32:
1268     case AMDGPU::OPERAND_REG_INLINE_C_FP32:
1269     case AMDGPU::OPERAND_REG_IMM_INT16:
1270     case AMDGPU::OPERAND_REG_IMM_FP16:
1271     case AMDGPU::OPERAND_REG_INLINE_C_INT16:
1272     case AMDGPU::OPERAND_REG_INLINE_C_FP16:
1273     case AMDGPU::OPERAND_REG_INLINE_C_V2INT16:
1274     case AMDGPU::OPERAND_REG_INLINE_C_V2FP16: {
1275       bool lost;
1276       APFloat FPLiteral(APFloat::IEEEdouble(), Literal);
1277       // Convert literal to single precision
1278       FPLiteral.convert(*getOpFltSemantics(OpTy),
1279                         APFloat::rmNearestTiesToEven, &lost);
1280       // We allow precision lost but not overflow or underflow. This should be
1281       // checked earlier in isLiteralImm()
1282 
1283       uint64_t ImmVal = FPLiteral.bitcastToAPInt().getZExtValue();
1284       if (OpTy == AMDGPU::OPERAND_REG_INLINE_C_V2INT16 ||
1285           OpTy == AMDGPU::OPERAND_REG_INLINE_C_V2FP16) {
1286         ImmVal |= (ImmVal << 16);
1287       }
1288 
1289       Inst.addOperand(MCOperand::createImm(ImmVal));
1290       return;
1291     }
1292     default:
1293       llvm_unreachable("invalid operand size");
1294     }
1295 
1296     return;
1297   }
1298 
1299    // We got int literal token.
1300   // Only sign extend inline immediates.
1301   // FIXME: No errors on truncation
1302   switch (OpTy) {
1303   case AMDGPU::OPERAND_REG_IMM_INT32:
1304   case AMDGPU::OPERAND_REG_IMM_FP32:
1305   case AMDGPU::OPERAND_REG_INLINE_C_INT32:
1306   case AMDGPU::OPERAND_REG_INLINE_C_FP32: {
1307     if (isInt<32>(Val) &&
1308         AMDGPU::isInlinableLiteral32(static_cast<int32_t>(Val),
1309                                      AsmParser->hasInv2PiInlineImm())) {
1310       Inst.addOperand(MCOperand::createImm(Val));
1311       return;
1312     }
1313 
1314     Inst.addOperand(MCOperand::createImm(Val & 0xffffffff));
1315     return;
1316   }
1317   case AMDGPU::OPERAND_REG_IMM_INT64:
1318   case AMDGPU::OPERAND_REG_IMM_FP64:
1319   case AMDGPU::OPERAND_REG_INLINE_C_INT64:
1320   case AMDGPU::OPERAND_REG_INLINE_C_FP64: {
1321     if (AMDGPU::isInlinableLiteral64(Val, AsmParser->hasInv2PiInlineImm())) {
1322       Inst.addOperand(MCOperand::createImm(Val));
1323       return;
1324     }
1325 
1326     Inst.addOperand(MCOperand::createImm(Lo_32(Val)));
1327     return;
1328   }
1329   case AMDGPU::OPERAND_REG_IMM_INT16:
1330   case AMDGPU::OPERAND_REG_IMM_FP16:
1331   case AMDGPU::OPERAND_REG_INLINE_C_INT16:
1332   case AMDGPU::OPERAND_REG_INLINE_C_FP16: {
1333     if (isInt<16>(Val) &&
1334         AMDGPU::isInlinableLiteral16(static_cast<int16_t>(Val),
1335                                      AsmParser->hasInv2PiInlineImm())) {
1336       Inst.addOperand(MCOperand::createImm(Val));
1337       return;
1338     }
1339 
1340     Inst.addOperand(MCOperand::createImm(Val & 0xffff));
1341     return;
1342   }
1343   case AMDGPU::OPERAND_REG_INLINE_C_V2INT16:
1344   case AMDGPU::OPERAND_REG_INLINE_C_V2FP16: {
1345     auto LiteralVal = static_cast<uint16_t>(Literal.getLoBits(16).getZExtValue());
1346     assert(AMDGPU::isInlinableLiteral16(LiteralVal,
1347                                         AsmParser->hasInv2PiInlineImm()));
1348 
1349     uint32_t ImmVal = static_cast<uint32_t>(LiteralVal) << 16 |
1350                       static_cast<uint32_t>(LiteralVal);
1351     Inst.addOperand(MCOperand::createImm(ImmVal));
1352     return;
1353   }
1354   default:
1355     llvm_unreachable("invalid operand size");
1356   }
1357 }
1358 
1359 template <unsigned Bitwidth>
1360 void AMDGPUOperand::addKImmFPOperands(MCInst &Inst, unsigned N) const {
1361   APInt Literal(64, Imm.Val);
1362 
1363   if (!Imm.IsFPImm) {
1364     // We got int literal token.
1365     Inst.addOperand(MCOperand::createImm(Literal.getLoBits(Bitwidth).getZExtValue()));
1366     return;
1367   }
1368 
1369   bool Lost;
1370   APFloat FPLiteral(APFloat::IEEEdouble(), Literal);
1371   FPLiteral.convert(*getFltSemantics(Bitwidth / 8),
1372                     APFloat::rmNearestTiesToEven, &Lost);
1373   Inst.addOperand(MCOperand::createImm(FPLiteral.bitcastToAPInt().getZExtValue()));
1374 }
1375 
1376 void AMDGPUOperand::addRegOperands(MCInst &Inst, unsigned N) const {
1377   Inst.addOperand(MCOperand::createReg(AMDGPU::getMCReg(getReg(), AsmParser->getSTI())));
1378 }
1379 
1380 //===----------------------------------------------------------------------===//
1381 // AsmParser
1382 //===----------------------------------------------------------------------===//
1383 
1384 static int getRegClass(RegisterKind Is, unsigned RegWidth) {
1385   if (Is == IS_VGPR) {
1386     switch (RegWidth) {
1387       default: return -1;
1388       case 1: return AMDGPU::VGPR_32RegClassID;
1389       case 2: return AMDGPU::VReg_64RegClassID;
1390       case 3: return AMDGPU::VReg_96RegClassID;
1391       case 4: return AMDGPU::VReg_128RegClassID;
1392       case 8: return AMDGPU::VReg_256RegClassID;
1393       case 16: return AMDGPU::VReg_512RegClassID;
1394     }
1395   } else if (Is == IS_TTMP) {
1396     switch (RegWidth) {
1397       default: return -1;
1398       case 1: return AMDGPU::TTMP_32RegClassID;
1399       case 2: return AMDGPU::TTMP_64RegClassID;
1400       case 4: return AMDGPU::TTMP_128RegClassID;
1401     }
1402   } else if (Is == IS_SGPR) {
1403     switch (RegWidth) {
1404       default: return -1;
1405       case 1: return AMDGPU::SGPR_32RegClassID;
1406       case 2: return AMDGPU::SGPR_64RegClassID;
1407       case 4: return AMDGPU::SGPR_128RegClassID;
1408       case 8: return AMDGPU::SReg_256RegClassID;
1409       case 16: return AMDGPU::SReg_512RegClassID;
1410     }
1411   }
1412   return -1;
1413 }
1414 
1415 static unsigned getSpecialRegForName(StringRef RegName) {
1416   return StringSwitch<unsigned>(RegName)
1417     .Case("exec", AMDGPU::EXEC)
1418     .Case("vcc", AMDGPU::VCC)
1419     .Case("flat_scratch", AMDGPU::FLAT_SCR)
1420     .Case("m0", AMDGPU::M0)
1421     .Case("scc", AMDGPU::SCC)
1422     .Case("tba", AMDGPU::TBA)
1423     .Case("tma", AMDGPU::TMA)
1424     .Case("flat_scratch_lo", AMDGPU::FLAT_SCR_LO)
1425     .Case("flat_scratch_hi", AMDGPU::FLAT_SCR_HI)
1426     .Case("vcc_lo", AMDGPU::VCC_LO)
1427     .Case("vcc_hi", AMDGPU::VCC_HI)
1428     .Case("exec_lo", AMDGPU::EXEC_LO)
1429     .Case("exec_hi", AMDGPU::EXEC_HI)
1430     .Case("tma_lo", AMDGPU::TMA_LO)
1431     .Case("tma_hi", AMDGPU::TMA_HI)
1432     .Case("tba_lo", AMDGPU::TBA_LO)
1433     .Case("tba_hi", AMDGPU::TBA_HI)
1434     .Default(0);
1435 }
1436 
1437 bool AMDGPUAsmParser::ParseRegister(unsigned &RegNo, SMLoc &StartLoc,
1438                                     SMLoc &EndLoc) {
1439   auto R = parseRegister();
1440   if (!R) return true;
1441   assert(R->isReg());
1442   RegNo = R->getReg();
1443   StartLoc = R->getStartLoc();
1444   EndLoc = R->getEndLoc();
1445   return false;
1446 }
1447 
1448 bool AMDGPUAsmParser::AddNextRegisterToList(unsigned &Reg, unsigned &RegWidth,
1449                                             RegisterKind RegKind, unsigned Reg1,
1450                                             unsigned RegNum) {
1451   switch (RegKind) {
1452   case IS_SPECIAL:
1453     if (Reg == AMDGPU::EXEC_LO && Reg1 == AMDGPU::EXEC_HI) {
1454       Reg = AMDGPU::EXEC;
1455       RegWidth = 2;
1456       return true;
1457     }
1458     if (Reg == AMDGPU::FLAT_SCR_LO && Reg1 == AMDGPU::FLAT_SCR_HI) {
1459       Reg = AMDGPU::FLAT_SCR;
1460       RegWidth = 2;
1461       return true;
1462     }
1463     if (Reg == AMDGPU::VCC_LO && Reg1 == AMDGPU::VCC_HI) {
1464       Reg = AMDGPU::VCC;
1465       RegWidth = 2;
1466       return true;
1467     }
1468     if (Reg == AMDGPU::TBA_LO && Reg1 == AMDGPU::TBA_HI) {
1469       Reg = AMDGPU::TBA;
1470       RegWidth = 2;
1471       return true;
1472     }
1473     if (Reg == AMDGPU::TMA_LO && Reg1 == AMDGPU::TMA_HI) {
1474       Reg = AMDGPU::TMA;
1475       RegWidth = 2;
1476       return true;
1477     }
1478     return false;
1479   case IS_VGPR:
1480   case IS_SGPR:
1481   case IS_TTMP:
1482     if (Reg1 != Reg + RegWidth) {
1483       return false;
1484     }
1485     RegWidth++;
1486     return true;
1487   default:
1488     llvm_unreachable("unexpected register kind");
1489   }
1490 }
1491 
1492 bool AMDGPUAsmParser::ParseAMDGPURegister(RegisterKind &RegKind, unsigned &Reg,
1493                                           unsigned &RegNum, unsigned &RegWidth,
1494                                           unsigned *DwordRegIndex) {
1495   if (DwordRegIndex) { *DwordRegIndex = 0; }
1496   const MCRegisterInfo *TRI = getContext().getRegisterInfo();
1497   if (getLexer().is(AsmToken::Identifier)) {
1498     StringRef RegName = Parser.getTok().getString();
1499     if ((Reg = getSpecialRegForName(RegName))) {
1500       Parser.Lex();
1501       RegKind = IS_SPECIAL;
1502     } else {
1503       unsigned RegNumIndex = 0;
1504       if (RegName[0] == 'v') {
1505         RegNumIndex = 1;
1506         RegKind = IS_VGPR;
1507       } else if (RegName[0] == 's') {
1508         RegNumIndex = 1;
1509         RegKind = IS_SGPR;
1510       } else if (RegName.startswith("ttmp")) {
1511         RegNumIndex = strlen("ttmp");
1512         RegKind = IS_TTMP;
1513       } else {
1514         return false;
1515       }
1516       if (RegName.size() > RegNumIndex) {
1517         // Single 32-bit register: vXX.
1518         if (RegName.substr(RegNumIndex).getAsInteger(10, RegNum))
1519           return false;
1520         Parser.Lex();
1521         RegWidth = 1;
1522       } else {
1523         // Range of registers: v[XX:YY]. ":YY" is optional.
1524         Parser.Lex();
1525         int64_t RegLo, RegHi;
1526         if (getLexer().isNot(AsmToken::LBrac))
1527           return false;
1528         Parser.Lex();
1529 
1530         if (getParser().parseAbsoluteExpression(RegLo))
1531           return false;
1532 
1533         const bool isRBrace = getLexer().is(AsmToken::RBrac);
1534         if (!isRBrace && getLexer().isNot(AsmToken::Colon))
1535           return false;
1536         Parser.Lex();
1537 
1538         if (isRBrace) {
1539           RegHi = RegLo;
1540         } else {
1541           if (getParser().parseAbsoluteExpression(RegHi))
1542             return false;
1543 
1544           if (getLexer().isNot(AsmToken::RBrac))
1545             return false;
1546           Parser.Lex();
1547         }
1548         RegNum = (unsigned) RegLo;
1549         RegWidth = (RegHi - RegLo) + 1;
1550       }
1551     }
1552   } else if (getLexer().is(AsmToken::LBrac)) {
1553     // List of consecutive registers: [s0,s1,s2,s3]
1554     Parser.Lex();
1555     if (!ParseAMDGPURegister(RegKind, Reg, RegNum, RegWidth, nullptr))
1556       return false;
1557     if (RegWidth != 1)
1558       return false;
1559     RegisterKind RegKind1;
1560     unsigned Reg1, RegNum1, RegWidth1;
1561     do {
1562       if (getLexer().is(AsmToken::Comma)) {
1563         Parser.Lex();
1564       } else if (getLexer().is(AsmToken::RBrac)) {
1565         Parser.Lex();
1566         break;
1567       } else if (ParseAMDGPURegister(RegKind1, Reg1, RegNum1, RegWidth1, nullptr)) {
1568         if (RegWidth1 != 1) {
1569           return false;
1570         }
1571         if (RegKind1 != RegKind) {
1572           return false;
1573         }
1574         if (!AddNextRegisterToList(Reg, RegWidth, RegKind1, Reg1, RegNum1)) {
1575           return false;
1576         }
1577       } else {
1578         return false;
1579       }
1580     } while (true);
1581   } else {
1582     return false;
1583   }
1584   switch (RegKind) {
1585   case IS_SPECIAL:
1586     RegNum = 0;
1587     RegWidth = 1;
1588     break;
1589   case IS_VGPR:
1590   case IS_SGPR:
1591   case IS_TTMP:
1592   {
1593     unsigned Size = 1;
1594     if (RegKind == IS_SGPR || RegKind == IS_TTMP) {
1595       // SGPR and TTMP registers must be aligned. Max required alignment is 4 dwords.
1596       Size = std::min(RegWidth, 4u);
1597     }
1598     if (RegNum % Size != 0)
1599       return false;
1600     if (DwordRegIndex) { *DwordRegIndex = RegNum; }
1601     RegNum = RegNum / Size;
1602     int RCID = getRegClass(RegKind, RegWidth);
1603     if (RCID == -1)
1604       return false;
1605     const MCRegisterClass RC = TRI->getRegClass(RCID);
1606     if (RegNum >= RC.getNumRegs())
1607       return false;
1608     Reg = RC.getRegister(RegNum);
1609     break;
1610   }
1611 
1612   default:
1613     llvm_unreachable("unexpected register kind");
1614   }
1615 
1616   if (!subtargetHasRegister(*TRI, Reg))
1617     return false;
1618   return true;
1619 }
1620 
1621 std::unique_ptr<AMDGPUOperand> AMDGPUAsmParser::parseRegister() {
1622   const auto &Tok = Parser.getTok();
1623   SMLoc StartLoc = Tok.getLoc();
1624   SMLoc EndLoc = Tok.getEndLoc();
1625   RegisterKind RegKind;
1626   unsigned Reg, RegNum, RegWidth, DwordRegIndex;
1627 
1628   if (!ParseAMDGPURegister(RegKind, Reg, RegNum, RegWidth, &DwordRegIndex)) {
1629     return nullptr;
1630   }
1631   KernelScope.usesRegister(RegKind, DwordRegIndex, RegWidth);
1632   return AMDGPUOperand::CreateReg(this, Reg, StartLoc, EndLoc, false);
1633 }
1634 
1635 OperandMatchResultTy
1636 AMDGPUAsmParser::parseImm(OperandVector &Operands) {
1637   // TODO: add syntactic sugar for 1/(2*PI)
1638   bool Minus = false;
1639   if (getLexer().getKind() == AsmToken::Minus) {
1640     Minus = true;
1641     Parser.Lex();
1642   }
1643 
1644   SMLoc S = Parser.getTok().getLoc();
1645   switch(getLexer().getKind()) {
1646   case AsmToken::Integer: {
1647     int64_t IntVal;
1648     if (getParser().parseAbsoluteExpression(IntVal))
1649       return MatchOperand_ParseFail;
1650     if (Minus)
1651       IntVal *= -1;
1652     Operands.push_back(AMDGPUOperand::CreateImm(this, IntVal, S));
1653     return MatchOperand_Success;
1654   }
1655   case AsmToken::Real: {
1656     int64_t IntVal;
1657     if (getParser().parseAbsoluteExpression(IntVal))
1658       return MatchOperand_ParseFail;
1659 
1660     APFloat F(BitsToDouble(IntVal));
1661     if (Minus)
1662       F.changeSign();
1663     Operands.push_back(
1664         AMDGPUOperand::CreateImm(this, F.bitcastToAPInt().getZExtValue(), S,
1665                                  AMDGPUOperand::ImmTyNone, true));
1666     return MatchOperand_Success;
1667   }
1668   default:
1669     return Minus ? MatchOperand_ParseFail : MatchOperand_NoMatch;
1670   }
1671 }
1672 
1673 OperandMatchResultTy
1674 AMDGPUAsmParser::parseReg(OperandVector &Operands) {
1675   if (auto R = parseRegister()) {
1676     assert(R->isReg());
1677     R->Reg.IsForcedVOP3 = isForcedVOP3();
1678     Operands.push_back(std::move(R));
1679     return MatchOperand_Success;
1680   }
1681   return MatchOperand_NoMatch;
1682 }
1683 
1684 OperandMatchResultTy
1685 AMDGPUAsmParser::parseRegOrImm(OperandVector &Operands) {
1686   auto res = parseImm(Operands);
1687   if (res != MatchOperand_NoMatch) {
1688     return res;
1689   }
1690 
1691   return parseReg(Operands);
1692 }
1693 
1694 OperandMatchResultTy
1695 AMDGPUAsmParser::parseRegOrImmWithFPInputMods(OperandVector &Operands,
1696                                               bool AllowImm) {
1697   // XXX: During parsing we can't determine if minus sign means
1698   // negate-modifier or negative immediate value.
1699   // By default we suppose it is modifier.
1700   bool Negate = false, Abs = false, Abs2 = false;
1701 
1702   if (getLexer().getKind()== AsmToken::Minus) {
1703     Parser.Lex();
1704     Negate = true;
1705   }
1706 
1707   if (getLexer().getKind() == AsmToken::Identifier &&
1708       Parser.getTok().getString() == "abs") {
1709     Parser.Lex();
1710     Abs2 = true;
1711     if (getLexer().isNot(AsmToken::LParen)) {
1712       Error(Parser.getTok().getLoc(), "expected left paren after abs");
1713       return MatchOperand_ParseFail;
1714     }
1715     Parser.Lex();
1716   }
1717 
1718   if (getLexer().getKind() == AsmToken::Pipe) {
1719     if (Abs2) {
1720       Error(Parser.getTok().getLoc(), "expected register or immediate");
1721       return MatchOperand_ParseFail;
1722     }
1723     Parser.Lex();
1724     Abs = true;
1725   }
1726 
1727   OperandMatchResultTy Res;
1728   if (AllowImm) {
1729     Res = parseRegOrImm(Operands);
1730   } else {
1731     Res = parseReg(Operands);
1732   }
1733   if (Res != MatchOperand_Success) {
1734     return Res;
1735   }
1736 
1737   AMDGPUOperand::Modifiers Mods;
1738   if (Negate) {
1739     Mods.Neg = true;
1740   }
1741   if (Abs) {
1742     if (getLexer().getKind() != AsmToken::Pipe) {
1743       Error(Parser.getTok().getLoc(), "expected vertical bar");
1744       return MatchOperand_ParseFail;
1745     }
1746     Parser.Lex();
1747     Mods.Abs = true;
1748   }
1749   if (Abs2) {
1750     if (getLexer().isNot(AsmToken::RParen)) {
1751       Error(Parser.getTok().getLoc(), "expected closing parentheses");
1752       return MatchOperand_ParseFail;
1753     }
1754     Parser.Lex();
1755     Mods.Abs = true;
1756   }
1757 
1758   if (Mods.hasFPModifiers()) {
1759     AMDGPUOperand &Op = static_cast<AMDGPUOperand &>(*Operands.back());
1760     Op.setModifiers(Mods);
1761   }
1762   return MatchOperand_Success;
1763 }
1764 
1765 OperandMatchResultTy
1766 AMDGPUAsmParser::parseRegOrImmWithIntInputMods(OperandVector &Operands,
1767                                                bool AllowImm) {
1768   bool Sext = false;
1769 
1770   if (getLexer().getKind() == AsmToken::Identifier &&
1771       Parser.getTok().getString() == "sext") {
1772     Parser.Lex();
1773     Sext = true;
1774     if (getLexer().isNot(AsmToken::LParen)) {
1775       Error(Parser.getTok().getLoc(), "expected left paren after sext");
1776       return MatchOperand_ParseFail;
1777     }
1778     Parser.Lex();
1779   }
1780 
1781   OperandMatchResultTy Res;
1782   if (AllowImm) {
1783     Res = parseRegOrImm(Operands);
1784   } else {
1785     Res = parseReg(Operands);
1786   }
1787   if (Res != MatchOperand_Success) {
1788     return Res;
1789   }
1790 
1791   AMDGPUOperand::Modifiers Mods;
1792   if (Sext) {
1793     if (getLexer().isNot(AsmToken::RParen)) {
1794       Error(Parser.getTok().getLoc(), "expected closing parentheses");
1795       return MatchOperand_ParseFail;
1796     }
1797     Parser.Lex();
1798     Mods.Sext = true;
1799   }
1800 
1801   if (Mods.hasIntModifiers()) {
1802     AMDGPUOperand &Op = static_cast<AMDGPUOperand &>(*Operands.back());
1803     Op.setModifiers(Mods);
1804   }
1805 
1806   return MatchOperand_Success;
1807 }
1808 
1809 OperandMatchResultTy
1810 AMDGPUAsmParser::parseRegWithFPInputMods(OperandVector &Operands) {
1811   return parseRegOrImmWithFPInputMods(Operands, false);
1812 }
1813 
1814 OperandMatchResultTy
1815 AMDGPUAsmParser::parseRegWithIntInputMods(OperandVector &Operands) {
1816   return parseRegOrImmWithIntInputMods(Operands, false);
1817 }
1818 
1819 OperandMatchResultTy AMDGPUAsmParser::parseVReg32OrOff(OperandVector &Operands) {
1820   std::unique_ptr<AMDGPUOperand> Reg = parseRegister();
1821   if (Reg) {
1822     Operands.push_back(std::move(Reg));
1823     return MatchOperand_Success;
1824   }
1825 
1826   const AsmToken &Tok = Parser.getTok();
1827   if (Tok.getString() == "off") {
1828     Operands.push_back(AMDGPUOperand::CreateImm(this, 0, Tok.getLoc(),
1829                                                 AMDGPUOperand::ImmTyOff, false));
1830     Parser.Lex();
1831     return MatchOperand_Success;
1832   }
1833 
1834   return MatchOperand_NoMatch;
1835 }
1836 
1837 unsigned AMDGPUAsmParser::checkTargetMatchPredicate(MCInst &Inst) {
1838   uint64_t TSFlags = MII.get(Inst.getOpcode()).TSFlags;
1839 
1840   if ((getForcedEncodingSize() == 32 && (TSFlags & SIInstrFlags::VOP3)) ||
1841       (getForcedEncodingSize() == 64 && !(TSFlags & SIInstrFlags::VOP3)) ||
1842       (isForcedDPP() && !(TSFlags & SIInstrFlags::DPP)) ||
1843       (isForcedSDWA() && !(TSFlags & SIInstrFlags::SDWA)) )
1844     return Match_InvalidOperand;
1845 
1846   if ((TSFlags & SIInstrFlags::VOP3) &&
1847       (TSFlags & SIInstrFlags::VOPAsmPrefer32Bit) &&
1848       getForcedEncodingSize() != 64)
1849     return Match_PreferE32;
1850 
1851   if (Inst.getOpcode() == AMDGPU::V_MAC_F32_sdwa_vi ||
1852       Inst.getOpcode() == AMDGPU::V_MAC_F16_sdwa_vi) {
1853     // v_mac_f32/16 allow only dst_sel == DWORD;
1854     auto OpNum =
1855         AMDGPU::getNamedOperandIdx(Inst.getOpcode(), AMDGPU::OpName::dst_sel);
1856     const auto &Op = Inst.getOperand(OpNum);
1857     if (!Op.isImm() || Op.getImm() != AMDGPU::SDWA::SdwaSel::DWORD) {
1858       return Match_InvalidOperand;
1859     }
1860   }
1861 
1862   return Match_Success;
1863 }
1864 
1865 // What asm variants we should check
1866 ArrayRef<unsigned> AMDGPUAsmParser::getMatchedVariants() const {
1867   if (getForcedEncodingSize() == 32) {
1868     static const unsigned Variants[] = {AMDGPUAsmVariants::DEFAULT};
1869     return makeArrayRef(Variants);
1870   }
1871 
1872   if (isForcedVOP3()) {
1873     static const unsigned Variants[] = {AMDGPUAsmVariants::VOP3};
1874     return makeArrayRef(Variants);
1875   }
1876 
1877   if (isForcedSDWA()) {
1878     static const unsigned Variants[] = {AMDGPUAsmVariants::SDWA};
1879     return makeArrayRef(Variants);
1880   }
1881 
1882   if (isForcedDPP()) {
1883     static const unsigned Variants[] = {AMDGPUAsmVariants::DPP};
1884     return makeArrayRef(Variants);
1885   }
1886 
1887   static const unsigned Variants[] = {
1888     AMDGPUAsmVariants::DEFAULT, AMDGPUAsmVariants::VOP3,
1889     AMDGPUAsmVariants::SDWA, AMDGPUAsmVariants::DPP
1890   };
1891 
1892   return makeArrayRef(Variants);
1893 }
1894 
1895 unsigned AMDGPUAsmParser::findImplicitSGPRReadInVOP(const MCInst &Inst) const {
1896   const MCInstrDesc &Desc = MII.get(Inst.getOpcode());
1897   const unsigned Num = Desc.getNumImplicitUses();
1898   for (unsigned i = 0; i < Num; ++i) {
1899     unsigned Reg = Desc.ImplicitUses[i];
1900     switch (Reg) {
1901     case AMDGPU::FLAT_SCR:
1902     case AMDGPU::VCC:
1903     case AMDGPU::M0:
1904       return Reg;
1905     default:
1906       break;
1907     }
1908   }
1909   return AMDGPU::NoRegister;
1910 }
1911 
1912 bool AMDGPUAsmParser::isSGPR(unsigned Reg) {
1913   const MCRegisterInfo *TRI = getContext().getRegisterInfo();
1914   const MCRegisterClass SGPRClass = TRI->getRegClass(AMDGPU::SReg_32RegClassID);
1915   const unsigned FirstSubReg = TRI->getSubReg(Reg, 1);
1916   return SGPRClass.contains(FirstSubReg != 0 ? FirstSubReg : Reg) ||
1917          Reg == AMDGPU::SCC;
1918 }
1919 
1920 // NB: This code is correct only when used to check constant
1921 // bus limitations because GFX7 support no f16 inline constants.
1922 // Note that there are no cases when a GFX7 opcode violates
1923 // constant bus limitations due to the use of an f16 constant.
1924 bool AMDGPUAsmParser::isInlineConstant(const MCInst &Inst,
1925                                        unsigned OpIdx) const {
1926   const MCInstrDesc &Desc = MII.get(Inst.getOpcode());
1927 
1928   if (!AMDGPU::isSISrcOperand(Desc, OpIdx)) {
1929     return false;
1930   }
1931 
1932   const MCOperand &MO = Inst.getOperand(OpIdx);
1933 
1934   int64_t Val = MO.getImm();
1935   auto OpSize = AMDGPU::getOperandSize(Desc, OpIdx);
1936 
1937   switch (OpSize) { // expected operand size
1938   case 8:
1939     return AMDGPU::isInlinableLiteral64(Val, hasInv2PiInlineImm());
1940   case 4:
1941     return AMDGPU::isInlinableLiteral32(Val, hasInv2PiInlineImm());
1942   case 2: {
1943     const unsigned OperandType = Desc.OpInfo[OpIdx].OperandType;
1944     if (OperandType == AMDGPU::OPERAND_REG_INLINE_C_V2INT16 ||
1945         OperandType == AMDGPU::OPERAND_REG_INLINE_C_V2FP16) {
1946       return AMDGPU::isInlinableLiteralV216(Val, hasInv2PiInlineImm());
1947     } else {
1948       return AMDGPU::isInlinableLiteral16(Val, hasInv2PiInlineImm());
1949     }
1950   }
1951   default:
1952     llvm_unreachable("invalid operand size");
1953   }
1954 }
1955 
1956 bool AMDGPUAsmParser::usesConstantBus(const MCInst &Inst, unsigned OpIdx) {
1957   const MCOperand &MO = Inst.getOperand(OpIdx);
1958   if (MO.isImm()) {
1959     return !isInlineConstant(Inst, OpIdx);
1960   }
1961   return !MO.isReg() || isSGPR(mc2PseudoReg(MO.getReg()));
1962 }
1963 
1964 bool AMDGPUAsmParser::validateOperandLimitations(const MCInst &Inst) {
1965   const unsigned Opcode = Inst.getOpcode();
1966   const MCInstrDesc &Desc = MII.get(Opcode);
1967   unsigned ConstantBusUseCount = 0;
1968 
1969   if (Desc.TSFlags &
1970       (SIInstrFlags::VOPC |
1971        SIInstrFlags::VOP1 | SIInstrFlags::VOP2 |
1972        SIInstrFlags::VOP3 | SIInstrFlags::VOP3P)) {
1973 
1974     // Check special imm operands (used by madmk, etc)
1975     if (AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::imm) != -1) {
1976       ++ConstantBusUseCount;
1977     }
1978 
1979     unsigned SGPRUsed = findImplicitSGPRReadInVOP(Inst);
1980     if (SGPRUsed != AMDGPU::NoRegister) {
1981       ++ConstantBusUseCount;
1982     }
1983 
1984     const int Src0Idx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::src0);
1985     const int Src1Idx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::src1);
1986     const int Src2Idx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::src2);
1987 
1988     const int OpIndices[] = { Src0Idx, Src1Idx, Src2Idx };
1989 
1990     for (int OpIdx : OpIndices) {
1991       if (OpIdx == -1) break;
1992 
1993       const MCOperand &MO = Inst.getOperand(OpIdx);
1994       if (usesConstantBus(Inst, OpIdx)) {
1995         if (MO.isReg()) {
1996           const unsigned Reg = mc2PseudoReg(MO.getReg());
1997           // Pairs of registers with a partial intersections like these
1998           //   s0, s[0:1]
1999           //   flat_scratch_lo, flat_scratch
2000           //   flat_scratch_lo, flat_scratch_hi
2001           // are theoretically valid but they are disabled anyway.
2002           // Note that this code mimics SIInstrInfo::verifyInstruction
2003           if (Reg != SGPRUsed) {
2004             ++ConstantBusUseCount;
2005           }
2006           SGPRUsed = Reg;
2007         } else { // Expression or a literal
2008           ++ConstantBusUseCount;
2009         }
2010       }
2011     }
2012   }
2013 
2014   return ConstantBusUseCount <= 1;
2015 }
2016 
2017 bool AMDGPUAsmParser::MatchAndEmitInstruction(SMLoc IDLoc, unsigned &Opcode,
2018                                               OperandVector &Operands,
2019                                               MCStreamer &Out,
2020                                               uint64_t &ErrorInfo,
2021                                               bool MatchingInlineAsm) {
2022   MCInst Inst;
2023   unsigned Result = Match_Success;
2024   for (auto Variant : getMatchedVariants()) {
2025     uint64_t EI;
2026     auto R = MatchInstructionImpl(Operands, Inst, EI, MatchingInlineAsm,
2027                                   Variant);
2028     // We order match statuses from least to most specific. We use most specific
2029     // status as resulting
2030     // Match_MnemonicFail < Match_InvalidOperand < Match_MissingFeature < Match_PreferE32
2031     if ((R == Match_Success) ||
2032         (R == Match_PreferE32) ||
2033         (R == Match_MissingFeature && Result != Match_PreferE32) ||
2034         (R == Match_InvalidOperand && Result != Match_MissingFeature
2035                                    && Result != Match_PreferE32) ||
2036         (R == Match_MnemonicFail   && Result != Match_InvalidOperand
2037                                    && Result != Match_MissingFeature
2038                                    && Result != Match_PreferE32)) {
2039       Result = R;
2040       ErrorInfo = EI;
2041     }
2042     if (R == Match_Success)
2043       break;
2044   }
2045 
2046   switch (Result) {
2047   default: break;
2048   case Match_Success:
2049     if (!validateOperandLimitations(Inst)) {
2050       return Error(IDLoc,
2051                    "invalid operand (violates constant bus restrictions)");
2052     }
2053     Inst.setLoc(IDLoc);
2054     Out.EmitInstruction(Inst, getSTI());
2055     return false;
2056 
2057   case Match_MissingFeature:
2058     return Error(IDLoc, "instruction not supported on this GPU");
2059 
2060   case Match_MnemonicFail:
2061     return Error(IDLoc, "unrecognized instruction mnemonic");
2062 
2063   case Match_InvalidOperand: {
2064     SMLoc ErrorLoc = IDLoc;
2065     if (ErrorInfo != ~0ULL) {
2066       if (ErrorInfo >= Operands.size()) {
2067         return Error(IDLoc, "too few operands for instruction");
2068       }
2069       ErrorLoc = ((AMDGPUOperand &)*Operands[ErrorInfo]).getStartLoc();
2070       if (ErrorLoc == SMLoc())
2071         ErrorLoc = IDLoc;
2072     }
2073     return Error(ErrorLoc, "invalid operand for instruction");
2074   }
2075 
2076   case Match_PreferE32:
2077     return Error(IDLoc, "internal error: instruction without _e64 suffix "
2078                         "should be encoded as e32");
2079   }
2080   llvm_unreachable("Implement any new match types added!");
2081 }
2082 
2083 bool AMDGPUAsmParser::ParseAsAbsoluteExpression(uint32_t &Ret) {
2084   int64_t Tmp = -1;
2085   if (getLexer().isNot(AsmToken::Integer) && getLexer().isNot(AsmToken::Identifier)) {
2086     return true;
2087   }
2088   if (getParser().parseAbsoluteExpression(Tmp)) {
2089     return true;
2090   }
2091   Ret = static_cast<uint32_t>(Tmp);
2092   return false;
2093 }
2094 
2095 bool AMDGPUAsmParser::ParseDirectiveMajorMinor(uint32_t &Major,
2096                                                uint32_t &Minor) {
2097   if (ParseAsAbsoluteExpression(Major))
2098     return TokError("invalid major version");
2099 
2100   if (getLexer().isNot(AsmToken::Comma))
2101     return TokError("minor version number required, comma expected");
2102   Lex();
2103 
2104   if (ParseAsAbsoluteExpression(Minor))
2105     return TokError("invalid minor version");
2106 
2107   return false;
2108 }
2109 
2110 bool AMDGPUAsmParser::ParseDirectiveHSACodeObjectVersion() {
2111   uint32_t Major;
2112   uint32_t Minor;
2113 
2114   if (ParseDirectiveMajorMinor(Major, Minor))
2115     return true;
2116 
2117   getTargetStreamer().EmitDirectiveHSACodeObjectVersion(Major, Minor);
2118   return false;
2119 }
2120 
2121 bool AMDGPUAsmParser::ParseDirectiveHSACodeObjectISA() {
2122   uint32_t Major;
2123   uint32_t Minor;
2124   uint32_t Stepping;
2125   StringRef VendorName;
2126   StringRef ArchName;
2127 
2128   // If this directive has no arguments, then use the ISA version for the
2129   // targeted GPU.
2130   if (getLexer().is(AsmToken::EndOfStatement)) {
2131     AMDGPU::IsaInfo::IsaVersion ISA =
2132         AMDGPU::IsaInfo::getIsaVersion(getFeatureBits());
2133     getTargetStreamer().EmitDirectiveHSACodeObjectISA(ISA.Major, ISA.Minor,
2134                                                       ISA.Stepping,
2135                                                       "AMD", "AMDGPU");
2136     return false;
2137   }
2138 
2139   if (ParseDirectiveMajorMinor(Major, Minor))
2140     return true;
2141 
2142   if (getLexer().isNot(AsmToken::Comma))
2143     return TokError("stepping version number required, comma expected");
2144   Lex();
2145 
2146   if (ParseAsAbsoluteExpression(Stepping))
2147     return TokError("invalid stepping version");
2148 
2149   if (getLexer().isNot(AsmToken::Comma))
2150     return TokError("vendor name required, comma expected");
2151   Lex();
2152 
2153   if (getLexer().isNot(AsmToken::String))
2154     return TokError("invalid vendor name");
2155 
2156   VendorName = getLexer().getTok().getStringContents();
2157   Lex();
2158 
2159   if (getLexer().isNot(AsmToken::Comma))
2160     return TokError("arch name required, comma expected");
2161   Lex();
2162 
2163   if (getLexer().isNot(AsmToken::String))
2164     return TokError("invalid arch name");
2165 
2166   ArchName = getLexer().getTok().getStringContents();
2167   Lex();
2168 
2169   getTargetStreamer().EmitDirectiveHSACodeObjectISA(Major, Minor, Stepping,
2170                                                     VendorName, ArchName);
2171   return false;
2172 }
2173 
2174 bool AMDGPUAsmParser::ParseDirectiveRuntimeMetadata() {
2175   std::string Metadata;
2176   raw_string_ostream MS(Metadata);
2177 
2178   getLexer().setSkipSpace(false);
2179 
2180   bool FoundEnd = false;
2181   while (!getLexer().is(AsmToken::Eof)) {
2182     while (getLexer().is(AsmToken::Space)) {
2183       MS << ' ';
2184       Lex();
2185     }
2186 
2187     if (getLexer().is(AsmToken::Identifier)) {
2188       StringRef ID = getLexer().getTok().getIdentifier();
2189       if (ID == ".end_amdgpu_runtime_metadata") {
2190         Lex();
2191         FoundEnd = true;
2192         break;
2193       }
2194     }
2195 
2196     MS << Parser.parseStringToEndOfStatement()
2197        << getContext().getAsmInfo()->getSeparatorString();
2198 
2199     Parser.eatToEndOfStatement();
2200   }
2201 
2202   getLexer().setSkipSpace(true);
2203 
2204   if (getLexer().is(AsmToken::Eof) && !FoundEnd)
2205     return TokError("expected directive .end_amdgpu_runtime_metadata not found");
2206 
2207   MS.flush();
2208 
2209   if (getTargetStreamer().EmitRuntimeMetadata(getFeatureBits(), Metadata))
2210     return Error(getParser().getTok().getLoc(), "invalid runtime metadata");
2211 
2212   return false;
2213 }
2214 
2215 bool AMDGPUAsmParser::ParseAMDKernelCodeTValue(StringRef ID,
2216                                                amd_kernel_code_t &Header) {
2217   SmallString<40> ErrStr;
2218   raw_svector_ostream Err(ErrStr);
2219   if (!parseAmdKernelCodeField(ID, getParser(), Header, Err)) {
2220     return TokError(Err.str());
2221   }
2222   Lex();
2223   return false;
2224 }
2225 
2226 bool AMDGPUAsmParser::ParseDirectiveAMDKernelCodeT() {
2227   amd_kernel_code_t Header;
2228   AMDGPU::initDefaultAMDKernelCodeT(Header, getFeatureBits());
2229 
2230   while (true) {
2231     // Lex EndOfStatement.  This is in a while loop, because lexing a comment
2232     // will set the current token to EndOfStatement.
2233     while(getLexer().is(AsmToken::EndOfStatement))
2234       Lex();
2235 
2236     if (getLexer().isNot(AsmToken::Identifier))
2237       return TokError("expected value identifier or .end_amd_kernel_code_t");
2238 
2239     StringRef ID = getLexer().getTok().getIdentifier();
2240     Lex();
2241 
2242     if (ID == ".end_amd_kernel_code_t")
2243       break;
2244 
2245     if (ParseAMDKernelCodeTValue(ID, Header))
2246       return true;
2247   }
2248 
2249   getTargetStreamer().EmitAMDKernelCodeT(Header);
2250 
2251   return false;
2252 }
2253 
2254 bool AMDGPUAsmParser::ParseSectionDirectiveHSAText() {
2255   getParser().getStreamer().SwitchSection(
2256       AMDGPU::getHSATextSection(getContext()));
2257   return false;
2258 }
2259 
2260 bool AMDGPUAsmParser::ParseDirectiveAMDGPUHsaKernel() {
2261   if (getLexer().isNot(AsmToken::Identifier))
2262     return TokError("expected symbol name");
2263 
2264   StringRef KernelName = Parser.getTok().getString();
2265 
2266   getTargetStreamer().EmitAMDGPUSymbolType(KernelName,
2267                                            ELF::STT_AMDGPU_HSA_KERNEL);
2268   Lex();
2269   KernelScope.initialize(getContext());
2270   return false;
2271 }
2272 
2273 bool AMDGPUAsmParser::ParseDirectiveAMDGPUHsaModuleGlobal() {
2274   if (getLexer().isNot(AsmToken::Identifier))
2275     return TokError("expected symbol name");
2276 
2277   StringRef GlobalName = Parser.getTok().getIdentifier();
2278 
2279   getTargetStreamer().EmitAMDGPUHsaModuleScopeGlobal(GlobalName);
2280   Lex();
2281   return false;
2282 }
2283 
2284 bool AMDGPUAsmParser::ParseDirectiveAMDGPUHsaProgramGlobal() {
2285   if (getLexer().isNot(AsmToken::Identifier))
2286     return TokError("expected symbol name");
2287 
2288   StringRef GlobalName = Parser.getTok().getIdentifier();
2289 
2290   getTargetStreamer().EmitAMDGPUHsaProgramScopeGlobal(GlobalName);
2291   Lex();
2292   return false;
2293 }
2294 
2295 bool AMDGPUAsmParser::ParseSectionDirectiveHSADataGlobalAgent() {
2296   getParser().getStreamer().SwitchSection(
2297       AMDGPU::getHSADataGlobalAgentSection(getContext()));
2298   return false;
2299 }
2300 
2301 bool AMDGPUAsmParser::ParseSectionDirectiveHSADataGlobalProgram() {
2302   getParser().getStreamer().SwitchSection(
2303       AMDGPU::getHSADataGlobalProgramSection(getContext()));
2304   return false;
2305 }
2306 
2307 bool AMDGPUAsmParser::ParseSectionDirectiveHSARodataReadonlyAgent() {
2308   getParser().getStreamer().SwitchSection(
2309       AMDGPU::getHSARodataReadonlyAgentSection(getContext()));
2310   return false;
2311 }
2312 
2313 bool AMDGPUAsmParser::ParseDirective(AsmToken DirectiveID) {
2314   StringRef IDVal = DirectiveID.getString();
2315 
2316   if (IDVal == ".hsa_code_object_version")
2317     return ParseDirectiveHSACodeObjectVersion();
2318 
2319   if (IDVal == ".hsa_code_object_isa")
2320     return ParseDirectiveHSACodeObjectISA();
2321 
2322   if (IDVal == ".amdgpu_runtime_metadata")
2323     return ParseDirectiveRuntimeMetadata();
2324 
2325   if (IDVal == ".amd_kernel_code_t")
2326     return ParseDirectiveAMDKernelCodeT();
2327 
2328   if (IDVal == ".hsatext")
2329     return ParseSectionDirectiveHSAText();
2330 
2331   if (IDVal == ".amdgpu_hsa_kernel")
2332     return ParseDirectiveAMDGPUHsaKernel();
2333 
2334   if (IDVal == ".amdgpu_hsa_module_global")
2335     return ParseDirectiveAMDGPUHsaModuleGlobal();
2336 
2337   if (IDVal == ".amdgpu_hsa_program_global")
2338     return ParseDirectiveAMDGPUHsaProgramGlobal();
2339 
2340   if (IDVal == ".hsadata_global_agent")
2341     return ParseSectionDirectiveHSADataGlobalAgent();
2342 
2343   if (IDVal == ".hsadata_global_program")
2344     return ParseSectionDirectiveHSADataGlobalProgram();
2345 
2346   if (IDVal == ".hsarodata_readonly_agent")
2347     return ParseSectionDirectiveHSARodataReadonlyAgent();
2348 
2349   return true;
2350 }
2351 
2352 bool AMDGPUAsmParser::subtargetHasRegister(const MCRegisterInfo &MRI,
2353                                            unsigned RegNo) const {
2354   if (isCI())
2355     return true;
2356 
2357   if (isSI()) {
2358     // No flat_scr
2359     switch (RegNo) {
2360     case AMDGPU::FLAT_SCR:
2361     case AMDGPU::FLAT_SCR_LO:
2362     case AMDGPU::FLAT_SCR_HI:
2363       return false;
2364     default:
2365       return true;
2366     }
2367   }
2368 
2369   // VI only has 102 SGPRs, so make sure we aren't trying to use the 2 more that
2370   // SI/CI have.
2371   for (MCRegAliasIterator R(AMDGPU::SGPR102_SGPR103, &MRI, true);
2372        R.isValid(); ++R) {
2373     if (*R == RegNo)
2374       return false;
2375   }
2376 
2377   return true;
2378 }
2379 
2380 OperandMatchResultTy
2381 AMDGPUAsmParser::parseOperand(OperandVector &Operands, StringRef Mnemonic) {
2382   // Try to parse with a custom parser
2383   OperandMatchResultTy ResTy = MatchOperandParserImpl(Operands, Mnemonic);
2384 
2385   // If we successfully parsed the operand or if there as an error parsing,
2386   // we are done.
2387   //
2388   // If we are parsing after we reach EndOfStatement then this means we
2389   // are appending default values to the Operands list.  This is only done
2390   // by custom parser, so we shouldn't continue on to the generic parsing.
2391   if (ResTy == MatchOperand_Success || ResTy == MatchOperand_ParseFail ||
2392       getLexer().is(AsmToken::EndOfStatement))
2393     return ResTy;
2394 
2395   ResTy = parseRegOrImm(Operands);
2396 
2397   if (ResTy == MatchOperand_Success)
2398     return ResTy;
2399 
2400   if (getLexer().getKind() == AsmToken::Identifier) {
2401     // If this identifier is a symbol, we want to create an expression for it.
2402     // It is a little difficult to distinguish between a symbol name, and
2403     // an instruction flag like 'gds'.  In order to do this, we parse
2404     // all tokens as expressions and then treate the symbol name as the token
2405     // string when we want to interpret the operand as a token.
2406     const auto &Tok = Parser.getTok();
2407     SMLoc S = Tok.getLoc();
2408     const MCExpr *Expr = nullptr;
2409     if (!Parser.parseExpression(Expr)) {
2410       Operands.push_back(AMDGPUOperand::CreateExpr(this, Expr, S));
2411       return MatchOperand_Success;
2412     }
2413 
2414     Operands.push_back(AMDGPUOperand::CreateToken(this, Tok.getString(), Tok.getLoc()));
2415     Parser.Lex();
2416     return MatchOperand_Success;
2417   }
2418   return MatchOperand_NoMatch;
2419 }
2420 
2421 StringRef AMDGPUAsmParser::parseMnemonicSuffix(StringRef Name) {
2422   // Clear any forced encodings from the previous instruction.
2423   setForcedEncodingSize(0);
2424   setForcedDPP(false);
2425   setForcedSDWA(false);
2426 
2427   if (Name.endswith("_e64")) {
2428     setForcedEncodingSize(64);
2429     return Name.substr(0, Name.size() - 4);
2430   } else if (Name.endswith("_e32")) {
2431     setForcedEncodingSize(32);
2432     return Name.substr(0, Name.size() - 4);
2433   } else if (Name.endswith("_dpp")) {
2434     setForcedDPP(true);
2435     return Name.substr(0, Name.size() - 4);
2436   } else if (Name.endswith("_sdwa")) {
2437     setForcedSDWA(true);
2438     return Name.substr(0, Name.size() - 5);
2439   }
2440   return Name;
2441 }
2442 
2443 bool AMDGPUAsmParser::ParseInstruction(ParseInstructionInfo &Info,
2444                                        StringRef Name,
2445                                        SMLoc NameLoc, OperandVector &Operands) {
2446   // Add the instruction mnemonic
2447   Name = parseMnemonicSuffix(Name);
2448   Operands.push_back(AMDGPUOperand::CreateToken(this, Name, NameLoc));
2449 
2450   while (!getLexer().is(AsmToken::EndOfStatement)) {
2451     OperandMatchResultTy Res = parseOperand(Operands, Name);
2452 
2453     // Eat the comma or space if there is one.
2454     if (getLexer().is(AsmToken::Comma))
2455       Parser.Lex();
2456 
2457     switch (Res) {
2458       case MatchOperand_Success: break;
2459       case MatchOperand_ParseFail:
2460         Error(getLexer().getLoc(), "failed parsing operand.");
2461         while (!getLexer().is(AsmToken::EndOfStatement)) {
2462           Parser.Lex();
2463         }
2464         return true;
2465       case MatchOperand_NoMatch:
2466         Error(getLexer().getLoc(), "not a valid operand.");
2467         while (!getLexer().is(AsmToken::EndOfStatement)) {
2468           Parser.Lex();
2469         }
2470         return true;
2471     }
2472   }
2473 
2474   return false;
2475 }
2476 
2477 //===----------------------------------------------------------------------===//
2478 // Utility functions
2479 //===----------------------------------------------------------------------===//
2480 
2481 OperandMatchResultTy
2482 AMDGPUAsmParser::parseIntWithPrefix(const char *Prefix, int64_t &Int) {
2483   switch(getLexer().getKind()) {
2484     default: return MatchOperand_NoMatch;
2485     case AsmToken::Identifier: {
2486       StringRef Name = Parser.getTok().getString();
2487       if (!Name.equals(Prefix)) {
2488         return MatchOperand_NoMatch;
2489       }
2490 
2491       Parser.Lex();
2492       if (getLexer().isNot(AsmToken::Colon))
2493         return MatchOperand_ParseFail;
2494 
2495       Parser.Lex();
2496       if (getLexer().isNot(AsmToken::Integer))
2497         return MatchOperand_ParseFail;
2498 
2499       if (getParser().parseAbsoluteExpression(Int))
2500         return MatchOperand_ParseFail;
2501       break;
2502     }
2503   }
2504   return MatchOperand_Success;
2505 }
2506 
2507 OperandMatchResultTy
2508 AMDGPUAsmParser::parseIntWithPrefix(const char *Prefix, OperandVector &Operands,
2509                                     AMDGPUOperand::ImmTy ImmTy,
2510                                     bool (*ConvertResult)(int64_t&)) {
2511   SMLoc S = Parser.getTok().getLoc();
2512   int64_t Value = 0;
2513 
2514   OperandMatchResultTy Res = parseIntWithPrefix(Prefix, Value);
2515   if (Res != MatchOperand_Success)
2516     return Res;
2517 
2518   if (ConvertResult && !ConvertResult(Value)) {
2519     return MatchOperand_ParseFail;
2520   }
2521 
2522   Operands.push_back(AMDGPUOperand::CreateImm(this, Value, S, ImmTy));
2523   return MatchOperand_Success;
2524 }
2525 
2526 OperandMatchResultTy AMDGPUAsmParser::parseOperandArrayWithPrefix(
2527   const char *Prefix,
2528   OperandVector &Operands,
2529   AMDGPUOperand::ImmTy ImmTy,
2530   bool (*ConvertResult)(int64_t&)) {
2531   StringRef Name = Parser.getTok().getString();
2532   if (!Name.equals(Prefix))
2533     return MatchOperand_NoMatch;
2534 
2535   Parser.Lex();
2536   if (getLexer().isNot(AsmToken::Colon))
2537     return MatchOperand_ParseFail;
2538 
2539   Parser.Lex();
2540   if (getLexer().isNot(AsmToken::LBrac))
2541     return MatchOperand_ParseFail;
2542   Parser.Lex();
2543 
2544   unsigned Val = 0;
2545   SMLoc S = Parser.getTok().getLoc();
2546 
2547   // FIXME: How to verify the number of elements matches the number of src
2548   // operands?
2549   for (int I = 0; I < 3; ++I) {
2550     if (I != 0) {
2551       if (getLexer().is(AsmToken::RBrac))
2552         break;
2553 
2554       if (getLexer().isNot(AsmToken::Comma))
2555         return MatchOperand_ParseFail;
2556       Parser.Lex();
2557     }
2558 
2559     if (getLexer().isNot(AsmToken::Integer))
2560       return MatchOperand_ParseFail;
2561 
2562     int64_t Op;
2563     if (getParser().parseAbsoluteExpression(Op))
2564       return MatchOperand_ParseFail;
2565 
2566     if (Op != 0 && Op != 1)
2567       return MatchOperand_ParseFail;
2568     Val |= (Op << I);
2569   }
2570 
2571   Parser.Lex();
2572   Operands.push_back(AMDGPUOperand::CreateImm(this, Val, S, ImmTy));
2573   return MatchOperand_Success;
2574 }
2575 
2576 OperandMatchResultTy
2577 AMDGPUAsmParser::parseNamedBit(const char *Name, OperandVector &Operands,
2578                                AMDGPUOperand::ImmTy ImmTy) {
2579   int64_t Bit = 0;
2580   SMLoc S = Parser.getTok().getLoc();
2581 
2582   // We are at the end of the statement, and this is a default argument, so
2583   // use a default value.
2584   if (getLexer().isNot(AsmToken::EndOfStatement)) {
2585     switch(getLexer().getKind()) {
2586       case AsmToken::Identifier: {
2587         StringRef Tok = Parser.getTok().getString();
2588         if (Tok == Name) {
2589           Bit = 1;
2590           Parser.Lex();
2591         } else if (Tok.startswith("no") && Tok.endswith(Name)) {
2592           Bit = 0;
2593           Parser.Lex();
2594         } else {
2595           return MatchOperand_NoMatch;
2596         }
2597         break;
2598       }
2599       default:
2600         return MatchOperand_NoMatch;
2601     }
2602   }
2603 
2604   Operands.push_back(AMDGPUOperand::CreateImm(this, Bit, S, ImmTy));
2605   return MatchOperand_Success;
2606 }
2607 
2608 static void addOptionalImmOperand(
2609   MCInst& Inst, const OperandVector& Operands,
2610   AMDGPUAsmParser::OptionalImmIndexMap& OptionalIdx,
2611   AMDGPUOperand::ImmTy ImmT,
2612   int64_t Default = 0) {
2613   auto i = OptionalIdx.find(ImmT);
2614   if (i != OptionalIdx.end()) {
2615     unsigned Idx = i->second;
2616     ((AMDGPUOperand &)*Operands[Idx]).addImmOperands(Inst, 1);
2617   } else {
2618     Inst.addOperand(MCOperand::createImm(Default));
2619   }
2620 }
2621 
2622 OperandMatchResultTy
2623 AMDGPUAsmParser::parseStringWithPrefix(StringRef Prefix, StringRef &Value) {
2624   if (getLexer().isNot(AsmToken::Identifier)) {
2625     return MatchOperand_NoMatch;
2626   }
2627   StringRef Tok = Parser.getTok().getString();
2628   if (Tok != Prefix) {
2629     return MatchOperand_NoMatch;
2630   }
2631 
2632   Parser.Lex();
2633   if (getLexer().isNot(AsmToken::Colon)) {
2634     return MatchOperand_ParseFail;
2635   }
2636 
2637   Parser.Lex();
2638   if (getLexer().isNot(AsmToken::Identifier)) {
2639     return MatchOperand_ParseFail;
2640   }
2641 
2642   Value = Parser.getTok().getString();
2643   return MatchOperand_Success;
2644 }
2645 
2646 //===----------------------------------------------------------------------===//
2647 // ds
2648 //===----------------------------------------------------------------------===//
2649 
2650 void AMDGPUAsmParser::cvtDSOffset01(MCInst &Inst,
2651                                     const OperandVector &Operands) {
2652   OptionalImmIndexMap OptionalIdx;
2653 
2654   for (unsigned i = 1, e = Operands.size(); i != e; ++i) {
2655     AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[i]);
2656 
2657     // Add the register arguments
2658     if (Op.isReg()) {
2659       Op.addRegOperands(Inst, 1);
2660       continue;
2661     }
2662 
2663     // Handle optional arguments
2664     OptionalIdx[Op.getImmTy()] = i;
2665   }
2666 
2667   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyOffset0);
2668   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyOffset1);
2669   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyGDS);
2670 
2671   Inst.addOperand(MCOperand::createReg(AMDGPU::M0)); // m0
2672 }
2673 
2674 void AMDGPUAsmParser::cvtDSImpl(MCInst &Inst, const OperandVector &Operands,
2675                                 bool IsGdsHardcoded) {
2676   OptionalImmIndexMap OptionalIdx;
2677 
2678   for (unsigned i = 1, e = Operands.size(); i != e; ++i) {
2679     AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[i]);
2680 
2681     // Add the register arguments
2682     if (Op.isReg()) {
2683       Op.addRegOperands(Inst, 1);
2684       continue;
2685     }
2686 
2687     if (Op.isToken() && Op.getToken() == "gds") {
2688       IsGdsHardcoded = true;
2689       continue;
2690     }
2691 
2692     // Handle optional arguments
2693     OptionalIdx[Op.getImmTy()] = i;
2694   }
2695 
2696   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyOffset);
2697   if (!IsGdsHardcoded) {
2698     addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyGDS);
2699   }
2700   Inst.addOperand(MCOperand::createReg(AMDGPU::M0)); // m0
2701 }
2702 
2703 void AMDGPUAsmParser::cvtExp(MCInst &Inst, const OperandVector &Operands) {
2704   OptionalImmIndexMap OptionalIdx;
2705 
2706   unsigned EnMask = 0;
2707   int SrcIdx = 0;
2708 
2709   for (unsigned i = 1, e = Operands.size(); i != e; ++i) {
2710     AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[i]);
2711 
2712     // Add the register arguments
2713     if (Op.isReg()) {
2714       EnMask |= (1 << SrcIdx);
2715       Op.addRegOperands(Inst, 1);
2716       ++SrcIdx;
2717       continue;
2718     }
2719 
2720     if (Op.isOff()) {
2721       ++SrcIdx;
2722       Inst.addOperand(MCOperand::createReg(AMDGPU::NoRegister));
2723       continue;
2724     }
2725 
2726     if (Op.isImm() && Op.getImmTy() == AMDGPUOperand::ImmTyExpTgt) {
2727       Op.addImmOperands(Inst, 1);
2728       continue;
2729     }
2730 
2731     if (Op.isToken() && Op.getToken() == "done")
2732       continue;
2733 
2734     // Handle optional arguments
2735     OptionalIdx[Op.getImmTy()] = i;
2736   }
2737 
2738   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyExpVM);
2739   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyExpCompr);
2740 
2741   Inst.addOperand(MCOperand::createImm(EnMask));
2742 }
2743 
2744 //===----------------------------------------------------------------------===//
2745 // s_waitcnt
2746 //===----------------------------------------------------------------------===//
2747 
2748 bool AMDGPUAsmParser::parseCnt(int64_t &IntVal) {
2749   StringRef CntName = Parser.getTok().getString();
2750   int64_t CntVal;
2751 
2752   Parser.Lex();
2753   if (getLexer().isNot(AsmToken::LParen))
2754     return true;
2755 
2756   Parser.Lex();
2757   if (getLexer().isNot(AsmToken::Integer))
2758     return true;
2759 
2760   if (getParser().parseAbsoluteExpression(CntVal))
2761     return true;
2762 
2763   if (getLexer().isNot(AsmToken::RParen))
2764     return true;
2765 
2766   Parser.Lex();
2767   if (getLexer().is(AsmToken::Amp) || getLexer().is(AsmToken::Comma))
2768     Parser.Lex();
2769 
2770   AMDGPU::IsaInfo::IsaVersion ISA =
2771       AMDGPU::IsaInfo::getIsaVersion(getFeatureBits());
2772   if (CntName == "vmcnt")
2773     IntVal = encodeVmcnt(ISA, IntVal, CntVal);
2774   else if (CntName == "expcnt")
2775     IntVal = encodeExpcnt(ISA, IntVal, CntVal);
2776   else if (CntName == "lgkmcnt")
2777     IntVal = encodeLgkmcnt(ISA, IntVal, CntVal);
2778   else
2779     return true;
2780 
2781   return false;
2782 }
2783 
2784 OperandMatchResultTy
2785 AMDGPUAsmParser::parseSWaitCntOps(OperandVector &Operands) {
2786   AMDGPU::IsaInfo::IsaVersion ISA =
2787       AMDGPU::IsaInfo::getIsaVersion(getFeatureBits());
2788   int64_t Waitcnt = getWaitcntBitMask(ISA);
2789   SMLoc S = Parser.getTok().getLoc();
2790 
2791   switch(getLexer().getKind()) {
2792     default: return MatchOperand_ParseFail;
2793     case AsmToken::Integer:
2794       // The operand can be an integer value.
2795       if (getParser().parseAbsoluteExpression(Waitcnt))
2796         return MatchOperand_ParseFail;
2797       break;
2798 
2799     case AsmToken::Identifier:
2800       do {
2801         if (parseCnt(Waitcnt))
2802           return MatchOperand_ParseFail;
2803       } while(getLexer().isNot(AsmToken::EndOfStatement));
2804       break;
2805   }
2806   Operands.push_back(AMDGPUOperand::CreateImm(this, Waitcnt, S));
2807   return MatchOperand_Success;
2808 }
2809 
2810 bool AMDGPUAsmParser::parseHwregConstruct(OperandInfoTy &HwReg, int64_t &Offset,
2811                                           int64_t &Width) {
2812   using namespace llvm::AMDGPU::Hwreg;
2813 
2814   if (Parser.getTok().getString() != "hwreg")
2815     return true;
2816   Parser.Lex();
2817 
2818   if (getLexer().isNot(AsmToken::LParen))
2819     return true;
2820   Parser.Lex();
2821 
2822   if (getLexer().is(AsmToken::Identifier)) {
2823     HwReg.IsSymbolic = true;
2824     HwReg.Id = ID_UNKNOWN_;
2825     const StringRef tok = Parser.getTok().getString();
2826     for (int i = ID_SYMBOLIC_FIRST_; i < ID_SYMBOLIC_LAST_; ++i) {
2827       if (tok == IdSymbolic[i]) {
2828         HwReg.Id = i;
2829         break;
2830       }
2831     }
2832     Parser.Lex();
2833   } else {
2834     HwReg.IsSymbolic = false;
2835     if (getLexer().isNot(AsmToken::Integer))
2836       return true;
2837     if (getParser().parseAbsoluteExpression(HwReg.Id))
2838       return true;
2839   }
2840 
2841   if (getLexer().is(AsmToken::RParen)) {
2842     Parser.Lex();
2843     return false;
2844   }
2845 
2846   // optional params
2847   if (getLexer().isNot(AsmToken::Comma))
2848     return true;
2849   Parser.Lex();
2850 
2851   if (getLexer().isNot(AsmToken::Integer))
2852     return true;
2853   if (getParser().parseAbsoluteExpression(Offset))
2854     return true;
2855 
2856   if (getLexer().isNot(AsmToken::Comma))
2857     return true;
2858   Parser.Lex();
2859 
2860   if (getLexer().isNot(AsmToken::Integer))
2861     return true;
2862   if (getParser().parseAbsoluteExpression(Width))
2863     return true;
2864 
2865   if (getLexer().isNot(AsmToken::RParen))
2866     return true;
2867   Parser.Lex();
2868 
2869   return false;
2870 }
2871 
2872 OperandMatchResultTy AMDGPUAsmParser::parseHwreg(OperandVector &Operands) {
2873   using namespace llvm::AMDGPU::Hwreg;
2874 
2875   int64_t Imm16Val = 0;
2876   SMLoc S = Parser.getTok().getLoc();
2877 
2878   switch(getLexer().getKind()) {
2879     default: return MatchOperand_NoMatch;
2880     case AsmToken::Integer:
2881       // The operand can be an integer value.
2882       if (getParser().parseAbsoluteExpression(Imm16Val))
2883         return MatchOperand_NoMatch;
2884       if (Imm16Val < 0 || !isUInt<16>(Imm16Val)) {
2885         Error(S, "invalid immediate: only 16-bit values are legal");
2886         // Do not return error code, but create an imm operand anyway and proceed
2887         // to the next operand, if any. That avoids unneccessary error messages.
2888       }
2889       break;
2890 
2891     case AsmToken::Identifier: {
2892         OperandInfoTy HwReg(ID_UNKNOWN_);
2893         int64_t Offset = OFFSET_DEFAULT_;
2894         int64_t Width = WIDTH_M1_DEFAULT_ + 1;
2895         if (parseHwregConstruct(HwReg, Offset, Width))
2896           return MatchOperand_ParseFail;
2897         if (HwReg.Id < 0 || !isUInt<ID_WIDTH_>(HwReg.Id)) {
2898           if (HwReg.IsSymbolic)
2899             Error(S, "invalid symbolic name of hardware register");
2900           else
2901             Error(S, "invalid code of hardware register: only 6-bit values are legal");
2902         }
2903         if (Offset < 0 || !isUInt<OFFSET_WIDTH_>(Offset))
2904           Error(S, "invalid bit offset: only 5-bit values are legal");
2905         if ((Width-1) < 0 || !isUInt<WIDTH_M1_WIDTH_>(Width-1))
2906           Error(S, "invalid bitfield width: only values from 1 to 32 are legal");
2907         Imm16Val = (HwReg.Id << ID_SHIFT_) | (Offset << OFFSET_SHIFT_) | ((Width-1) << WIDTH_M1_SHIFT_);
2908       }
2909       break;
2910   }
2911   Operands.push_back(AMDGPUOperand::CreateImm(this, Imm16Val, S, AMDGPUOperand::ImmTyHwreg));
2912   return MatchOperand_Success;
2913 }
2914 
2915 bool AMDGPUOperand::isSWaitCnt() const {
2916   return isImm();
2917 }
2918 
2919 bool AMDGPUOperand::isHwreg() const {
2920   return isImmTy(ImmTyHwreg);
2921 }
2922 
2923 bool AMDGPUAsmParser::parseSendMsgConstruct(OperandInfoTy &Msg, OperandInfoTy &Operation, int64_t &StreamId) {
2924   using namespace llvm::AMDGPU::SendMsg;
2925 
2926   if (Parser.getTok().getString() != "sendmsg")
2927     return true;
2928   Parser.Lex();
2929 
2930   if (getLexer().isNot(AsmToken::LParen))
2931     return true;
2932   Parser.Lex();
2933 
2934   if (getLexer().is(AsmToken::Identifier)) {
2935     Msg.IsSymbolic = true;
2936     Msg.Id = ID_UNKNOWN_;
2937     const std::string tok = Parser.getTok().getString();
2938     for (int i = ID_GAPS_FIRST_; i < ID_GAPS_LAST_; ++i) {
2939       switch(i) {
2940         default: continue; // Omit gaps.
2941         case ID_INTERRUPT: case ID_GS: case ID_GS_DONE:  case ID_SYSMSG: break;
2942       }
2943       if (tok == IdSymbolic[i]) {
2944         Msg.Id = i;
2945         break;
2946       }
2947     }
2948     Parser.Lex();
2949   } else {
2950     Msg.IsSymbolic = false;
2951     if (getLexer().isNot(AsmToken::Integer))
2952       return true;
2953     if (getParser().parseAbsoluteExpression(Msg.Id))
2954       return true;
2955     if (getLexer().is(AsmToken::Integer))
2956       if (getParser().parseAbsoluteExpression(Msg.Id))
2957         Msg.Id = ID_UNKNOWN_;
2958   }
2959   if (Msg.Id == ID_UNKNOWN_) // Don't know how to parse the rest.
2960     return false;
2961 
2962   if (!(Msg.Id == ID_GS || Msg.Id == ID_GS_DONE || Msg.Id == ID_SYSMSG)) {
2963     if (getLexer().isNot(AsmToken::RParen))
2964       return true;
2965     Parser.Lex();
2966     return false;
2967   }
2968 
2969   if (getLexer().isNot(AsmToken::Comma))
2970     return true;
2971   Parser.Lex();
2972 
2973   assert(Msg.Id == ID_GS || Msg.Id == ID_GS_DONE || Msg.Id == ID_SYSMSG);
2974   Operation.Id = ID_UNKNOWN_;
2975   if (getLexer().is(AsmToken::Identifier)) {
2976     Operation.IsSymbolic = true;
2977     const char* const *S = (Msg.Id == ID_SYSMSG) ? OpSysSymbolic : OpGsSymbolic;
2978     const int F = (Msg.Id == ID_SYSMSG) ? OP_SYS_FIRST_ : OP_GS_FIRST_;
2979     const int L = (Msg.Id == ID_SYSMSG) ? OP_SYS_LAST_ : OP_GS_LAST_;
2980     const StringRef Tok = Parser.getTok().getString();
2981     for (int i = F; i < L; ++i) {
2982       if (Tok == S[i]) {
2983         Operation.Id = i;
2984         break;
2985       }
2986     }
2987     Parser.Lex();
2988   } else {
2989     Operation.IsSymbolic = false;
2990     if (getLexer().isNot(AsmToken::Integer))
2991       return true;
2992     if (getParser().parseAbsoluteExpression(Operation.Id))
2993       return true;
2994   }
2995 
2996   if ((Msg.Id == ID_GS || Msg.Id == ID_GS_DONE) && Operation.Id != OP_GS_NOP) {
2997     // Stream id is optional.
2998     if (getLexer().is(AsmToken::RParen)) {
2999       Parser.Lex();
3000       return false;
3001     }
3002 
3003     if (getLexer().isNot(AsmToken::Comma))
3004       return true;
3005     Parser.Lex();
3006 
3007     if (getLexer().isNot(AsmToken::Integer))
3008       return true;
3009     if (getParser().parseAbsoluteExpression(StreamId))
3010       return true;
3011   }
3012 
3013   if (getLexer().isNot(AsmToken::RParen))
3014     return true;
3015   Parser.Lex();
3016   return false;
3017 }
3018 
3019 OperandMatchResultTy AMDGPUAsmParser::parseInterpSlot(OperandVector &Operands) {
3020   if (getLexer().getKind() != AsmToken::Identifier)
3021     return MatchOperand_NoMatch;
3022 
3023   StringRef Str = Parser.getTok().getString();
3024   int Slot = StringSwitch<int>(Str)
3025     .Case("p10", 0)
3026     .Case("p20", 1)
3027     .Case("p0", 2)
3028     .Default(-1);
3029 
3030   SMLoc S = Parser.getTok().getLoc();
3031   if (Slot == -1)
3032     return MatchOperand_ParseFail;
3033 
3034   Parser.Lex();
3035   Operands.push_back(AMDGPUOperand::CreateImm(this, Slot, S,
3036                                               AMDGPUOperand::ImmTyInterpSlot));
3037   return MatchOperand_Success;
3038 }
3039 
3040 OperandMatchResultTy AMDGPUAsmParser::parseInterpAttr(OperandVector &Operands) {
3041   if (getLexer().getKind() != AsmToken::Identifier)
3042     return MatchOperand_NoMatch;
3043 
3044   StringRef Str = Parser.getTok().getString();
3045   if (!Str.startswith("attr"))
3046     return MatchOperand_NoMatch;
3047 
3048   StringRef Chan = Str.take_back(2);
3049   int AttrChan = StringSwitch<int>(Chan)
3050     .Case(".x", 0)
3051     .Case(".y", 1)
3052     .Case(".z", 2)
3053     .Case(".w", 3)
3054     .Default(-1);
3055   if (AttrChan == -1)
3056     return MatchOperand_ParseFail;
3057 
3058   Str = Str.drop_back(2).drop_front(4);
3059 
3060   uint8_t Attr;
3061   if (Str.getAsInteger(10, Attr))
3062     return MatchOperand_ParseFail;
3063 
3064   SMLoc S = Parser.getTok().getLoc();
3065   Parser.Lex();
3066   if (Attr > 63) {
3067     Error(S, "out of bounds attr");
3068     return MatchOperand_Success;
3069   }
3070 
3071   SMLoc SChan = SMLoc::getFromPointer(Chan.data());
3072 
3073   Operands.push_back(AMDGPUOperand::CreateImm(this, Attr, S,
3074                                               AMDGPUOperand::ImmTyInterpAttr));
3075   Operands.push_back(AMDGPUOperand::CreateImm(this, AttrChan, SChan,
3076                                               AMDGPUOperand::ImmTyAttrChan));
3077   return MatchOperand_Success;
3078 }
3079 
3080 void AMDGPUAsmParser::errorExpTgt() {
3081   Error(Parser.getTok().getLoc(), "invalid exp target");
3082 }
3083 
3084 OperandMatchResultTy AMDGPUAsmParser::parseExpTgtImpl(StringRef Str,
3085                                                       uint8_t &Val) {
3086   if (Str == "null") {
3087     Val = 9;
3088     return MatchOperand_Success;
3089   }
3090 
3091   if (Str.startswith("mrt")) {
3092     Str = Str.drop_front(3);
3093     if (Str == "z") { // == mrtz
3094       Val = 8;
3095       return MatchOperand_Success;
3096     }
3097 
3098     if (Str.getAsInteger(10, Val))
3099       return MatchOperand_ParseFail;
3100 
3101     if (Val > 7)
3102       errorExpTgt();
3103 
3104     return MatchOperand_Success;
3105   }
3106 
3107   if (Str.startswith("pos")) {
3108     Str = Str.drop_front(3);
3109     if (Str.getAsInteger(10, Val))
3110       return MatchOperand_ParseFail;
3111 
3112     if (Val > 3)
3113       errorExpTgt();
3114 
3115     Val += 12;
3116     return MatchOperand_Success;
3117   }
3118 
3119   if (Str.startswith("param")) {
3120     Str = Str.drop_front(5);
3121     if (Str.getAsInteger(10, Val))
3122       return MatchOperand_ParseFail;
3123 
3124     if (Val >= 32)
3125       errorExpTgt();
3126 
3127     Val += 32;
3128     return MatchOperand_Success;
3129   }
3130 
3131   if (Str.startswith("invalid_target_")) {
3132     Str = Str.drop_front(15);
3133     if (Str.getAsInteger(10, Val))
3134       return MatchOperand_ParseFail;
3135 
3136     errorExpTgt();
3137     return MatchOperand_Success;
3138   }
3139 
3140   return MatchOperand_NoMatch;
3141 }
3142 
3143 OperandMatchResultTy AMDGPUAsmParser::parseExpTgt(OperandVector &Operands) {
3144   uint8_t Val;
3145   StringRef Str = Parser.getTok().getString();
3146 
3147   auto Res = parseExpTgtImpl(Str, Val);
3148   if (Res != MatchOperand_Success)
3149     return Res;
3150 
3151   SMLoc S = Parser.getTok().getLoc();
3152   Parser.Lex();
3153 
3154   Operands.push_back(AMDGPUOperand::CreateImm(this, Val, S,
3155                                               AMDGPUOperand::ImmTyExpTgt));
3156   return MatchOperand_Success;
3157 }
3158 
3159 OperandMatchResultTy
3160 AMDGPUAsmParser::parseSendMsgOp(OperandVector &Operands) {
3161   using namespace llvm::AMDGPU::SendMsg;
3162 
3163   int64_t Imm16Val = 0;
3164   SMLoc S = Parser.getTok().getLoc();
3165 
3166   switch(getLexer().getKind()) {
3167   default:
3168     return MatchOperand_NoMatch;
3169   case AsmToken::Integer:
3170     // The operand can be an integer value.
3171     if (getParser().parseAbsoluteExpression(Imm16Val))
3172       return MatchOperand_NoMatch;
3173     if (Imm16Val < 0 || !isUInt<16>(Imm16Val)) {
3174       Error(S, "invalid immediate: only 16-bit values are legal");
3175       // Do not return error code, but create an imm operand anyway and proceed
3176       // to the next operand, if any. That avoids unneccessary error messages.
3177     }
3178     break;
3179   case AsmToken::Identifier: {
3180       OperandInfoTy Msg(ID_UNKNOWN_);
3181       OperandInfoTy Operation(OP_UNKNOWN_);
3182       int64_t StreamId = STREAM_ID_DEFAULT_;
3183       if (parseSendMsgConstruct(Msg, Operation, StreamId))
3184         return MatchOperand_ParseFail;
3185       do {
3186         // Validate and encode message ID.
3187         if (! ((ID_INTERRUPT <= Msg.Id && Msg.Id <= ID_GS_DONE)
3188                 || Msg.Id == ID_SYSMSG)) {
3189           if (Msg.IsSymbolic)
3190             Error(S, "invalid/unsupported symbolic name of message");
3191           else
3192             Error(S, "invalid/unsupported code of message");
3193           break;
3194         }
3195         Imm16Val = (Msg.Id << ID_SHIFT_);
3196         // Validate and encode operation ID.
3197         if (Msg.Id == ID_GS || Msg.Id == ID_GS_DONE) {
3198           if (! (OP_GS_FIRST_ <= Operation.Id && Operation.Id < OP_GS_LAST_)) {
3199             if (Operation.IsSymbolic)
3200               Error(S, "invalid symbolic name of GS_OP");
3201             else
3202               Error(S, "invalid code of GS_OP: only 2-bit values are legal");
3203             break;
3204           }
3205           if (Operation.Id == OP_GS_NOP
3206               && Msg.Id != ID_GS_DONE) {
3207             Error(S, "invalid GS_OP: NOP is for GS_DONE only");
3208             break;
3209           }
3210           Imm16Val |= (Operation.Id << OP_SHIFT_);
3211         }
3212         if (Msg.Id == ID_SYSMSG) {
3213           if (! (OP_SYS_FIRST_ <= Operation.Id && Operation.Id < OP_SYS_LAST_)) {
3214             if (Operation.IsSymbolic)
3215               Error(S, "invalid/unsupported symbolic name of SYSMSG_OP");
3216             else
3217               Error(S, "invalid/unsupported code of SYSMSG_OP");
3218             break;
3219           }
3220           Imm16Val |= (Operation.Id << OP_SHIFT_);
3221         }
3222         // Validate and encode stream ID.
3223         if ((Msg.Id == ID_GS || Msg.Id == ID_GS_DONE) && Operation.Id != OP_GS_NOP) {
3224           if (! (STREAM_ID_FIRST_ <= StreamId && StreamId < STREAM_ID_LAST_)) {
3225             Error(S, "invalid stream id: only 2-bit values are legal");
3226             break;
3227           }
3228           Imm16Val |= (StreamId << STREAM_ID_SHIFT_);
3229         }
3230       } while (false);
3231     }
3232     break;
3233   }
3234   Operands.push_back(AMDGPUOperand::CreateImm(this, Imm16Val, S, AMDGPUOperand::ImmTySendMsg));
3235   return MatchOperand_Success;
3236 }
3237 
3238 bool AMDGPUOperand::isSendMsg() const {
3239   return isImmTy(ImmTySendMsg);
3240 }
3241 
3242 //===----------------------------------------------------------------------===//
3243 // sopp branch targets
3244 //===----------------------------------------------------------------------===//
3245 
3246 OperandMatchResultTy
3247 AMDGPUAsmParser::parseSOppBrTarget(OperandVector &Operands) {
3248   SMLoc S = Parser.getTok().getLoc();
3249 
3250   switch (getLexer().getKind()) {
3251     default: return MatchOperand_ParseFail;
3252     case AsmToken::Integer: {
3253       int64_t Imm;
3254       if (getParser().parseAbsoluteExpression(Imm))
3255         return MatchOperand_ParseFail;
3256       Operands.push_back(AMDGPUOperand::CreateImm(this, Imm, S));
3257       return MatchOperand_Success;
3258     }
3259 
3260     case AsmToken::Identifier:
3261       Operands.push_back(AMDGPUOperand::CreateExpr(this,
3262           MCSymbolRefExpr::create(getContext().getOrCreateSymbol(
3263                                   Parser.getTok().getString()), getContext()), S));
3264       Parser.Lex();
3265       return MatchOperand_Success;
3266   }
3267 }
3268 
3269 //===----------------------------------------------------------------------===//
3270 // mubuf
3271 //===----------------------------------------------------------------------===//
3272 
3273 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultGLC() const {
3274   return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTyGLC);
3275 }
3276 
3277 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultSLC() const {
3278   return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTySLC);
3279 }
3280 
3281 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultTFE() const {
3282   return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTyTFE);
3283 }
3284 
3285 void AMDGPUAsmParser::cvtMubufImpl(MCInst &Inst,
3286                                const OperandVector &Operands,
3287                                bool IsAtomic, bool IsAtomicReturn) {
3288   OptionalImmIndexMap OptionalIdx;
3289   assert(IsAtomicReturn ? IsAtomic : true);
3290 
3291   for (unsigned i = 1, e = Operands.size(); i != e; ++i) {
3292     AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[i]);
3293 
3294     // Add the register arguments
3295     if (Op.isReg()) {
3296       Op.addRegOperands(Inst, 1);
3297       continue;
3298     }
3299 
3300     // Handle the case where soffset is an immediate
3301     if (Op.isImm() && Op.getImmTy() == AMDGPUOperand::ImmTyNone) {
3302       Op.addImmOperands(Inst, 1);
3303       continue;
3304     }
3305 
3306     // Handle tokens like 'offen' which are sometimes hard-coded into the
3307     // asm string.  There are no MCInst operands for these.
3308     if (Op.isToken()) {
3309       continue;
3310     }
3311     assert(Op.isImm());
3312 
3313     // Handle optional arguments
3314     OptionalIdx[Op.getImmTy()] = i;
3315   }
3316 
3317   // Copy $vdata_in operand and insert as $vdata for MUBUF_Atomic RTN insns.
3318   if (IsAtomicReturn) {
3319     MCInst::iterator I = Inst.begin(); // $vdata_in is always at the beginning.
3320     Inst.insert(I, *I);
3321   }
3322 
3323   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyOffset);
3324   if (!IsAtomic) { // glc is hard-coded.
3325     addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyGLC);
3326   }
3327   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySLC);
3328   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyTFE);
3329 }
3330 
3331 //===----------------------------------------------------------------------===//
3332 // mimg
3333 //===----------------------------------------------------------------------===//
3334 
3335 void AMDGPUAsmParser::cvtMIMG(MCInst &Inst, const OperandVector &Operands) {
3336   unsigned I = 1;
3337   const MCInstrDesc &Desc = MII.get(Inst.getOpcode());
3338   for (unsigned J = 0; J < Desc.getNumDefs(); ++J) {
3339     ((AMDGPUOperand &)*Operands[I++]).addRegOperands(Inst, 1);
3340   }
3341 
3342   OptionalImmIndexMap OptionalIdx;
3343 
3344   for (unsigned E = Operands.size(); I != E; ++I) {
3345     AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[I]);
3346 
3347     // Add the register arguments
3348     if (Op.isRegOrImm()) {
3349       Op.addRegOrImmOperands(Inst, 1);
3350       continue;
3351     } else if (Op.isImmModifier()) {
3352       OptionalIdx[Op.getImmTy()] = I;
3353     } else {
3354       llvm_unreachable("unexpected operand type");
3355     }
3356   }
3357 
3358   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyDMask);
3359   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyUNorm);
3360   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyGLC);
3361   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyDA);
3362   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyR128);
3363   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyTFE);
3364   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyLWE);
3365   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySLC);
3366 }
3367 
3368 void AMDGPUAsmParser::cvtMIMGAtomic(MCInst &Inst, const OperandVector &Operands) {
3369   unsigned I = 1;
3370   const MCInstrDesc &Desc = MII.get(Inst.getOpcode());
3371   for (unsigned J = 0; J < Desc.getNumDefs(); ++J) {
3372     ((AMDGPUOperand &)*Operands[I++]).addRegOperands(Inst, 1);
3373   }
3374 
3375   // Add src, same as dst
3376   ((AMDGPUOperand &)*Operands[I]).addRegOperands(Inst, 1);
3377 
3378   OptionalImmIndexMap OptionalIdx;
3379 
3380   for (unsigned E = Operands.size(); I != E; ++I) {
3381     AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[I]);
3382 
3383     // Add the register arguments
3384     if (Op.isRegOrImm()) {
3385       Op.addRegOrImmOperands(Inst, 1);
3386       continue;
3387     } else if (Op.isImmModifier()) {
3388       OptionalIdx[Op.getImmTy()] = I;
3389     } else {
3390       llvm_unreachable("unexpected operand type");
3391     }
3392   }
3393 
3394   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyDMask);
3395   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyUNorm);
3396   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyGLC);
3397   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyDA);
3398   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyR128);
3399   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyTFE);
3400   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyLWE);
3401   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySLC);
3402 }
3403 
3404 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultDMask() const {
3405   return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTyDMask);
3406 }
3407 
3408 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultUNorm() const {
3409   return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTyUNorm);
3410 }
3411 
3412 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultDA() const {
3413   return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTyDA);
3414 }
3415 
3416 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultR128() const {
3417   return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTyR128);
3418 }
3419 
3420 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultLWE() const {
3421   return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTyLWE);
3422 }
3423 
3424 //===----------------------------------------------------------------------===//
3425 // smrd
3426 //===----------------------------------------------------------------------===//
3427 
3428 bool AMDGPUOperand::isSMRDOffset8() const {
3429   return isImm() && isUInt<8>(getImm());
3430 }
3431 
3432 bool AMDGPUOperand::isSMRDOffset20() const {
3433   return isImm() && isUInt<20>(getImm());
3434 }
3435 
3436 bool AMDGPUOperand::isSMRDLiteralOffset() const {
3437   // 32-bit literals are only supported on CI and we only want to use them
3438   // when the offset is > 8-bits.
3439   return isImm() && !isUInt<8>(getImm()) && isUInt<32>(getImm());
3440 }
3441 
3442 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultSMRDOffset8() const {
3443   return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTyOffset);
3444 }
3445 
3446 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultSMRDOffset20() const {
3447   return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTyOffset);
3448 }
3449 
3450 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultSMRDLiteralOffset() const {
3451   return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTyOffset);
3452 }
3453 
3454 //===----------------------------------------------------------------------===//
3455 // vop3
3456 //===----------------------------------------------------------------------===//
3457 
3458 static bool ConvertOmodMul(int64_t &Mul) {
3459   if (Mul != 1 && Mul != 2 && Mul != 4)
3460     return false;
3461 
3462   Mul >>= 1;
3463   return true;
3464 }
3465 
3466 static bool ConvertOmodDiv(int64_t &Div) {
3467   if (Div == 1) {
3468     Div = 0;
3469     return true;
3470   }
3471 
3472   if (Div == 2) {
3473     Div = 3;
3474     return true;
3475   }
3476 
3477   return false;
3478 }
3479 
3480 static bool ConvertBoundCtrl(int64_t &BoundCtrl) {
3481   if (BoundCtrl == 0) {
3482     BoundCtrl = 1;
3483     return true;
3484   }
3485 
3486   if (BoundCtrl == -1) {
3487     BoundCtrl = 0;
3488     return true;
3489   }
3490 
3491   return false;
3492 }
3493 
3494 // Note: the order in this table matches the order of operands in AsmString.
3495 static const OptionalOperand AMDGPUOptionalOperandTable[] = {
3496   {"offen",   AMDGPUOperand::ImmTyOffen, true, nullptr},
3497   {"idxen",   AMDGPUOperand::ImmTyIdxen, true, nullptr},
3498   {"addr64",  AMDGPUOperand::ImmTyAddr64, true, nullptr},
3499   {"offset0", AMDGPUOperand::ImmTyOffset0, false, nullptr},
3500   {"offset1", AMDGPUOperand::ImmTyOffset1, false, nullptr},
3501   {"gds",     AMDGPUOperand::ImmTyGDS, true, nullptr},
3502   {"offset",  AMDGPUOperand::ImmTyOffset, false, nullptr},
3503   {"glc",     AMDGPUOperand::ImmTyGLC, true, nullptr},
3504   {"slc",     AMDGPUOperand::ImmTySLC, true, nullptr},
3505   {"tfe",     AMDGPUOperand::ImmTyTFE, true, nullptr},
3506   {"clamp",   AMDGPUOperand::ImmTyClampSI, true, nullptr},
3507   {"omod",    AMDGPUOperand::ImmTyOModSI, false, ConvertOmodMul},
3508   {"unorm",   AMDGPUOperand::ImmTyUNorm, true, nullptr},
3509   {"da",      AMDGPUOperand::ImmTyDA,    true, nullptr},
3510   {"r128",    AMDGPUOperand::ImmTyR128,  true, nullptr},
3511   {"lwe",     AMDGPUOperand::ImmTyLWE,   true, nullptr},
3512   {"dmask",   AMDGPUOperand::ImmTyDMask, false, nullptr},
3513   {"row_mask",   AMDGPUOperand::ImmTyDppRowMask, false, nullptr},
3514   {"bank_mask",  AMDGPUOperand::ImmTyDppBankMask, false, nullptr},
3515   {"bound_ctrl", AMDGPUOperand::ImmTyDppBoundCtrl, false, ConvertBoundCtrl},
3516   {"dst_sel",    AMDGPUOperand::ImmTySdwaDstSel, false, nullptr},
3517   {"src0_sel",   AMDGPUOperand::ImmTySdwaSrc0Sel, false, nullptr},
3518   {"src1_sel",   AMDGPUOperand::ImmTySdwaSrc1Sel, false, nullptr},
3519   {"dst_unused", AMDGPUOperand::ImmTySdwaDstUnused, false, nullptr},
3520   {"vm", AMDGPUOperand::ImmTyExpVM, true, nullptr},
3521   {"op_sel", AMDGPUOperand::ImmTyOpSel, false, nullptr},
3522   {"op_sel_hi", AMDGPUOperand::ImmTyOpSelHi, false, nullptr},
3523   {"neg_lo", AMDGPUOperand::ImmTyNegLo, false, nullptr},
3524   {"neg_hi", AMDGPUOperand::ImmTyNegHi, false, nullptr}
3525 };
3526 
3527 OperandMatchResultTy AMDGPUAsmParser::parseOptionalOperand(OperandVector &Operands) {
3528   OperandMatchResultTy res;
3529   for (const OptionalOperand &Op : AMDGPUOptionalOperandTable) {
3530     // try to parse any optional operand here
3531     if (Op.IsBit) {
3532       res = parseNamedBit(Op.Name, Operands, Op.Type);
3533     } else if (Op.Type == AMDGPUOperand::ImmTyOModSI) {
3534       res = parseOModOperand(Operands);
3535     } else if (Op.Type == AMDGPUOperand::ImmTySdwaDstSel ||
3536                Op.Type == AMDGPUOperand::ImmTySdwaSrc0Sel ||
3537                Op.Type == AMDGPUOperand::ImmTySdwaSrc1Sel) {
3538       res = parseSDWASel(Operands, Op.Name, Op.Type);
3539     } else if (Op.Type == AMDGPUOperand::ImmTySdwaDstUnused) {
3540       res = parseSDWADstUnused(Operands);
3541     } else if (Op.Type == AMDGPUOperand::ImmTyOpSel ||
3542                Op.Type == AMDGPUOperand::ImmTyOpSelHi ||
3543                Op.Type == AMDGPUOperand::ImmTyNegLo ||
3544                Op.Type == AMDGPUOperand::ImmTyNegHi) {
3545       res = parseOperandArrayWithPrefix(Op.Name, Operands, Op.Type,
3546                                         Op.ConvertResult);
3547     } else {
3548       res = parseIntWithPrefix(Op.Name, Operands, Op.Type, Op.ConvertResult);
3549     }
3550     if (res != MatchOperand_NoMatch) {
3551       return res;
3552     }
3553   }
3554   return MatchOperand_NoMatch;
3555 }
3556 
3557 OperandMatchResultTy AMDGPUAsmParser::parseOModOperand(OperandVector &Operands) {
3558   StringRef Name = Parser.getTok().getString();
3559   if (Name == "mul") {
3560     return parseIntWithPrefix("mul", Operands,
3561                               AMDGPUOperand::ImmTyOModSI, ConvertOmodMul);
3562   }
3563 
3564   if (Name == "div") {
3565     return parseIntWithPrefix("div", Operands,
3566                               AMDGPUOperand::ImmTyOModSI, ConvertOmodDiv);
3567   }
3568 
3569   return MatchOperand_NoMatch;
3570 }
3571 
3572 void AMDGPUAsmParser::cvtId(MCInst &Inst, const OperandVector &Operands) {
3573   unsigned I = 1;
3574   const MCInstrDesc &Desc = MII.get(Inst.getOpcode());
3575   for (unsigned J = 0; J < Desc.getNumDefs(); ++J) {
3576     ((AMDGPUOperand &)*Operands[I++]).addRegOperands(Inst, 1);
3577   }
3578   for (unsigned E = Operands.size(); I != E; ++I)
3579     ((AMDGPUOperand &)*Operands[I]).addRegOrImmOperands(Inst, 1);
3580 }
3581 
3582 void AMDGPUAsmParser::cvtVOP3_2_mod(MCInst &Inst, const OperandVector &Operands) {
3583   uint64_t TSFlags = MII.get(Inst.getOpcode()).TSFlags;
3584   if (TSFlags & SIInstrFlags::VOP3) {
3585     cvtVOP3(Inst, Operands);
3586   } else {
3587     cvtId(Inst, Operands);
3588   }
3589 }
3590 
3591 static bool isRegOrImmWithInputMods(const MCInstrDesc &Desc, unsigned OpNum) {
3592       // 1. This operand is input modifiers
3593   return Desc.OpInfo[OpNum].OperandType == AMDGPU::OPERAND_INPUT_MODS
3594       // 2. This is not last operand
3595       && Desc.NumOperands > (OpNum + 1)
3596       // 3. Next operand is register class
3597       && Desc.OpInfo[OpNum + 1].RegClass != -1
3598       // 4. Next register is not tied to any other operand
3599       && Desc.getOperandConstraint(OpNum + 1, MCOI::OperandConstraint::TIED_TO) == -1;
3600 }
3601 
3602 void AMDGPUAsmParser::cvtVOP3Impl(MCInst &Inst, const OperandVector &Operands,
3603                                   OptionalImmIndexMap &OptionalIdx) {
3604   unsigned I = 1;
3605   const MCInstrDesc &Desc = MII.get(Inst.getOpcode());
3606   for (unsigned J = 0; J < Desc.getNumDefs(); ++J) {
3607     ((AMDGPUOperand &)*Operands[I++]).addRegOperands(Inst, 1);
3608   }
3609 
3610   for (unsigned E = Operands.size(); I != E; ++I) {
3611     AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[I]);
3612     if (isRegOrImmWithInputMods(Desc, Inst.getNumOperands())) {
3613       Op.addRegOrImmWithFPInputModsOperands(Inst, 2);
3614     } else if (Op.isImm()) {
3615       OptionalIdx[Op.getImmTy()] = I;
3616     } else {
3617       llvm_unreachable("unhandled operand type");
3618     }
3619   }
3620 }
3621 
3622 void AMDGPUAsmParser::cvtVOP3(MCInst &Inst, const OperandVector &Operands) {
3623   OptionalImmIndexMap OptionalIdx;
3624 
3625   cvtVOP3Impl(Inst, Operands, OptionalIdx);
3626 
3627   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyClampSI);
3628   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyOModSI);
3629 
3630   // special case v_mac_{f16, f32}:
3631   // it has src2 register operand that is tied to dst operand
3632   // we don't allow modifiers for this operand in assembler so src2_modifiers
3633   // should be 0
3634   if (Inst.getOpcode() == AMDGPU::V_MAC_F32_e64_si ||
3635       Inst.getOpcode() == AMDGPU::V_MAC_F32_e64_vi ||
3636       Inst.getOpcode() == AMDGPU::V_MAC_F16_e64_vi) {
3637     auto it = Inst.begin();
3638     std::advance(
3639       it,
3640       AMDGPU::getNamedOperandIdx(Inst.getOpcode() == AMDGPU::V_MAC_F16_e64_vi ?
3641                                      AMDGPU::V_MAC_F16_e64 :
3642                                      AMDGPU::V_MAC_F32_e64,
3643                                  AMDGPU::OpName::src2_modifiers));
3644     it = Inst.insert(it, MCOperand::createImm(0)); // no modifiers for src2
3645     ++it;
3646     Inst.insert(it, Inst.getOperand(0)); // src2 = dst
3647   }
3648 }
3649 
3650 void AMDGPUAsmParser::cvtVOP3P(MCInst &Inst, const OperandVector &Operands) {
3651   OptionalImmIndexMap OptIdx;
3652 
3653   cvtVOP3Impl(Inst, Operands, OptIdx);
3654 
3655   // FIXME: This is messy. Parse the modifiers as if it was a normal VOP3
3656   // instruction, and then figure out where to actually put the modifiers
3657   int Opc = Inst.getOpcode();
3658 
3659   if (AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::clamp) != -1) {
3660     addOptionalImmOperand(Inst, Operands, OptIdx, AMDGPUOperand::ImmTyClampSI);
3661   }
3662 
3663   addOptionalImmOperand(Inst, Operands, OptIdx, AMDGPUOperand::ImmTyOpSel);
3664   addOptionalImmOperand(Inst, Operands, OptIdx, AMDGPUOperand::ImmTyOpSelHi, -1);
3665 
3666   int NegLoIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::neg_lo);
3667   if (NegLoIdx != -1) {
3668     addOptionalImmOperand(Inst, Operands, OptIdx, AMDGPUOperand::ImmTyNegLo);
3669     addOptionalImmOperand(Inst, Operands, OptIdx, AMDGPUOperand::ImmTyNegHi);
3670   }
3671 
3672   const int Ops[] = { AMDGPU::OpName::src0,
3673                       AMDGPU::OpName::src1,
3674                       AMDGPU::OpName::src2 };
3675   const int ModOps[] = { AMDGPU::OpName::src0_modifiers,
3676                          AMDGPU::OpName::src1_modifiers,
3677                          AMDGPU::OpName::src2_modifiers };
3678 
3679   int OpSelIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::op_sel);
3680   int OpSelHiIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::op_sel_hi);
3681 
3682   unsigned OpSel = Inst.getOperand(OpSelIdx).getImm();
3683   unsigned OpSelHi = Inst.getOperand(OpSelHiIdx).getImm();
3684   unsigned NegLo = 0;
3685   unsigned NegHi = 0;
3686 
3687   if (NegLoIdx != -1) {
3688     int NegHiIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::neg_hi);
3689     NegLo = Inst.getOperand(NegLoIdx).getImm();
3690     NegHi = Inst.getOperand(NegHiIdx).getImm();
3691   }
3692 
3693   for (int J = 0; J < 3; ++J) {
3694     int OpIdx = AMDGPU::getNamedOperandIdx(Opc, Ops[J]);
3695     if (OpIdx == -1)
3696       break;
3697 
3698     uint32_t ModVal = 0;
3699 
3700     if ((OpSel & (1 << J)) != 0)
3701       ModVal |= SISrcMods::OP_SEL_0;
3702 
3703     if ((OpSelHi & (1 << J)) != 0)
3704       ModVal |= SISrcMods::OP_SEL_1;
3705 
3706     if ((NegLo & (1 << J)) != 0)
3707       ModVal |= SISrcMods::NEG;
3708 
3709     if ((NegHi & (1 << J)) != 0)
3710       ModVal |= SISrcMods::NEG_HI;
3711 
3712     int ModIdx = AMDGPU::getNamedOperandIdx(Opc, ModOps[J]);
3713 
3714     Inst.getOperand(ModIdx).setImm(ModVal);
3715   }
3716 }
3717 
3718 //===----------------------------------------------------------------------===//
3719 // dpp
3720 //===----------------------------------------------------------------------===//
3721 
3722 bool AMDGPUOperand::isDPPCtrl() const {
3723   bool result = isImm() && getImmTy() == ImmTyDppCtrl && isUInt<9>(getImm());
3724   if (result) {
3725     int64_t Imm = getImm();
3726     return ((Imm >= 0x000) && (Imm <= 0x0ff)) ||
3727            ((Imm >= 0x101) && (Imm <= 0x10f)) ||
3728            ((Imm >= 0x111) && (Imm <= 0x11f)) ||
3729            ((Imm >= 0x121) && (Imm <= 0x12f)) ||
3730            (Imm == 0x130) ||
3731            (Imm == 0x134) ||
3732            (Imm == 0x138) ||
3733            (Imm == 0x13c) ||
3734            (Imm == 0x140) ||
3735            (Imm == 0x141) ||
3736            (Imm == 0x142) ||
3737            (Imm == 0x143);
3738   }
3739   return false;
3740 }
3741 
3742 bool AMDGPUOperand::isGPRIdxMode() const {
3743   return isImm() && isUInt<4>(getImm());
3744 }
3745 
3746 OperandMatchResultTy
3747 AMDGPUAsmParser::parseDPPCtrl(OperandVector &Operands) {
3748   SMLoc S = Parser.getTok().getLoc();
3749   StringRef Prefix;
3750   int64_t Int;
3751 
3752   if (getLexer().getKind() == AsmToken::Identifier) {
3753     Prefix = Parser.getTok().getString();
3754   } else {
3755     return MatchOperand_NoMatch;
3756   }
3757 
3758   if (Prefix == "row_mirror") {
3759     Int = 0x140;
3760     Parser.Lex();
3761   } else if (Prefix == "row_half_mirror") {
3762     Int = 0x141;
3763     Parser.Lex();
3764   } else {
3765     // Check to prevent parseDPPCtrlOps from eating invalid tokens
3766     if (Prefix != "quad_perm"
3767         && Prefix != "row_shl"
3768         && Prefix != "row_shr"
3769         && Prefix != "row_ror"
3770         && Prefix != "wave_shl"
3771         && Prefix != "wave_rol"
3772         && Prefix != "wave_shr"
3773         && Prefix != "wave_ror"
3774         && Prefix != "row_bcast") {
3775       return MatchOperand_NoMatch;
3776     }
3777 
3778     Parser.Lex();
3779     if (getLexer().isNot(AsmToken::Colon))
3780       return MatchOperand_ParseFail;
3781 
3782     if (Prefix == "quad_perm") {
3783       // quad_perm:[%d,%d,%d,%d]
3784       Parser.Lex();
3785       if (getLexer().isNot(AsmToken::LBrac))
3786         return MatchOperand_ParseFail;
3787       Parser.Lex();
3788 
3789       if (getParser().parseAbsoluteExpression(Int) || !(0 <= Int && Int <=3))
3790         return MatchOperand_ParseFail;
3791 
3792       for (int i = 0; i < 3; ++i) {
3793         if (getLexer().isNot(AsmToken::Comma))
3794           return MatchOperand_ParseFail;
3795         Parser.Lex();
3796 
3797         int64_t Temp;
3798         if (getParser().parseAbsoluteExpression(Temp) || !(0 <= Temp && Temp <=3))
3799           return MatchOperand_ParseFail;
3800         const int shift = i*2 + 2;
3801         Int += (Temp << shift);
3802       }
3803 
3804       if (getLexer().isNot(AsmToken::RBrac))
3805         return MatchOperand_ParseFail;
3806       Parser.Lex();
3807 
3808     } else {
3809       // sel:%d
3810       Parser.Lex();
3811       if (getParser().parseAbsoluteExpression(Int))
3812         return MatchOperand_ParseFail;
3813 
3814       if (Prefix == "row_shl" && 1 <= Int && Int <= 15) {
3815         Int |= 0x100;
3816       } else if (Prefix == "row_shr" && 1 <= Int && Int <= 15) {
3817         Int |= 0x110;
3818       } else if (Prefix == "row_ror" && 1 <= Int && Int <= 15) {
3819         Int |= 0x120;
3820       } else if (Prefix == "wave_shl" && 1 == Int) {
3821         Int = 0x130;
3822       } else if (Prefix == "wave_rol" && 1 == Int) {
3823         Int = 0x134;
3824       } else if (Prefix == "wave_shr" && 1 == Int) {
3825         Int = 0x138;
3826       } else if (Prefix == "wave_ror" && 1 == Int) {
3827         Int = 0x13C;
3828       } else if (Prefix == "row_bcast") {
3829         if (Int == 15) {
3830           Int = 0x142;
3831         } else if (Int == 31) {
3832           Int = 0x143;
3833         } else {
3834           return MatchOperand_ParseFail;
3835         }
3836       } else {
3837         return MatchOperand_ParseFail;
3838       }
3839     }
3840   }
3841 
3842   Operands.push_back(AMDGPUOperand::CreateImm(this, Int, S, AMDGPUOperand::ImmTyDppCtrl));
3843   return MatchOperand_Success;
3844 }
3845 
3846 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultRowMask() const {
3847   return AMDGPUOperand::CreateImm(this, 0xf, SMLoc(), AMDGPUOperand::ImmTyDppRowMask);
3848 }
3849 
3850 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultBankMask() const {
3851   return AMDGPUOperand::CreateImm(this, 0xf, SMLoc(), AMDGPUOperand::ImmTyDppBankMask);
3852 }
3853 
3854 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultBoundCtrl() const {
3855   return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTyDppBoundCtrl);
3856 }
3857 
3858 void AMDGPUAsmParser::cvtDPP(MCInst &Inst, const OperandVector &Operands) {
3859   OptionalImmIndexMap OptionalIdx;
3860 
3861   unsigned I = 1;
3862   const MCInstrDesc &Desc = MII.get(Inst.getOpcode());
3863   for (unsigned J = 0; J < Desc.getNumDefs(); ++J) {
3864     ((AMDGPUOperand &)*Operands[I++]).addRegOperands(Inst, 1);
3865   }
3866 
3867   for (unsigned E = Operands.size(); I != E; ++I) {
3868     AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[I]);
3869     // Add the register arguments
3870     if (Op.isReg() && Op.Reg.RegNo == AMDGPU::VCC) {
3871       // VOP2b (v_add_u32, v_sub_u32 ...) dpp use "vcc" token.
3872       // Skip it.
3873       continue;
3874     } if (isRegOrImmWithInputMods(Desc, Inst.getNumOperands())) {
3875       Op.addRegWithFPInputModsOperands(Inst, 2);
3876     } else if (Op.isDPPCtrl()) {
3877       Op.addImmOperands(Inst, 1);
3878     } else if (Op.isImm()) {
3879       // Handle optional arguments
3880       OptionalIdx[Op.getImmTy()] = I;
3881     } else {
3882       llvm_unreachable("Invalid operand type");
3883     }
3884   }
3885 
3886   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyDppRowMask, 0xf);
3887   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyDppBankMask, 0xf);
3888   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyDppBoundCtrl);
3889 
3890   // special case v_mac_{f16, f32}:
3891   // it has src2 register operand that is tied to dst operand
3892   if (Inst.getOpcode() == AMDGPU::V_MAC_F32_dpp ||
3893       Inst.getOpcode() == AMDGPU::V_MAC_F16_dpp) {
3894     auto it = Inst.begin();
3895     std::advance(
3896         it, AMDGPU::getNamedOperandIdx(Inst.getOpcode(), AMDGPU::OpName::src2));
3897     Inst.insert(it, Inst.getOperand(0)); // src2 = dst
3898   }
3899 }
3900 
3901 //===----------------------------------------------------------------------===//
3902 // sdwa
3903 //===----------------------------------------------------------------------===//
3904 
3905 OperandMatchResultTy
3906 AMDGPUAsmParser::parseSDWASel(OperandVector &Operands, StringRef Prefix,
3907                               AMDGPUOperand::ImmTy Type) {
3908   using namespace llvm::AMDGPU::SDWA;
3909 
3910   SMLoc S = Parser.getTok().getLoc();
3911   StringRef Value;
3912   OperandMatchResultTy res;
3913 
3914   res = parseStringWithPrefix(Prefix, Value);
3915   if (res != MatchOperand_Success) {
3916     return res;
3917   }
3918 
3919   int64_t Int;
3920   Int = StringSwitch<int64_t>(Value)
3921         .Case("BYTE_0", SdwaSel::BYTE_0)
3922         .Case("BYTE_1", SdwaSel::BYTE_1)
3923         .Case("BYTE_2", SdwaSel::BYTE_2)
3924         .Case("BYTE_3", SdwaSel::BYTE_3)
3925         .Case("WORD_0", SdwaSel::WORD_0)
3926         .Case("WORD_1", SdwaSel::WORD_1)
3927         .Case("DWORD", SdwaSel::DWORD)
3928         .Default(0xffffffff);
3929   Parser.Lex(); // eat last token
3930 
3931   if (Int == 0xffffffff) {
3932     return MatchOperand_ParseFail;
3933   }
3934 
3935   Operands.push_back(AMDGPUOperand::CreateImm(this, Int, S, Type));
3936   return MatchOperand_Success;
3937 }
3938 
3939 OperandMatchResultTy
3940 AMDGPUAsmParser::parseSDWADstUnused(OperandVector &Operands) {
3941   using namespace llvm::AMDGPU::SDWA;
3942 
3943   SMLoc S = Parser.getTok().getLoc();
3944   StringRef Value;
3945   OperandMatchResultTy res;
3946 
3947   res = parseStringWithPrefix("dst_unused", Value);
3948   if (res != MatchOperand_Success) {
3949     return res;
3950   }
3951 
3952   int64_t Int;
3953   Int = StringSwitch<int64_t>(Value)
3954         .Case("UNUSED_PAD", DstUnused::UNUSED_PAD)
3955         .Case("UNUSED_SEXT", DstUnused::UNUSED_SEXT)
3956         .Case("UNUSED_PRESERVE", DstUnused::UNUSED_PRESERVE)
3957         .Default(0xffffffff);
3958   Parser.Lex(); // eat last token
3959 
3960   if (Int == 0xffffffff) {
3961     return MatchOperand_ParseFail;
3962   }
3963 
3964   Operands.push_back(AMDGPUOperand::CreateImm(this, Int, S, AMDGPUOperand::ImmTySdwaDstUnused));
3965   return MatchOperand_Success;
3966 }
3967 
3968 void AMDGPUAsmParser::cvtSdwaVOP1(MCInst &Inst, const OperandVector &Operands) {
3969   cvtSDWA(Inst, Operands, SIInstrFlags::VOP1);
3970 }
3971 
3972 void AMDGPUAsmParser::cvtSdwaVOP2(MCInst &Inst, const OperandVector &Operands) {
3973   cvtSDWA(Inst, Operands, SIInstrFlags::VOP2);
3974 }
3975 
3976 void AMDGPUAsmParser::cvtSdwaVOPC(MCInst &Inst, const OperandVector &Operands) {
3977   cvtSDWA(Inst, Operands, SIInstrFlags::VOPC);
3978 }
3979 
3980 void AMDGPUAsmParser::cvtSDWA(MCInst &Inst, const OperandVector &Operands,
3981                               uint64_t BasicInstType) {
3982   using namespace llvm::AMDGPU::SDWA;
3983   OptionalImmIndexMap OptionalIdx;
3984 
3985   unsigned I = 1;
3986   const MCInstrDesc &Desc = MII.get(Inst.getOpcode());
3987   for (unsigned J = 0; J < Desc.getNumDefs(); ++J) {
3988     ((AMDGPUOperand &)*Operands[I++]).addRegOperands(Inst, 1);
3989   }
3990 
3991   for (unsigned E = Operands.size(); I != E; ++I) {
3992     AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[I]);
3993     // Add the register arguments
3994     if ((BasicInstType == SIInstrFlags::VOPC ||
3995          BasicInstType == SIInstrFlags::VOP2)&&
3996         Op.isReg() &&
3997         Op.Reg.RegNo == AMDGPU::VCC) {
3998       // VOPC and VOP2b (v_add_u32, v_sub_u32 ...) sdwa use "vcc" token as dst.
3999       // Skip it.
4000       continue;
4001     } else if (isRegOrImmWithInputMods(Desc, Inst.getNumOperands())) {
4002       Op.addRegWithInputModsOperands(Inst, 2);
4003     } else if (Op.isImm()) {
4004       // Handle optional arguments
4005       OptionalIdx[Op.getImmTy()] = I;
4006     } else {
4007       llvm_unreachable("Invalid operand type");
4008     }
4009   }
4010 
4011   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyClampSI, 0);
4012 
4013   if (Inst.getOpcode() != AMDGPU::V_NOP_sdwa_vi) {
4014     // V_NOP_sdwa_vi has no optional sdwa arguments
4015     switch (BasicInstType) {
4016     case SIInstrFlags::VOP1:
4017       addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySdwaDstSel, SdwaSel::DWORD);
4018       addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySdwaDstUnused, DstUnused::UNUSED_PRESERVE);
4019       addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySdwaSrc0Sel, SdwaSel::DWORD);
4020       break;
4021 
4022     case SIInstrFlags::VOP2:
4023       addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySdwaDstSel, SdwaSel::DWORD);
4024       addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySdwaDstUnused, DstUnused::UNUSED_PRESERVE);
4025       addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySdwaSrc0Sel, SdwaSel::DWORD);
4026       addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySdwaSrc1Sel, SdwaSel::DWORD);
4027       break;
4028 
4029     case SIInstrFlags::VOPC:
4030       addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySdwaSrc0Sel, SdwaSel::DWORD);
4031       addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySdwaSrc1Sel, SdwaSel::DWORD);
4032       break;
4033 
4034     default:
4035       llvm_unreachable("Invalid instruction type. Only VOP1, VOP2 and VOPC allowed");
4036     }
4037   }
4038 
4039   // special case v_mac_{f16, f32}:
4040   // it has src2 register operand that is tied to dst operand
4041   if (Inst.getOpcode() == AMDGPU::V_MAC_F32_sdwa_vi ||
4042       Inst.getOpcode() == AMDGPU::V_MAC_F16_sdwa_vi)  {
4043     auto it = Inst.begin();
4044     std::advance(
4045         it, AMDGPU::getNamedOperandIdx(Inst.getOpcode(), AMDGPU::OpName::src2));
4046     Inst.insert(it, Inst.getOperand(0)); // src2 = dst
4047   }
4048 
4049 }
4050 
4051 /// Force static initialization.
4052 extern "C" void LLVMInitializeAMDGPUAsmParser() {
4053   RegisterMCAsmParser<AMDGPUAsmParser> A(getTheAMDGPUTarget());
4054   RegisterMCAsmParser<AMDGPUAsmParser> B(getTheGCNTarget());
4055 }
4056 
4057 #define GET_REGISTER_MATCHER
4058 #define GET_MATCHER_IMPLEMENTATION
4059 #include "AMDGPUGenAsmMatcher.inc"
4060 
4061 // This fuction should be defined after auto-generated include so that we have
4062 // MatchClassKind enum defined
4063 unsigned AMDGPUAsmParser::validateTargetOperandClass(MCParsedAsmOperand &Op,
4064                                                      unsigned Kind) {
4065   // Tokens like "glc" would be parsed as immediate operands in ParseOperand().
4066   // But MatchInstructionImpl() expects to meet token and fails to validate
4067   // operand. This method checks if we are given immediate operand but expect to
4068   // get corresponding token.
4069   AMDGPUOperand &Operand = (AMDGPUOperand&)Op;
4070   switch (Kind) {
4071   case MCK_addr64:
4072     return Operand.isAddr64() ? Match_Success : Match_InvalidOperand;
4073   case MCK_gds:
4074     return Operand.isGDS() ? Match_Success : Match_InvalidOperand;
4075   case MCK_glc:
4076     return Operand.isGLC() ? Match_Success : Match_InvalidOperand;
4077   case MCK_idxen:
4078     return Operand.isIdxen() ? Match_Success : Match_InvalidOperand;
4079   case MCK_offen:
4080     return Operand.isOffen() ? Match_Success : Match_InvalidOperand;
4081   case MCK_SSrcB32:
4082     // When operands have expression values, they will return true for isToken,
4083     // because it is not possible to distinguish between a token and an
4084     // expression at parse time. MatchInstructionImpl() will always try to
4085     // match an operand as a token, when isToken returns true, and when the
4086     // name of the expression is not a valid token, the match will fail,
4087     // so we need to handle it here.
4088     return Operand.isSSrcB32() ? Match_Success : Match_InvalidOperand;
4089   case MCK_SSrcF32:
4090     return Operand.isSSrcF32() ? Match_Success : Match_InvalidOperand;
4091   case MCK_SoppBrTarget:
4092     return Operand.isSoppBrTarget() ? Match_Success : Match_InvalidOperand;
4093   case MCK_VReg32OrOff:
4094     return Operand.isVReg32OrOff() ? Match_Success : Match_InvalidOperand;
4095   case MCK_InterpSlot:
4096     return Operand.isInterpSlot() ? Match_Success : Match_InvalidOperand;
4097   case MCK_Attr:
4098     return Operand.isInterpAttr() ? Match_Success : Match_InvalidOperand;
4099   case MCK_AttrChan:
4100     return Operand.isAttrChan() ? Match_Success : Match_InvalidOperand;
4101   default:
4102     return Match_InvalidOperand;
4103   }
4104 }
4105