1 //===-- AMDGPUAsmParser.cpp - Parse SI asm to MCInst instructions ---------===//
2 //
3 //                     The LLVM Compiler Infrastructure
4 //
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
7 //
8 //===----------------------------------------------------------------------===//
9 
10 #include "AMDKernelCodeT.h"
11 #include "MCTargetDesc/AMDGPUMCTargetDesc.h"
12 #include "MCTargetDesc/AMDGPUTargetStreamer.h"
13 #include "SIDefines.h"
14 #include "Utils/AMDGPUBaseInfo.h"
15 #include "Utils/AMDKernelCodeTUtils.h"
16 #include "Utils/AMDGPUAsmUtils.h"
17 #include "llvm/ADT/APFloat.h"
18 #include "llvm/ADT/APInt.h"
19 #include "llvm/ADT/SmallBitVector.h"
20 #include "llvm/ADT/SmallString.h"
21 #include "llvm/ADT/STLExtras.h"
22 #include "llvm/ADT/StringRef.h"
23 #include "llvm/ADT/StringSwitch.h"
24 #include "llvm/ADT/Twine.h"
25 #include "llvm/CodeGen/MachineValueType.h"
26 #include "llvm/MC/MCContext.h"
27 #include "llvm/MC/MCExpr.h"
28 #include "llvm/MC/MCInst.h"
29 #include "llvm/MC/MCInstrDesc.h"
30 #include "llvm/MC/MCInstrInfo.h"
31 #include "llvm/MC/MCParser/MCAsmLexer.h"
32 #include "llvm/MC/MCParser/MCAsmParser.h"
33 #include "llvm/MC/MCParser/MCAsmParserExtension.h"
34 #include "llvm/MC/MCParser/MCParsedAsmOperand.h"
35 #include "llvm/MC/MCParser/MCTargetAsmParser.h"
36 #include "llvm/MC/MCRegisterInfo.h"
37 #include "llvm/MC/MCStreamer.h"
38 #include "llvm/MC/MCSubtargetInfo.h"
39 #include "llvm/MC/MCSymbol.h"
40 #include "llvm/Support/Casting.h"
41 #include "llvm/Support/Debug.h"
42 #include "llvm/Support/ELF.h"
43 #include "llvm/Support/ErrorHandling.h"
44 #include "llvm/Support/MathExtras.h"
45 #include "llvm/Support/raw_ostream.h"
46 #include "llvm/Support/SMLoc.h"
47 #include "llvm/Support/TargetRegistry.h"
48 #include <algorithm>
49 #include <cassert>
50 #include <cstdint>
51 #include <cstring>
52 #include <iterator>
53 #include <map>
54 #include <memory>
55 #include <string>
56 #include <vector>
57 
58 using namespace llvm;
59 using namespace llvm::AMDGPU;
60 
61 namespace {
62 
63 class AMDGPUAsmParser;
64 
65 enum RegisterKind { IS_UNKNOWN, IS_VGPR, IS_SGPR, IS_TTMP, IS_SPECIAL };
66 
67 //===----------------------------------------------------------------------===//
68 // Operand
69 //===----------------------------------------------------------------------===//
70 
71 class AMDGPUOperand : public MCParsedAsmOperand {
72   enum KindTy {
73     Token,
74     Immediate,
75     Register,
76     Expression
77   } Kind;
78 
79   SMLoc StartLoc, EndLoc;
80   const AMDGPUAsmParser *AsmParser;
81 
82 public:
83   AMDGPUOperand(enum KindTy Kind_, const AMDGPUAsmParser *AsmParser_)
84     : MCParsedAsmOperand(), Kind(Kind_), AsmParser(AsmParser_) {}
85 
86   typedef std::unique_ptr<AMDGPUOperand> Ptr;
87 
88   struct Modifiers {
89     bool Abs = false;
90     bool Neg = false;
91     bool Sext = false;
92 
93     bool hasFPModifiers() const { return Abs || Neg; }
94     bool hasIntModifiers() const { return Sext; }
95     bool hasModifiers() const { return hasFPModifiers() || hasIntModifiers(); }
96 
97     int64_t getFPModifiersOperand() const {
98       int64_t Operand = 0;
99       Operand |= Abs ? SISrcMods::ABS : 0;
100       Operand |= Neg ? SISrcMods::NEG : 0;
101       return Operand;
102     }
103 
104     int64_t getIntModifiersOperand() const {
105       int64_t Operand = 0;
106       Operand |= Sext ? SISrcMods::SEXT : 0;
107       return Operand;
108     }
109 
110     int64_t getModifiersOperand() const {
111       assert(!(hasFPModifiers() && hasIntModifiers())
112            && "fp and int modifiers should not be used simultaneously");
113       if (hasFPModifiers()) {
114         return getFPModifiersOperand();
115       } else if (hasIntModifiers()) {
116         return getIntModifiersOperand();
117       } else {
118         return 0;
119       }
120     }
121 
122     friend raw_ostream &operator <<(raw_ostream &OS, AMDGPUOperand::Modifiers Mods);
123   };
124 
125   enum ImmTy {
126     ImmTyNone,
127     ImmTyGDS,
128     ImmTyOffen,
129     ImmTyIdxen,
130     ImmTyAddr64,
131     ImmTyOffset,
132     ImmTyOffset0,
133     ImmTyOffset1,
134     ImmTyGLC,
135     ImmTySLC,
136     ImmTyTFE,
137     ImmTyClampSI,
138     ImmTyOModSI,
139     ImmTyDppCtrl,
140     ImmTyDppRowMask,
141     ImmTyDppBankMask,
142     ImmTyDppBoundCtrl,
143     ImmTySdwaDstSel,
144     ImmTySdwaSrc0Sel,
145     ImmTySdwaSrc1Sel,
146     ImmTySdwaDstUnused,
147     ImmTyDMask,
148     ImmTyUNorm,
149     ImmTyDA,
150     ImmTyR128,
151     ImmTyLWE,
152     ImmTyExpTgt,
153     ImmTyExpCompr,
154     ImmTyExpVM,
155     ImmTyHwreg,
156     ImmTyOff,
157     ImmTySendMsg,
158   };
159 
160   struct TokOp {
161     const char *Data;
162     unsigned Length;
163   };
164 
165   struct ImmOp {
166     int64_t Val;
167     ImmTy Type;
168     bool IsFPImm;
169     Modifiers Mods;
170   };
171 
172   struct RegOp {
173     unsigned RegNo;
174     bool IsForcedVOP3;
175     Modifiers Mods;
176   };
177 
178   union {
179     TokOp Tok;
180     ImmOp Imm;
181     RegOp Reg;
182     const MCExpr *Expr;
183   };
184 
185   bool isToken() const override {
186     if (Kind == Token)
187       return true;
188 
189     if (Kind != Expression || !Expr)
190       return false;
191 
192     // When parsing operands, we can't always tell if something was meant to be
193     // a token, like 'gds', or an expression that references a global variable.
194     // In this case, we assume the string is an expression, and if we need to
195     // interpret is a token, then we treat the symbol name as the token.
196     return isa<MCSymbolRefExpr>(Expr);
197   }
198 
199   bool isImm() const override {
200     return Kind == Immediate;
201   }
202 
203   bool isInlinableImm(MVT type) const;
204   bool isLiteralImm(MVT type) const;
205 
206   bool isRegKind() const {
207     return Kind == Register;
208   }
209 
210   bool isReg() const override {
211     return isRegKind() && !Reg.Mods.hasModifiers();
212   }
213 
214   bool isRegOrImmWithInputMods(MVT type) const {
215     return isRegKind() || isInlinableImm(type);
216   }
217 
218   bool isRegOrImmWithInt16InputMods() const {
219     return isRegOrImmWithInputMods(MVT::i16);
220   }
221 
222   bool isRegOrImmWithInt32InputMods() const {
223     return isRegOrImmWithInputMods(MVT::i32);
224   }
225 
226   bool isRegOrImmWithInt64InputMods() const {
227     return isRegOrImmWithInputMods(MVT::i64);
228   }
229 
230   bool isRegOrImmWithFP16InputMods() const {
231     return isRegOrImmWithInputMods(MVT::f16);
232   }
233 
234   bool isRegOrImmWithFP32InputMods() const {
235     return isRegOrImmWithInputMods(MVT::f32);
236   }
237 
238   bool isRegOrImmWithFP64InputMods() const {
239     return isRegOrImmWithInputMods(MVT::f64);
240   }
241 
242   bool isVReg32OrOff() const {
243     return isOff() || isRegClass(AMDGPU::VGPR_32RegClassID);
244   }
245 
246   bool isImmTy(ImmTy ImmT) const {
247     return isImm() && Imm.Type == ImmT;
248   }
249 
250   bool isImmModifier() const {
251     return isImm() && Imm.Type != ImmTyNone;
252   }
253 
254   bool isClampSI() const { return isImmTy(ImmTyClampSI); }
255   bool isOModSI() const { return isImmTy(ImmTyOModSI); }
256   bool isDMask() const { return isImmTy(ImmTyDMask); }
257   bool isUNorm() const { return isImmTy(ImmTyUNorm); }
258   bool isDA() const { return isImmTy(ImmTyDA); }
259   bool isR128() const { return isImmTy(ImmTyUNorm); }
260   bool isLWE() const { return isImmTy(ImmTyLWE); }
261   bool isOff() const { return isImmTy(ImmTyOff); }
262   bool isExpTgt() const { return isImmTy(ImmTyExpTgt); }
263   bool isExpVM() const { return isImmTy(ImmTyExpVM); }
264   bool isExpCompr() const { return isImmTy(ImmTyExpCompr); }
265   bool isOffen() const { return isImmTy(ImmTyOffen); }
266   bool isIdxen() const { return isImmTy(ImmTyIdxen); }
267   bool isAddr64() const { return isImmTy(ImmTyAddr64); }
268   bool isOffset() const { return isImmTy(ImmTyOffset) && isUInt<16>(getImm()); }
269   bool isOffset0() const { return isImmTy(ImmTyOffset0) && isUInt<16>(getImm()); }
270   bool isOffset1() const { return isImmTy(ImmTyOffset1) && isUInt<8>(getImm()); }
271   bool isGDS() const { return isImmTy(ImmTyGDS); }
272   bool isGLC() const { return isImmTy(ImmTyGLC); }
273   bool isSLC() const { return isImmTy(ImmTySLC); }
274   bool isTFE() const { return isImmTy(ImmTyTFE); }
275   bool isBankMask() const { return isImmTy(ImmTyDppBankMask); }
276   bool isRowMask() const { return isImmTy(ImmTyDppRowMask); }
277   bool isBoundCtrl() const { return isImmTy(ImmTyDppBoundCtrl); }
278   bool isSDWADstSel() const { return isImmTy(ImmTySdwaDstSel); }
279   bool isSDWASrc0Sel() const { return isImmTy(ImmTySdwaSrc0Sel); }
280   bool isSDWASrc1Sel() const { return isImmTy(ImmTySdwaSrc1Sel); }
281   bool isSDWADstUnused() const { return isImmTy(ImmTySdwaDstUnused); }
282 
283   bool isMod() const {
284     return isClampSI() || isOModSI();
285   }
286 
287   bool isRegOrImm() const {
288     return isReg() || isImm();
289   }
290 
291   bool isRegClass(unsigned RCID) const;
292 
293   bool isSCSrcB16() const {
294     return isRegClass(AMDGPU::SReg_32RegClassID) || isInlinableImm(MVT::i16);
295   }
296 
297   bool isSCSrcB32() const {
298     return isRegClass(AMDGPU::SReg_32RegClassID) || isInlinableImm(MVT::i32);
299   }
300 
301   bool isSCSrcB64() const {
302     return isRegClass(AMDGPU::SReg_64RegClassID) || isInlinableImm(MVT::i64);
303   }
304 
305   bool isSCSrcF16() const {
306     return isRegClass(AMDGPU::SReg_32RegClassID) || isInlinableImm(MVT::f16);
307   }
308 
309   bool isSCSrcF32() const {
310     return isRegClass(AMDGPU::SReg_32RegClassID) || isInlinableImm(MVT::f32);
311   }
312 
313   bool isSCSrcF64() const {
314     return isRegClass(AMDGPU::SReg_64RegClassID) || isInlinableImm(MVT::f64);
315   }
316 
317   bool isSSrcB32() const {
318     return isSCSrcB32() || isLiteralImm(MVT::i32) || isExpr();
319   }
320 
321   bool isSSrcB16() const {
322     return isSCSrcB16() || isLiteralImm(MVT::i16);
323   }
324 
325   bool isSSrcB64() const {
326     // TODO: Find out how SALU supports extension of 32-bit literals to 64 bits.
327     // See isVSrc64().
328     return isSCSrcB64() || isLiteralImm(MVT::i64);
329   }
330 
331   bool isSSrcF32() const {
332     return isSCSrcB32() || isLiteralImm(MVT::f32) || isExpr();
333   }
334 
335   bool isSSrcF64() const {
336     return isSCSrcB64() || isLiteralImm(MVT::f64);
337   }
338 
339   bool isSSrcF16() const {
340     return isSCSrcB16() || isLiteralImm(MVT::f16);
341   }
342 
343   bool isVCSrcB32() const {
344     return isRegClass(AMDGPU::VS_32RegClassID) || isInlinableImm(MVT::i32);
345   }
346 
347   bool isVCSrcB64() const {
348     return isRegClass(AMDGPU::VS_64RegClassID) || isInlinableImm(MVT::i64);
349   }
350 
351   bool isVCSrcB16() const {
352     return isRegClass(AMDGPU::VS_32RegClassID) || isInlinableImm(MVT::i16);
353   }
354 
355   bool isVCSrcF32() const {
356     return isRegClass(AMDGPU::VS_32RegClassID) || isInlinableImm(MVT::f32);
357   }
358 
359   bool isVCSrcF64() const {
360     return isRegClass(AMDGPU::VS_64RegClassID) || isInlinableImm(MVT::f64);
361   }
362 
363   bool isVCSrcF16() const {
364     return isRegClass(AMDGPU::VS_32RegClassID) || isInlinableImm(MVT::f16);
365   }
366 
367   bool isVSrcB32() const {
368     return isVCSrcF32() || isLiteralImm(MVT::i32);
369   }
370 
371   bool isVSrcB64() const {
372     return isVCSrcF64() || isLiteralImm(MVT::i64);
373   }
374 
375   bool isVSrcB16() const {
376     return isVCSrcF16() || isLiteralImm(MVT::i16);
377   }
378 
379   bool isVSrcF32() const {
380     return isVCSrcF32() || isLiteralImm(MVT::f32);
381   }
382 
383   bool isVSrcF64() const {
384     return isVCSrcF64() || isLiteralImm(MVT::f64);
385   }
386 
387   bool isVSrcF16() const {
388     return isVCSrcF16() || isLiteralImm(MVT::f16);
389   }
390 
391   bool isKImmFP32() const {
392     return isLiteralImm(MVT::f32);
393   }
394 
395   bool isKImmFP16() const {
396     return isLiteralImm(MVT::f16);
397   }
398 
399   bool isMem() const override {
400     return false;
401   }
402 
403   bool isExpr() const {
404     return Kind == Expression;
405   }
406 
407   bool isSoppBrTarget() const {
408     return isExpr() || isImm();
409   }
410 
411   bool isSWaitCnt() const;
412   bool isHwreg() const;
413   bool isSendMsg() const;
414   bool isSMRDOffset8() const;
415   bool isSMRDOffset20() const;
416   bool isSMRDLiteralOffset() const;
417   bool isDPPCtrl() const;
418   bool isGPRIdxMode() const;
419 
420   StringRef getExpressionAsToken() const {
421     assert(isExpr());
422     const MCSymbolRefExpr *S = cast<MCSymbolRefExpr>(Expr);
423     return S->getSymbol().getName();
424   }
425 
426   StringRef getToken() const {
427     assert(isToken());
428 
429     if (Kind == Expression)
430       return getExpressionAsToken();
431 
432     return StringRef(Tok.Data, Tok.Length);
433   }
434 
435   int64_t getImm() const {
436     assert(isImm());
437     return Imm.Val;
438   }
439 
440   enum ImmTy getImmTy() const {
441     assert(isImm());
442     return Imm.Type;
443   }
444 
445   unsigned getReg() const override {
446     return Reg.RegNo;
447   }
448 
449   SMLoc getStartLoc() const override {
450     return StartLoc;
451   }
452 
453   SMLoc getEndLoc() const override {
454     return EndLoc;
455   }
456 
457   Modifiers getModifiers() const {
458     assert(isRegKind() || isImmTy(ImmTyNone));
459     return isRegKind() ? Reg.Mods : Imm.Mods;
460   }
461 
462   void setModifiers(Modifiers Mods) {
463     assert(isRegKind() || isImmTy(ImmTyNone));
464     if (isRegKind())
465       Reg.Mods = Mods;
466     else
467       Imm.Mods = Mods;
468   }
469 
470   bool hasModifiers() const {
471     return getModifiers().hasModifiers();
472   }
473 
474   bool hasFPModifiers() const {
475     return getModifiers().hasFPModifiers();
476   }
477 
478   bool hasIntModifiers() const {
479     return getModifiers().hasIntModifiers();
480   }
481 
482   void addImmOperands(MCInst &Inst, unsigned N, bool ApplyModifiers = true) const;
483 
484   void addLiteralImmOperand(MCInst &Inst, int64_t Val) const;
485 
486   template <unsigned Bitwidth>
487   void addKImmFPOperands(MCInst &Inst, unsigned N) const;
488 
489   void addKImmFP16Operands(MCInst &Inst, unsigned N) const {
490     addKImmFPOperands<16>(Inst, N);
491   }
492 
493   void addKImmFP32Operands(MCInst &Inst, unsigned N) const {
494     addKImmFPOperands<32>(Inst, N);
495   }
496 
497   void addRegOperands(MCInst &Inst, unsigned N) const;
498 
499   void addRegOrImmOperands(MCInst &Inst, unsigned N) const {
500     if (isRegKind())
501       addRegOperands(Inst, N);
502     else if (isExpr())
503       Inst.addOperand(MCOperand::createExpr(Expr));
504     else
505       addImmOperands(Inst, N);
506   }
507 
508   void addRegOrImmWithInputModsOperands(MCInst &Inst, unsigned N) const {
509     Modifiers Mods = getModifiers();
510     Inst.addOperand(MCOperand::createImm(Mods.getModifiersOperand()));
511     if (isRegKind()) {
512       addRegOperands(Inst, N);
513     } else {
514       addImmOperands(Inst, N, false);
515     }
516   }
517 
518   void addRegOrImmWithFPInputModsOperands(MCInst &Inst, unsigned N) const {
519     assert(!hasIntModifiers());
520     addRegOrImmWithInputModsOperands(Inst, N);
521   }
522 
523   void addRegOrImmWithIntInputModsOperands(MCInst &Inst, unsigned N) const {
524     assert(!hasFPModifiers());
525     addRegOrImmWithInputModsOperands(Inst, N);
526   }
527 
528   void addSoppBrTargetOperands(MCInst &Inst, unsigned N) const {
529     if (isImm())
530       addImmOperands(Inst, N);
531     else {
532       assert(isExpr());
533       Inst.addOperand(MCOperand::createExpr(Expr));
534     }
535   }
536 
537   static void printImmTy(raw_ostream& OS, ImmTy Type) {
538     switch (Type) {
539     case ImmTyNone: OS << "None"; break;
540     case ImmTyGDS: OS << "GDS"; break;
541     case ImmTyOffen: OS << "Offen"; break;
542     case ImmTyIdxen: OS << "Idxen"; break;
543     case ImmTyAddr64: OS << "Addr64"; break;
544     case ImmTyOffset: OS << "Offset"; break;
545     case ImmTyOffset0: OS << "Offset0"; break;
546     case ImmTyOffset1: OS << "Offset1"; break;
547     case ImmTyGLC: OS << "GLC"; break;
548     case ImmTySLC: OS << "SLC"; break;
549     case ImmTyTFE: OS << "TFE"; break;
550     case ImmTyClampSI: OS << "ClampSI"; break;
551     case ImmTyOModSI: OS << "OModSI"; break;
552     case ImmTyDppCtrl: OS << "DppCtrl"; break;
553     case ImmTyDppRowMask: OS << "DppRowMask"; break;
554     case ImmTyDppBankMask: OS << "DppBankMask"; break;
555     case ImmTyDppBoundCtrl: OS << "DppBoundCtrl"; break;
556     case ImmTySdwaDstSel: OS << "SdwaDstSel"; break;
557     case ImmTySdwaSrc0Sel: OS << "SdwaSrc0Sel"; break;
558     case ImmTySdwaSrc1Sel: OS << "SdwaSrc1Sel"; break;
559     case ImmTySdwaDstUnused: OS << "SdwaDstUnused"; break;
560     case ImmTyDMask: OS << "DMask"; break;
561     case ImmTyUNorm: OS << "UNorm"; break;
562     case ImmTyDA: OS << "DA"; break;
563     case ImmTyR128: OS << "R128"; break;
564     case ImmTyLWE: OS << "LWE"; break;
565     case ImmTyOff: OS << "Off"; break;
566     case ImmTyExpTgt: OS << "ExpTgt"; break;
567     case ImmTyExpCompr: OS << "ExpCompr"; break;
568     case ImmTyExpVM: OS << "ExpVM"; break;
569     case ImmTyHwreg: OS << "Hwreg"; break;
570     case ImmTySendMsg: OS << "SendMsg"; break;
571     }
572   }
573 
574   void print(raw_ostream &OS) const override {
575     switch (Kind) {
576     case Register:
577       OS << "<register " << getReg() << " mods: " << Reg.Mods << '>';
578       break;
579     case Immediate:
580       OS << '<' << getImm();
581       if (getImmTy() != ImmTyNone) {
582         OS << " type: "; printImmTy(OS, getImmTy());
583       }
584       OS << " mods: " << Imm.Mods << '>';
585       break;
586     case Token:
587       OS << '\'' << getToken() << '\'';
588       break;
589     case Expression:
590       OS << "<expr " << *Expr << '>';
591       break;
592     }
593   }
594 
595   static AMDGPUOperand::Ptr CreateImm(const AMDGPUAsmParser *AsmParser,
596                                       int64_t Val, SMLoc Loc,
597                                       enum ImmTy Type = ImmTyNone,
598                                       bool IsFPImm = false) {
599     auto Op = llvm::make_unique<AMDGPUOperand>(Immediate, AsmParser);
600     Op->Imm.Val = Val;
601     Op->Imm.IsFPImm = IsFPImm;
602     Op->Imm.Type = Type;
603     Op->Imm.Mods = Modifiers();
604     Op->StartLoc = Loc;
605     Op->EndLoc = Loc;
606     return Op;
607   }
608 
609   static AMDGPUOperand::Ptr CreateToken(const AMDGPUAsmParser *AsmParser,
610                                         StringRef Str, SMLoc Loc,
611                                         bool HasExplicitEncodingSize = true) {
612     auto Res = llvm::make_unique<AMDGPUOperand>(Token, AsmParser);
613     Res->Tok.Data = Str.data();
614     Res->Tok.Length = Str.size();
615     Res->StartLoc = Loc;
616     Res->EndLoc = Loc;
617     return Res;
618   }
619 
620   static AMDGPUOperand::Ptr CreateReg(const AMDGPUAsmParser *AsmParser,
621                                       unsigned RegNo, SMLoc S,
622                                       SMLoc E,
623                                       bool ForceVOP3) {
624     auto Op = llvm::make_unique<AMDGPUOperand>(Register, AsmParser);
625     Op->Reg.RegNo = RegNo;
626     Op->Reg.Mods = Modifiers();
627     Op->Reg.IsForcedVOP3 = ForceVOP3;
628     Op->StartLoc = S;
629     Op->EndLoc = E;
630     return Op;
631   }
632 
633   static AMDGPUOperand::Ptr CreateExpr(const AMDGPUAsmParser *AsmParser,
634                                        const class MCExpr *Expr, SMLoc S) {
635     auto Op = llvm::make_unique<AMDGPUOperand>(Expression, AsmParser);
636     Op->Expr = Expr;
637     Op->StartLoc = S;
638     Op->EndLoc = S;
639     return Op;
640   }
641 };
642 
643 raw_ostream &operator <<(raw_ostream &OS, AMDGPUOperand::Modifiers Mods) {
644   OS << "abs:" << Mods.Abs << " neg: " << Mods.Neg << " sext:" << Mods.Sext;
645   return OS;
646 }
647 
648 //===----------------------------------------------------------------------===//
649 // AsmParser
650 //===----------------------------------------------------------------------===//
651 
652 class AMDGPUAsmParser : public MCTargetAsmParser {
653   const MCInstrInfo &MII;
654   MCAsmParser &Parser;
655 
656   unsigned ForcedEncodingSize;
657   bool ForcedDPP;
658   bool ForcedSDWA;
659 
660   /// @name Auto-generated Match Functions
661   /// {
662 
663 #define GET_ASSEMBLER_HEADER
664 #include "AMDGPUGenAsmMatcher.inc"
665 
666   /// }
667 
668 private:
669   bool ParseDirectiveMajorMinor(uint32_t &Major, uint32_t &Minor);
670   bool ParseDirectiveHSACodeObjectVersion();
671   bool ParseDirectiveHSACodeObjectISA();
672   bool ParseAMDKernelCodeTValue(StringRef ID, amd_kernel_code_t &Header);
673   bool ParseDirectiveAMDKernelCodeT();
674   bool ParseSectionDirectiveHSAText();
675   bool subtargetHasRegister(const MCRegisterInfo &MRI, unsigned RegNo) const;
676   bool ParseDirectiveAMDGPUHsaKernel();
677   bool ParseDirectiveAMDGPUHsaModuleGlobal();
678   bool ParseDirectiveAMDGPUHsaProgramGlobal();
679   bool ParseSectionDirectiveHSADataGlobalAgent();
680   bool ParseSectionDirectiveHSADataGlobalProgram();
681   bool ParseSectionDirectiveHSARodataReadonlyAgent();
682   bool AddNextRegisterToList(unsigned& Reg, unsigned& RegWidth, RegisterKind RegKind, unsigned Reg1, unsigned RegNum);
683   bool ParseAMDGPURegister(RegisterKind& RegKind, unsigned& Reg, unsigned& RegNum, unsigned& RegWidth);
684   void cvtMubufImpl(MCInst &Inst, const OperandVector &Operands, bool IsAtomic, bool IsAtomicReturn);
685 
686 public:
687   enum AMDGPUMatchResultTy {
688     Match_PreferE32 = FIRST_TARGET_MATCH_RESULT_TY
689   };
690 
691   AMDGPUAsmParser(const MCSubtargetInfo &STI, MCAsmParser &_Parser,
692                const MCInstrInfo &MII,
693                const MCTargetOptions &Options)
694       : MCTargetAsmParser(Options, STI), MII(MII), Parser(_Parser),
695         ForcedEncodingSize(0),
696         ForcedDPP(false),
697         ForcedSDWA(false) {
698     MCAsmParserExtension::Initialize(Parser);
699 
700     if (getSTI().getFeatureBits().none()) {
701       // Set default features.
702       copySTI().ToggleFeature("SOUTHERN_ISLANDS");
703     }
704 
705     setAvailableFeatures(ComputeAvailableFeatures(getSTI().getFeatureBits()));
706 
707     {
708       // TODO: make those pre-defined variables read-only.
709       // Currently there is none suitable machinery in the core llvm-mc for this.
710       // MCSymbol::isRedefinable is intended for another purpose, and
711       // AsmParser::parseDirectiveSet() cannot be specialized for specific target.
712       AMDGPU::IsaVersion Isa = AMDGPU::getIsaVersion(getSTI().getFeatureBits());
713       MCContext &Ctx = getContext();
714       MCSymbol *Sym = Ctx.getOrCreateSymbol(Twine(".option.machine_version_major"));
715       Sym->setVariableValue(MCConstantExpr::create(Isa.Major, Ctx));
716       Sym = Ctx.getOrCreateSymbol(Twine(".option.machine_version_minor"));
717       Sym->setVariableValue(MCConstantExpr::create(Isa.Minor, Ctx));
718       Sym = Ctx.getOrCreateSymbol(Twine(".option.machine_version_stepping"));
719       Sym->setVariableValue(MCConstantExpr::create(Isa.Stepping, Ctx));
720     }
721   }
722 
723   bool isSI() const {
724     return AMDGPU::isSI(getSTI());
725   }
726 
727   bool isCI() const {
728     return AMDGPU::isCI(getSTI());
729   }
730 
731   bool isVI() const {
732     return AMDGPU::isVI(getSTI());
733   }
734 
735   bool hasInv2PiInlineImm() const {
736     return getSTI().getFeatureBits()[AMDGPU::FeatureInv2PiInlineImm];
737   }
738 
739   bool hasSGPR102_SGPR103() const {
740     return !isVI();
741   }
742 
743   AMDGPUTargetStreamer &getTargetStreamer() {
744     MCTargetStreamer &TS = *getParser().getStreamer().getTargetStreamer();
745     return static_cast<AMDGPUTargetStreamer &>(TS);
746   }
747 
748   const MCRegisterInfo *getMRI() const {
749     // We need this const_cast because for some reason getContext() is not const
750     // in MCAsmParser.
751     return const_cast<AMDGPUAsmParser*>(this)->getContext().getRegisterInfo();
752   }
753 
754   const MCInstrInfo *getMII() const {
755     return &MII;
756   }
757 
758   void setForcedEncodingSize(unsigned Size) { ForcedEncodingSize = Size; }
759   void setForcedDPP(bool ForceDPP_) { ForcedDPP = ForceDPP_; }
760   void setForcedSDWA(bool ForceSDWA_) { ForcedSDWA = ForceSDWA_; }
761 
762   unsigned getForcedEncodingSize() const { return ForcedEncodingSize; }
763   bool isForcedVOP3() const { return ForcedEncodingSize == 64; }
764   bool isForcedDPP() const { return ForcedDPP; }
765   bool isForcedSDWA() const { return ForcedSDWA; }
766 
767   std::unique_ptr<AMDGPUOperand> parseRegister();
768   bool ParseRegister(unsigned &RegNo, SMLoc &StartLoc, SMLoc &EndLoc) override;
769   unsigned checkTargetMatchPredicate(MCInst &Inst) override;
770   unsigned validateTargetOperandClass(MCParsedAsmOperand &Op,
771                                       unsigned Kind) override;
772   bool MatchAndEmitInstruction(SMLoc IDLoc, unsigned &Opcode,
773                                OperandVector &Operands, MCStreamer &Out,
774                                uint64_t &ErrorInfo,
775                                bool MatchingInlineAsm) override;
776   bool ParseDirective(AsmToken DirectiveID) override;
777   OperandMatchResultTy parseOperand(OperandVector &Operands, StringRef Mnemonic);
778   StringRef parseMnemonicSuffix(StringRef Name);
779   bool ParseInstruction(ParseInstructionInfo &Info, StringRef Name,
780                         SMLoc NameLoc, OperandVector &Operands) override;
781   //bool ProcessInstruction(MCInst &Inst);
782 
783   OperandMatchResultTy parseIntWithPrefix(const char *Prefix, int64_t &Int);
784   OperandMatchResultTy
785   parseIntWithPrefix(const char *Prefix, OperandVector &Operands,
786                      enum AMDGPUOperand::ImmTy ImmTy = AMDGPUOperand::ImmTyNone,
787                      bool (*ConvertResult)(int64_t &) = nullptr);
788   OperandMatchResultTy
789   parseNamedBit(const char *Name, OperandVector &Operands,
790                 enum AMDGPUOperand::ImmTy ImmTy = AMDGPUOperand::ImmTyNone);
791   OperandMatchResultTy parseStringWithPrefix(StringRef Prefix,
792                                              StringRef &Value);
793 
794   OperandMatchResultTy parseImm(OperandVector &Operands);
795   OperandMatchResultTy parseRegOrImm(OperandVector &Operands);
796   OperandMatchResultTy parseRegOrImmWithFPInputMods(OperandVector &Operands);
797   OperandMatchResultTy parseRegOrImmWithIntInputMods(OperandVector &Operands);
798   OperandMatchResultTy parseVReg32OrOff(OperandVector &Operands);
799 
800   void cvtDSOffset01(MCInst &Inst, const OperandVector &Operands);
801   void cvtDS(MCInst &Inst, const OperandVector &Operands);
802   void cvtExp(MCInst &Inst, const OperandVector &Operands);
803 
804   bool parseCnt(int64_t &IntVal);
805   OperandMatchResultTy parseSWaitCntOps(OperandVector &Operands);
806   OperandMatchResultTy parseHwreg(OperandVector &Operands);
807 
808 private:
809   struct OperandInfoTy {
810     int64_t Id;
811     bool IsSymbolic;
812     OperandInfoTy(int64_t Id_) : Id(Id_), IsSymbolic(false) { }
813   };
814 
815   bool parseSendMsgConstruct(OperandInfoTy &Msg, OperandInfoTy &Operation, int64_t &StreamId);
816   bool parseHwregConstruct(OperandInfoTy &HwReg, int64_t &Offset, int64_t &Width);
817 
818   void errorExpTgt();
819   OperandMatchResultTy parseExpTgtImpl(StringRef Str, uint8_t &Val);
820 
821 public:
822   OperandMatchResultTy parseOptionalOperand(OperandVector &Operands);
823 
824   OperandMatchResultTy parseExpTgt(OperandVector &Operands);
825   OperandMatchResultTy parseSendMsgOp(OperandVector &Operands);
826   OperandMatchResultTy parseSOppBrTarget(OperandVector &Operands);
827 
828   void cvtMubuf(MCInst &Inst, const OperandVector &Operands) { cvtMubufImpl(Inst, Operands, false, false); }
829   void cvtMubufAtomic(MCInst &Inst, const OperandVector &Operands) { cvtMubufImpl(Inst, Operands, true, false); }
830   void cvtMubufAtomicReturn(MCInst &Inst, const OperandVector &Operands) { cvtMubufImpl(Inst, Operands, true, true); }
831   AMDGPUOperand::Ptr defaultGLC() const;
832   AMDGPUOperand::Ptr defaultSLC() const;
833   AMDGPUOperand::Ptr defaultTFE() const;
834 
835   AMDGPUOperand::Ptr defaultDMask() const;
836   AMDGPUOperand::Ptr defaultUNorm() const;
837   AMDGPUOperand::Ptr defaultDA() const;
838   AMDGPUOperand::Ptr defaultR128() const;
839   AMDGPUOperand::Ptr defaultLWE() const;
840   AMDGPUOperand::Ptr defaultSMRDOffset8() const;
841   AMDGPUOperand::Ptr defaultSMRDOffset20() const;
842   AMDGPUOperand::Ptr defaultSMRDLiteralOffset() const;
843   AMDGPUOperand::Ptr defaultExpTgt() const;
844   AMDGPUOperand::Ptr defaultExpCompr() const;
845   AMDGPUOperand::Ptr defaultExpVM() const;
846 
847   OperandMatchResultTy parseOModOperand(OperandVector &Operands);
848 
849   void cvtId(MCInst &Inst, const OperandVector &Operands);
850   void cvtVOP3_2_mod(MCInst &Inst, const OperandVector &Operands);
851   void cvtVOP3(MCInst &Inst, const OperandVector &Operands);
852 
853   void cvtMIMG(MCInst &Inst, const OperandVector &Operands);
854   void cvtMIMGAtomic(MCInst &Inst, const OperandVector &Operands);
855 
856   OperandMatchResultTy parseDPPCtrl(OperandVector &Operands);
857   AMDGPUOperand::Ptr defaultRowMask() const;
858   AMDGPUOperand::Ptr defaultBankMask() const;
859   AMDGPUOperand::Ptr defaultBoundCtrl() const;
860   void cvtDPP(MCInst &Inst, const OperandVector &Operands);
861 
862   OperandMatchResultTy parseSDWASel(OperandVector &Operands, StringRef Prefix,
863                                     AMDGPUOperand::ImmTy Type);
864   OperandMatchResultTy parseSDWADstUnused(OperandVector &Operands);
865   void cvtSdwaVOP1(MCInst &Inst, const OperandVector &Operands);
866   void cvtSdwaVOP2(MCInst &Inst, const OperandVector &Operands);
867   void cvtSdwaVOPC(MCInst &Inst, const OperandVector &Operands);
868   void cvtSDWA(MCInst &Inst, const OperandVector &Operands,
869                uint64_t BasicInstType);
870 };
871 
872 struct OptionalOperand {
873   const char *Name;
874   AMDGPUOperand::ImmTy Type;
875   bool IsBit;
876   bool (*ConvertResult)(int64_t&);
877 };
878 
879 } // end anonymous namespace
880 
881 // May be called with integer type with equivalent bitwidth.
882 static const fltSemantics *getFltSemantics(unsigned Size) {
883   switch (Size) {
884   case 4:
885     return &APFloat::IEEEsingle();
886   case 8:
887     return &APFloat::IEEEdouble();
888   case 2:
889     return &APFloat::IEEEhalf();
890   default:
891     llvm_unreachable("unsupported fp type");
892   }
893 }
894 
895 static const fltSemantics *getFltSemantics(MVT VT) {
896   return getFltSemantics(VT.getSizeInBits() / 8);
897 }
898 
899 //===----------------------------------------------------------------------===//
900 // Operand
901 //===----------------------------------------------------------------------===//
902 
903 static bool canLosslesslyConvertToFPType(APFloat &FPLiteral, MVT VT) {
904   bool Lost;
905 
906   // Convert literal to single precision
907   APFloat::opStatus Status = FPLiteral.convert(*getFltSemantics(VT),
908                                                APFloat::rmNearestTiesToEven,
909                                                &Lost);
910   // We allow precision lost but not overflow or underflow
911   if (Status != APFloat::opOK &&
912       Lost &&
913       ((Status & APFloat::opOverflow)  != 0 ||
914        (Status & APFloat::opUnderflow) != 0)) {
915     return false;
916   }
917 
918   return true;
919 }
920 
921 bool AMDGPUOperand::isInlinableImm(MVT type) const {
922   if (!isImmTy(ImmTyNone)) {
923     // Only plain immediates are inlinable (e.g. "clamp" attribute is not)
924     return false;
925   }
926   // TODO: We should avoid using host float here. It would be better to
927   // check the float bit values which is what a few other places do.
928   // We've had bot failures before due to weird NaN support on mips hosts.
929 
930   APInt Literal(64, Imm.Val);
931 
932   if (Imm.IsFPImm) { // We got fp literal token
933     if (type == MVT::f64 || type == MVT::i64) { // Expected 64-bit operand
934       return AMDGPU::isInlinableLiteral64(Imm.Val,
935                                           AsmParser->hasInv2PiInlineImm());
936     }
937 
938     APFloat FPLiteral(APFloat::IEEEdouble(), APInt(64, Imm.Val));
939     if (!canLosslesslyConvertToFPType(FPLiteral, type))
940       return false;
941 
942     // Check if single precision literal is inlinable
943     return AMDGPU::isInlinableLiteral32(
944       static_cast<int32_t>(FPLiteral.bitcastToAPInt().getZExtValue()),
945       AsmParser->hasInv2PiInlineImm());
946   }
947 
948 
949   // We got int literal token.
950   if (type == MVT::f64 || type == MVT::i64) { // Expected 64-bit operand
951     return AMDGPU::isInlinableLiteral64(Imm.Val,
952                                         AsmParser->hasInv2PiInlineImm());
953   }
954 
955   if (type.getScalarSizeInBits() == 16) {
956     return AMDGPU::isInlinableLiteral16(
957       static_cast<int16_t>(Literal.getLoBits(16).getSExtValue()),
958       AsmParser->hasInv2PiInlineImm());
959   }
960 
961   return AMDGPU::isInlinableLiteral32(
962     static_cast<int32_t>(Literal.getLoBits(32).getZExtValue()),
963     AsmParser->hasInv2PiInlineImm());
964 }
965 
966 bool AMDGPUOperand::isLiteralImm(MVT type) const {
967   // Check that this imediate can be added as literal
968   if (!isImmTy(ImmTyNone)) {
969     return false;
970   }
971 
972   if (!Imm.IsFPImm) {
973     // We got int literal token.
974 
975     unsigned Size = type.getSizeInBits();
976     if (Size == 64)
977       Size = 32;
978 
979     // FIXME: 64-bit operands can zero extend, sign extend, or pad zeroes for FP
980     // types.
981     return isUIntN(Size, Imm.Val) || isIntN(Size, Imm.Val);
982   }
983 
984   // We got fp literal token
985   if (type == MVT::f64) { // Expected 64-bit fp operand
986     // We would set low 64-bits of literal to zeroes but we accept this literals
987     return true;
988   }
989 
990   if (type == MVT::i64) { // Expected 64-bit int operand
991     // We don't allow fp literals in 64-bit integer instructions. It is
992     // unclear how we should encode them.
993     return false;
994   }
995 
996   APFloat FPLiteral(APFloat::IEEEdouble(), APInt(64, Imm.Val));
997   return canLosslesslyConvertToFPType(FPLiteral, type);
998 }
999 
1000 bool AMDGPUOperand::isRegClass(unsigned RCID) const {
1001   return isReg() && AsmParser->getMRI()->getRegClass(RCID).contains(getReg());
1002 }
1003 
1004 void AMDGPUOperand::addImmOperands(MCInst &Inst, unsigned N, bool ApplyModifiers) const {
1005   int64_t Val = Imm.Val;
1006   if (isImmTy(ImmTyNone) && ApplyModifiers && Imm.Mods.hasFPModifiers() && Imm.Mods.Neg) {
1007     // Apply modifiers to immediate value. Only negate can get here
1008     if (Imm.IsFPImm) {
1009       APFloat F(BitsToDouble(Val));
1010       F.changeSign();
1011       Val = F.bitcastToAPInt().getZExtValue();
1012     } else {
1013       Val = -Val;
1014     }
1015   }
1016 
1017   if (AMDGPU::isSISrcOperand(AsmParser->getMII()->get(Inst.getOpcode()),
1018                              Inst.getNumOperands())) {
1019     addLiteralImmOperand(Inst, Val);
1020   } else {
1021     Inst.addOperand(MCOperand::createImm(Val));
1022   }
1023 }
1024 
1025 void AMDGPUOperand::addLiteralImmOperand(MCInst &Inst, int64_t Val) const {
1026   const auto& InstDesc = AsmParser->getMII()->get(Inst.getOpcode());
1027   auto OpNum = Inst.getNumOperands();
1028   // Check that this operand accepts literals
1029   assert(AMDGPU::isSISrcOperand(InstDesc, OpNum));
1030 
1031   auto OpSize = AMDGPU::getOperandSize(InstDesc, OpNum); // expected operand size
1032 
1033   if (Imm.IsFPImm) { // We got fp literal token
1034     APInt Literal(64, Val);
1035 
1036     switch (OpSize) {
1037     case 8: {
1038       if (AMDGPU::isInlinableLiteral64(Literal.getZExtValue(),
1039                                        AsmParser->hasInv2PiInlineImm())) {
1040         Inst.addOperand(MCOperand::createImm(Literal.getZExtValue()));
1041         return;
1042       }
1043 
1044       // Non-inlineable
1045       if (AMDGPU::isSISrcFPOperand(InstDesc, OpNum)) { // Expected 64-bit fp operand
1046         // For fp operands we check if low 32 bits are zeros
1047         if (Literal.getLoBits(32) != 0) {
1048           const_cast<AMDGPUAsmParser *>(AsmParser)->Warning(Inst.getLoc(),
1049           "Can't encode literal as exact 64-bit floating-point operand. "
1050           "Low 32-bits will be set to zero");
1051         }
1052 
1053         Inst.addOperand(MCOperand::createImm(Literal.lshr(32).getZExtValue()));
1054         return;
1055       }
1056 
1057       // We don't allow fp literals in 64-bit integer instructions. It is
1058       // unclear how we should encode them. This case should be checked earlier
1059       // in predicate methods (isLiteralImm())
1060       llvm_unreachable("fp literal in 64-bit integer instruction.");
1061     }
1062     case 4:
1063     case 2: {
1064       bool lost;
1065       APFloat FPLiteral(APFloat::IEEEdouble(), Literal);
1066       // Convert literal to single precision
1067       FPLiteral.convert(*getFltSemantics(OpSize),
1068                         APFloat::rmNearestTiesToEven, &lost);
1069       // We allow precision lost but not overflow or underflow. This should be
1070       // checked earlier in isLiteralImm()
1071       Inst.addOperand(MCOperand::createImm(FPLiteral.bitcastToAPInt().getZExtValue()));
1072       return;
1073     }
1074     default:
1075       llvm_unreachable("invalid operand size");
1076     }
1077 
1078     return;
1079   }
1080 
1081    // We got int literal token.
1082   // Only sign extend inline immediates.
1083   // FIXME: No errors on truncation
1084   switch (OpSize) {
1085   case 4: {
1086     if (isInt<32>(Val) &&
1087         AMDGPU::isInlinableLiteral32(static_cast<int32_t>(Val),
1088                                      AsmParser->hasInv2PiInlineImm())) {
1089       Inst.addOperand(MCOperand::createImm(Val));
1090       return;
1091     }
1092 
1093     Inst.addOperand(MCOperand::createImm(Val & 0xffffffff));
1094     return;
1095   }
1096   case 8: {
1097     if (AMDGPU::isInlinableLiteral64(Val,
1098                                      AsmParser->hasInv2PiInlineImm())) {
1099       Inst.addOperand(MCOperand::createImm(Val));
1100       return;
1101     }
1102 
1103     Inst.addOperand(MCOperand::createImm(Lo_32(Val)));
1104     return;
1105   }
1106   case 2: {
1107     if (isInt<16>(Val) &&
1108         AMDGPU::isInlinableLiteral16(static_cast<int16_t>(Val),
1109                                      AsmParser->hasInv2PiInlineImm())) {
1110       Inst.addOperand(MCOperand::createImm(Val));
1111       return;
1112     }
1113 
1114     Inst.addOperand(MCOperand::createImm(Val & 0xffff));
1115     return;
1116   }
1117   default:
1118     llvm_unreachable("invalid operand size");
1119   }
1120 }
1121 
1122 template <unsigned Bitwidth>
1123 void AMDGPUOperand::addKImmFPOperands(MCInst &Inst, unsigned N) const {
1124   APInt Literal(64, Imm.Val);
1125 
1126   if (!Imm.IsFPImm) {
1127     // We got int literal token.
1128     Inst.addOperand(MCOperand::createImm(Literal.getLoBits(Bitwidth).getZExtValue()));
1129     return;
1130   }
1131 
1132   bool Lost;
1133   APFloat FPLiteral(APFloat::IEEEdouble(), Literal);
1134   FPLiteral.convert(*getFltSemantics(Bitwidth / 8),
1135                     APFloat::rmNearestTiesToEven, &Lost);
1136   Inst.addOperand(MCOperand::createImm(FPLiteral.bitcastToAPInt().getZExtValue()));
1137 }
1138 
1139 void AMDGPUOperand::addRegOperands(MCInst &Inst, unsigned N) const {
1140   Inst.addOperand(MCOperand::createReg(AMDGPU::getMCReg(getReg(), AsmParser->getSTI())));
1141 }
1142 
1143 //===----------------------------------------------------------------------===//
1144 // AsmParser
1145 //===----------------------------------------------------------------------===//
1146 
1147 static int getRegClass(RegisterKind Is, unsigned RegWidth) {
1148   if (Is == IS_VGPR) {
1149     switch (RegWidth) {
1150       default: return -1;
1151       case 1: return AMDGPU::VGPR_32RegClassID;
1152       case 2: return AMDGPU::VReg_64RegClassID;
1153       case 3: return AMDGPU::VReg_96RegClassID;
1154       case 4: return AMDGPU::VReg_128RegClassID;
1155       case 8: return AMDGPU::VReg_256RegClassID;
1156       case 16: return AMDGPU::VReg_512RegClassID;
1157     }
1158   } else if (Is == IS_TTMP) {
1159     switch (RegWidth) {
1160       default: return -1;
1161       case 1: return AMDGPU::TTMP_32RegClassID;
1162       case 2: return AMDGPU::TTMP_64RegClassID;
1163       case 4: return AMDGPU::TTMP_128RegClassID;
1164     }
1165   } else if (Is == IS_SGPR) {
1166     switch (RegWidth) {
1167       default: return -1;
1168       case 1: return AMDGPU::SGPR_32RegClassID;
1169       case 2: return AMDGPU::SGPR_64RegClassID;
1170       case 4: return AMDGPU::SGPR_128RegClassID;
1171       case 8: return AMDGPU::SReg_256RegClassID;
1172       case 16: return AMDGPU::SReg_512RegClassID;
1173     }
1174   }
1175   return -1;
1176 }
1177 
1178 static unsigned getSpecialRegForName(StringRef RegName) {
1179   return StringSwitch<unsigned>(RegName)
1180     .Case("exec", AMDGPU::EXEC)
1181     .Case("vcc", AMDGPU::VCC)
1182     .Case("flat_scratch", AMDGPU::FLAT_SCR)
1183     .Case("m0", AMDGPU::M0)
1184     .Case("scc", AMDGPU::SCC)
1185     .Case("tba", AMDGPU::TBA)
1186     .Case("tma", AMDGPU::TMA)
1187     .Case("flat_scratch_lo", AMDGPU::FLAT_SCR_LO)
1188     .Case("flat_scratch_hi", AMDGPU::FLAT_SCR_HI)
1189     .Case("vcc_lo", AMDGPU::VCC_LO)
1190     .Case("vcc_hi", AMDGPU::VCC_HI)
1191     .Case("exec_lo", AMDGPU::EXEC_LO)
1192     .Case("exec_hi", AMDGPU::EXEC_HI)
1193     .Case("tma_lo", AMDGPU::TMA_LO)
1194     .Case("tma_hi", AMDGPU::TMA_HI)
1195     .Case("tba_lo", AMDGPU::TBA_LO)
1196     .Case("tba_hi", AMDGPU::TBA_HI)
1197     .Default(0);
1198 }
1199 
1200 bool AMDGPUAsmParser::ParseRegister(unsigned &RegNo, SMLoc &StartLoc, SMLoc &EndLoc) {
1201   auto R = parseRegister();
1202   if (!R) return true;
1203   assert(R->isReg());
1204   RegNo = R->getReg();
1205   StartLoc = R->getStartLoc();
1206   EndLoc = R->getEndLoc();
1207   return false;
1208 }
1209 
1210 bool AMDGPUAsmParser::AddNextRegisterToList(unsigned& Reg, unsigned& RegWidth, RegisterKind RegKind, unsigned Reg1, unsigned RegNum)
1211 {
1212   switch (RegKind) {
1213   case IS_SPECIAL:
1214     if (Reg == AMDGPU::EXEC_LO && Reg1 == AMDGPU::EXEC_HI) { Reg = AMDGPU::EXEC; RegWidth = 2; return true; }
1215     if (Reg == AMDGPU::FLAT_SCR_LO && Reg1 == AMDGPU::FLAT_SCR_HI) { Reg = AMDGPU::FLAT_SCR; RegWidth = 2; return true; }
1216     if (Reg == AMDGPU::VCC_LO && Reg1 == AMDGPU::VCC_HI) { Reg = AMDGPU::VCC; RegWidth = 2; return true; }
1217     if (Reg == AMDGPU::TBA_LO && Reg1 == AMDGPU::TBA_HI) { Reg = AMDGPU::TBA; RegWidth = 2; return true; }
1218     if (Reg == AMDGPU::TMA_LO && Reg1 == AMDGPU::TMA_HI) { Reg = AMDGPU::TMA; RegWidth = 2; return true; }
1219     return false;
1220   case IS_VGPR:
1221   case IS_SGPR:
1222   case IS_TTMP:
1223     if (Reg1 != Reg + RegWidth) { return false; }
1224     RegWidth++;
1225     return true;
1226   default:
1227     llvm_unreachable("unexpected register kind");
1228   }
1229 }
1230 
1231 bool AMDGPUAsmParser::ParseAMDGPURegister(RegisterKind& RegKind, unsigned& Reg, unsigned& RegNum, unsigned& RegWidth)
1232 {
1233   const MCRegisterInfo *TRI = getContext().getRegisterInfo();
1234   if (getLexer().is(AsmToken::Identifier)) {
1235     StringRef RegName = Parser.getTok().getString();
1236     if ((Reg = getSpecialRegForName(RegName))) {
1237       Parser.Lex();
1238       RegKind = IS_SPECIAL;
1239     } else {
1240       unsigned RegNumIndex = 0;
1241       if (RegName[0] == 'v') {
1242         RegNumIndex = 1;
1243         RegKind = IS_VGPR;
1244       } else if (RegName[0] == 's') {
1245         RegNumIndex = 1;
1246         RegKind = IS_SGPR;
1247       } else if (RegName.startswith("ttmp")) {
1248         RegNumIndex = strlen("ttmp");
1249         RegKind = IS_TTMP;
1250       } else {
1251         return false;
1252       }
1253       if (RegName.size() > RegNumIndex) {
1254         // Single 32-bit register: vXX.
1255         if (RegName.substr(RegNumIndex).getAsInteger(10, RegNum))
1256           return false;
1257         Parser.Lex();
1258         RegWidth = 1;
1259       } else {
1260         // Range of registers: v[XX:YY]. ":YY" is optional.
1261         Parser.Lex();
1262         int64_t RegLo, RegHi;
1263         if (getLexer().isNot(AsmToken::LBrac))
1264           return false;
1265         Parser.Lex();
1266 
1267         if (getParser().parseAbsoluteExpression(RegLo))
1268           return false;
1269 
1270         const bool isRBrace = getLexer().is(AsmToken::RBrac);
1271         if (!isRBrace && getLexer().isNot(AsmToken::Colon))
1272           return false;
1273         Parser.Lex();
1274 
1275         if (isRBrace) {
1276           RegHi = RegLo;
1277         } else {
1278           if (getParser().parseAbsoluteExpression(RegHi))
1279             return false;
1280 
1281           if (getLexer().isNot(AsmToken::RBrac))
1282             return false;
1283           Parser.Lex();
1284         }
1285         RegNum = (unsigned) RegLo;
1286         RegWidth = (RegHi - RegLo) + 1;
1287       }
1288     }
1289   } else if (getLexer().is(AsmToken::LBrac)) {
1290     // List of consecutive registers: [s0,s1,s2,s3]
1291     Parser.Lex();
1292     if (!ParseAMDGPURegister(RegKind, Reg, RegNum, RegWidth))
1293       return false;
1294     if (RegWidth != 1)
1295       return false;
1296     RegisterKind RegKind1;
1297     unsigned Reg1, RegNum1, RegWidth1;
1298     do {
1299       if (getLexer().is(AsmToken::Comma)) {
1300         Parser.Lex();
1301       } else if (getLexer().is(AsmToken::RBrac)) {
1302         Parser.Lex();
1303         break;
1304       } else if (ParseAMDGPURegister(RegKind1, Reg1, RegNum1, RegWidth1)) {
1305         if (RegWidth1 != 1) {
1306           return false;
1307         }
1308         if (RegKind1 != RegKind) {
1309           return false;
1310         }
1311         if (!AddNextRegisterToList(Reg, RegWidth, RegKind1, Reg1, RegNum1)) {
1312           return false;
1313         }
1314       } else {
1315         return false;
1316       }
1317     } while (true);
1318   } else {
1319     return false;
1320   }
1321   switch (RegKind) {
1322   case IS_SPECIAL:
1323     RegNum = 0;
1324     RegWidth = 1;
1325     break;
1326   case IS_VGPR:
1327   case IS_SGPR:
1328   case IS_TTMP:
1329   {
1330     unsigned Size = 1;
1331     if (RegKind == IS_SGPR || RegKind == IS_TTMP) {
1332       // SGPR and TTMP registers must be are aligned. Max required alignment is 4 dwords.
1333       Size = std::min(RegWidth, 4u);
1334     }
1335     if (RegNum % Size != 0)
1336       return false;
1337     RegNum = RegNum / Size;
1338     int RCID = getRegClass(RegKind, RegWidth);
1339     if (RCID == -1)
1340       return false;
1341     const MCRegisterClass RC = TRI->getRegClass(RCID);
1342     if (RegNum >= RC.getNumRegs())
1343       return false;
1344     Reg = RC.getRegister(RegNum);
1345     break;
1346   }
1347 
1348   default:
1349     llvm_unreachable("unexpected register kind");
1350   }
1351 
1352   if (!subtargetHasRegister(*TRI, Reg))
1353     return false;
1354   return true;
1355 }
1356 
1357 std::unique_ptr<AMDGPUOperand> AMDGPUAsmParser::parseRegister() {
1358   const auto &Tok = Parser.getTok();
1359   SMLoc StartLoc = Tok.getLoc();
1360   SMLoc EndLoc = Tok.getEndLoc();
1361   RegisterKind RegKind;
1362   unsigned Reg, RegNum, RegWidth;
1363 
1364   if (!ParseAMDGPURegister(RegKind, Reg, RegNum, RegWidth)) {
1365     return nullptr;
1366   }
1367   return AMDGPUOperand::CreateReg(this, Reg, StartLoc, EndLoc, false);
1368 }
1369 
1370 OperandMatchResultTy
1371 AMDGPUAsmParser::parseImm(OperandVector &Operands) {
1372   // TODO: add syntactic sugar for 1/(2*PI)
1373   bool Minus = false;
1374   if (getLexer().getKind() == AsmToken::Minus) {
1375     Minus = true;
1376     Parser.Lex();
1377   }
1378 
1379   SMLoc S = Parser.getTok().getLoc();
1380   switch(getLexer().getKind()) {
1381   case AsmToken::Integer: {
1382     int64_t IntVal;
1383     if (getParser().parseAbsoluteExpression(IntVal))
1384       return MatchOperand_ParseFail;
1385     if (Minus)
1386       IntVal *= -1;
1387     Operands.push_back(AMDGPUOperand::CreateImm(this, IntVal, S));
1388     return MatchOperand_Success;
1389   }
1390   case AsmToken::Real: {
1391     int64_t IntVal;
1392     if (getParser().parseAbsoluteExpression(IntVal))
1393       return MatchOperand_ParseFail;
1394 
1395     APFloat F(BitsToDouble(IntVal));
1396     if (Minus)
1397       F.changeSign();
1398     Operands.push_back(
1399         AMDGPUOperand::CreateImm(this, F.bitcastToAPInt().getZExtValue(), S,
1400                                  AMDGPUOperand::ImmTyNone, true));
1401     return MatchOperand_Success;
1402   }
1403   default:
1404     return Minus ? MatchOperand_ParseFail : MatchOperand_NoMatch;
1405   }
1406 }
1407 
1408 OperandMatchResultTy
1409 AMDGPUAsmParser::parseRegOrImm(OperandVector &Operands) {
1410   auto res = parseImm(Operands);
1411   if (res != MatchOperand_NoMatch) {
1412     return res;
1413   }
1414 
1415   if (auto R = parseRegister()) {
1416     assert(R->isReg());
1417     R->Reg.IsForcedVOP3 = isForcedVOP3();
1418     Operands.push_back(std::move(R));
1419     return MatchOperand_Success;
1420   }
1421   return MatchOperand_ParseFail;
1422 }
1423 
1424 OperandMatchResultTy
1425 AMDGPUAsmParser::parseRegOrImmWithFPInputMods(OperandVector &Operands) {
1426   // XXX: During parsing we can't determine if minus sign means
1427   // negate-modifier or negative immediate value.
1428   // By default we suppose it is modifier.
1429   bool Negate = false, Abs = false, Abs2 = false;
1430 
1431   if (getLexer().getKind()== AsmToken::Minus) {
1432     Parser.Lex();
1433     Negate = true;
1434   }
1435 
1436   if (getLexer().getKind() == AsmToken::Identifier && Parser.getTok().getString() == "abs") {
1437     Parser.Lex();
1438     Abs2 = true;
1439     if (getLexer().isNot(AsmToken::LParen)) {
1440       Error(Parser.getTok().getLoc(), "expected left paren after abs");
1441       return MatchOperand_ParseFail;
1442     }
1443     Parser.Lex();
1444   }
1445 
1446   if (getLexer().getKind() == AsmToken::Pipe) {
1447     if (Abs2) {
1448       Error(Parser.getTok().getLoc(), "expected register or immediate");
1449       return MatchOperand_ParseFail;
1450     }
1451     Parser.Lex();
1452     Abs = true;
1453   }
1454 
1455   auto Res = parseRegOrImm(Operands);
1456   if (Res != MatchOperand_Success) {
1457     return Res;
1458   }
1459 
1460   AMDGPUOperand::Modifiers Mods;
1461   if (Negate) {
1462     Mods.Neg = true;
1463   }
1464   if (Abs) {
1465     if (getLexer().getKind() != AsmToken::Pipe) {
1466       Error(Parser.getTok().getLoc(), "expected vertical bar");
1467       return MatchOperand_ParseFail;
1468     }
1469     Parser.Lex();
1470     Mods.Abs = true;
1471   }
1472   if (Abs2) {
1473     if (getLexer().isNot(AsmToken::RParen)) {
1474       Error(Parser.getTok().getLoc(), "expected closing parentheses");
1475       return MatchOperand_ParseFail;
1476     }
1477     Parser.Lex();
1478     Mods.Abs = true;
1479   }
1480 
1481   if (Mods.hasFPModifiers()) {
1482     AMDGPUOperand &Op = static_cast<AMDGPUOperand &>(*Operands.back());
1483     Op.setModifiers(Mods);
1484   }
1485   return MatchOperand_Success;
1486 }
1487 
1488 OperandMatchResultTy
1489 AMDGPUAsmParser::parseRegOrImmWithIntInputMods(OperandVector &Operands) {
1490   bool Sext = false;
1491 
1492   if (getLexer().getKind() == AsmToken::Identifier && Parser.getTok().getString() == "sext") {
1493     Parser.Lex();
1494     Sext = true;
1495     if (getLexer().isNot(AsmToken::LParen)) {
1496       Error(Parser.getTok().getLoc(), "expected left paren after sext");
1497       return MatchOperand_ParseFail;
1498     }
1499     Parser.Lex();
1500   }
1501 
1502   auto Res = parseRegOrImm(Operands);
1503   if (Res != MatchOperand_Success) {
1504     return Res;
1505   }
1506 
1507   AMDGPUOperand::Modifiers Mods;
1508   if (Sext) {
1509     if (getLexer().isNot(AsmToken::RParen)) {
1510       Error(Parser.getTok().getLoc(), "expected closing parentheses");
1511       return MatchOperand_ParseFail;
1512     }
1513     Parser.Lex();
1514     Mods.Sext = true;
1515   }
1516 
1517   if (Mods.hasIntModifiers()) {
1518     AMDGPUOperand &Op = static_cast<AMDGPUOperand &>(*Operands.back());
1519     Op.setModifiers(Mods);
1520   }
1521 
1522   return MatchOperand_Success;
1523 }
1524 
1525 OperandMatchResultTy AMDGPUAsmParser::parseVReg32OrOff(OperandVector &Operands) {
1526   std::unique_ptr<AMDGPUOperand> Reg = parseRegister();
1527   if (Reg) {
1528     Operands.push_back(std::move(Reg));
1529     return MatchOperand_Success;
1530   }
1531 
1532   const AsmToken &Tok = Parser.getTok();
1533   if (Tok.getString() == "off") {
1534     Operands.push_back(AMDGPUOperand::CreateImm(this, 0, Tok.getLoc(),
1535                                                 AMDGPUOperand::ImmTyOff, false));
1536     Parser.Lex();
1537     return MatchOperand_Success;
1538   }
1539 
1540   return MatchOperand_NoMatch;
1541 }
1542 
1543 unsigned AMDGPUAsmParser::checkTargetMatchPredicate(MCInst &Inst) {
1544 
1545   uint64_t TSFlags = MII.get(Inst.getOpcode()).TSFlags;
1546 
1547   if ((getForcedEncodingSize() == 32 && (TSFlags & SIInstrFlags::VOP3)) ||
1548       (getForcedEncodingSize() == 64 && !(TSFlags & SIInstrFlags::VOP3)) ||
1549       (isForcedDPP() && !(TSFlags & SIInstrFlags::DPP)) ||
1550       (isForcedSDWA() && !(TSFlags & SIInstrFlags::SDWA)) )
1551     return Match_InvalidOperand;
1552 
1553   if ((TSFlags & SIInstrFlags::VOP3) &&
1554       (TSFlags & SIInstrFlags::VOPAsmPrefer32Bit) &&
1555       getForcedEncodingSize() != 64)
1556     return Match_PreferE32;
1557 
1558   if (Inst.getOpcode() == AMDGPU::V_MAC_F32_sdwa ||
1559       Inst.getOpcode() == AMDGPU::V_MAC_F16_sdwa) {
1560     // v_mac_f32/16 allow only dst_sel == DWORD;
1561     auto OpNum =
1562         AMDGPU::getNamedOperandIdx(Inst.getOpcode(), AMDGPU::OpName::dst_sel);
1563     const auto &Op = Inst.getOperand(OpNum);
1564     if (!Op.isImm() || Op.getImm() != AMDGPU::SDWA::SdwaSel::DWORD) {
1565       return Match_InvalidOperand;
1566     }
1567   }
1568 
1569   return Match_Success;
1570 }
1571 
1572 bool AMDGPUAsmParser::MatchAndEmitInstruction(SMLoc IDLoc, unsigned &Opcode,
1573                                               OperandVector &Operands,
1574                                               MCStreamer &Out,
1575                                               uint64_t &ErrorInfo,
1576                                               bool MatchingInlineAsm) {
1577   // What asm variants we should check
1578   std::vector<unsigned> MatchedVariants;
1579   if (getForcedEncodingSize() == 32) {
1580     MatchedVariants = {AMDGPUAsmVariants::DEFAULT};
1581   } else if (isForcedVOP3()) {
1582     MatchedVariants = {AMDGPUAsmVariants::VOP3};
1583   } else if (isForcedSDWA()) {
1584     MatchedVariants = {AMDGPUAsmVariants::SDWA};
1585   } else if (isForcedDPP()) {
1586     MatchedVariants = {AMDGPUAsmVariants::DPP};
1587   } else {
1588     MatchedVariants = {AMDGPUAsmVariants::DEFAULT,
1589                        AMDGPUAsmVariants::VOP3,
1590                        AMDGPUAsmVariants::SDWA,
1591                        AMDGPUAsmVariants::DPP};
1592   }
1593 
1594   MCInst Inst;
1595   unsigned Result = Match_Success;
1596   for (auto Variant : MatchedVariants) {
1597     uint64_t EI;
1598     auto R = MatchInstructionImpl(Operands, Inst, EI, MatchingInlineAsm,
1599                                   Variant);
1600     // We order match statuses from least to most specific. We use most specific
1601     // status as resulting
1602     // Match_MnemonicFail < Match_InvalidOperand < Match_MissingFeature < Match_PreferE32
1603     if ((R == Match_Success) ||
1604         (R == Match_PreferE32) ||
1605         (R == Match_MissingFeature && Result != Match_PreferE32) ||
1606         (R == Match_InvalidOperand && Result != Match_MissingFeature
1607                                    && Result != Match_PreferE32) ||
1608         (R == Match_MnemonicFail   && Result != Match_InvalidOperand
1609                                    && Result != Match_MissingFeature
1610                                    && Result != Match_PreferE32)) {
1611       Result = R;
1612       ErrorInfo = EI;
1613     }
1614     if (R == Match_Success)
1615       break;
1616   }
1617 
1618   switch (Result) {
1619   default: break;
1620   case Match_Success:
1621     Inst.setLoc(IDLoc);
1622     Out.EmitInstruction(Inst, getSTI());
1623     return false;
1624 
1625   case Match_MissingFeature:
1626     return Error(IDLoc, "instruction not supported on this GPU");
1627 
1628   case Match_MnemonicFail:
1629     return Error(IDLoc, "unrecognized instruction mnemonic");
1630 
1631   case Match_InvalidOperand: {
1632     SMLoc ErrorLoc = IDLoc;
1633     if (ErrorInfo != ~0ULL) {
1634       if (ErrorInfo >= Operands.size()) {
1635         return Error(IDLoc, "too few operands for instruction");
1636       }
1637       ErrorLoc = ((AMDGPUOperand &)*Operands[ErrorInfo]).getStartLoc();
1638       if (ErrorLoc == SMLoc())
1639         ErrorLoc = IDLoc;
1640     }
1641     return Error(ErrorLoc, "invalid operand for instruction");
1642   }
1643 
1644   case Match_PreferE32:
1645     return Error(IDLoc, "internal error: instruction without _e64 suffix "
1646                         "should be encoded as e32");
1647   }
1648   llvm_unreachable("Implement any new match types added!");
1649 }
1650 
1651 bool AMDGPUAsmParser::ParseDirectiveMajorMinor(uint32_t &Major,
1652                                                uint32_t &Minor) {
1653   if (getLexer().isNot(AsmToken::Integer))
1654     return TokError("invalid major version");
1655 
1656   Major = getLexer().getTok().getIntVal();
1657   Lex();
1658 
1659   if (getLexer().isNot(AsmToken::Comma))
1660     return TokError("minor version number required, comma expected");
1661   Lex();
1662 
1663   if (getLexer().isNot(AsmToken::Integer))
1664     return TokError("invalid minor version");
1665 
1666   Minor = getLexer().getTok().getIntVal();
1667   Lex();
1668 
1669   return false;
1670 }
1671 
1672 bool AMDGPUAsmParser::ParseDirectiveHSACodeObjectVersion() {
1673 
1674   uint32_t Major;
1675   uint32_t Minor;
1676 
1677   if (ParseDirectiveMajorMinor(Major, Minor))
1678     return true;
1679 
1680   getTargetStreamer().EmitDirectiveHSACodeObjectVersion(Major, Minor);
1681   return false;
1682 }
1683 
1684 bool AMDGPUAsmParser::ParseDirectiveHSACodeObjectISA() {
1685   uint32_t Major;
1686   uint32_t Minor;
1687   uint32_t Stepping;
1688   StringRef VendorName;
1689   StringRef ArchName;
1690 
1691   // If this directive has no arguments, then use the ISA version for the
1692   // targeted GPU.
1693   if (getLexer().is(AsmToken::EndOfStatement)) {
1694     AMDGPU::IsaVersion Isa = AMDGPU::getIsaVersion(getSTI().getFeatureBits());
1695     getTargetStreamer().EmitDirectiveHSACodeObjectISA(Isa.Major, Isa.Minor,
1696                                                       Isa.Stepping,
1697                                                       "AMD", "AMDGPU");
1698     return false;
1699   }
1700 
1701   if (ParseDirectiveMajorMinor(Major, Minor))
1702     return true;
1703 
1704   if (getLexer().isNot(AsmToken::Comma))
1705     return TokError("stepping version number required, comma expected");
1706   Lex();
1707 
1708   if (getLexer().isNot(AsmToken::Integer))
1709     return TokError("invalid stepping version");
1710 
1711   Stepping = getLexer().getTok().getIntVal();
1712   Lex();
1713 
1714   if (getLexer().isNot(AsmToken::Comma))
1715     return TokError("vendor name required, comma expected");
1716   Lex();
1717 
1718   if (getLexer().isNot(AsmToken::String))
1719     return TokError("invalid vendor name");
1720 
1721   VendorName = getLexer().getTok().getStringContents();
1722   Lex();
1723 
1724   if (getLexer().isNot(AsmToken::Comma))
1725     return TokError("arch name required, comma expected");
1726   Lex();
1727 
1728   if (getLexer().isNot(AsmToken::String))
1729     return TokError("invalid arch name");
1730 
1731   ArchName = getLexer().getTok().getStringContents();
1732   Lex();
1733 
1734   getTargetStreamer().EmitDirectiveHSACodeObjectISA(Major, Minor, Stepping,
1735                                                     VendorName, ArchName);
1736   return false;
1737 }
1738 
1739 bool AMDGPUAsmParser::ParseAMDKernelCodeTValue(StringRef ID,
1740                                                amd_kernel_code_t &Header) {
1741   SmallString<40> ErrStr;
1742   raw_svector_ostream Err(ErrStr);
1743   if (!parseAmdKernelCodeField(ID, getParser(), Header, Err)) {
1744     return TokError(Err.str());
1745   }
1746   Lex();
1747   return false;
1748 }
1749 
1750 bool AMDGPUAsmParser::ParseDirectiveAMDKernelCodeT() {
1751   amd_kernel_code_t Header;
1752   AMDGPU::initDefaultAMDKernelCodeT(Header, getSTI().getFeatureBits());
1753 
1754   while (true) {
1755     // Lex EndOfStatement.  This is in a while loop, because lexing a comment
1756     // will set the current token to EndOfStatement.
1757     while(getLexer().is(AsmToken::EndOfStatement))
1758       Lex();
1759 
1760     if (getLexer().isNot(AsmToken::Identifier))
1761       return TokError("expected value identifier or .end_amd_kernel_code_t");
1762 
1763     StringRef ID = getLexer().getTok().getIdentifier();
1764     Lex();
1765 
1766     if (ID == ".end_amd_kernel_code_t")
1767       break;
1768 
1769     if (ParseAMDKernelCodeTValue(ID, Header))
1770       return true;
1771   }
1772 
1773   getTargetStreamer().EmitAMDKernelCodeT(Header);
1774 
1775   return false;
1776 }
1777 
1778 bool AMDGPUAsmParser::ParseSectionDirectiveHSAText() {
1779   getParser().getStreamer().SwitchSection(
1780       AMDGPU::getHSATextSection(getContext()));
1781   return false;
1782 }
1783 
1784 bool AMDGPUAsmParser::ParseDirectiveAMDGPUHsaKernel() {
1785   if (getLexer().isNot(AsmToken::Identifier))
1786     return TokError("expected symbol name");
1787 
1788   StringRef KernelName = Parser.getTok().getString();
1789 
1790   getTargetStreamer().EmitAMDGPUSymbolType(KernelName,
1791                                            ELF::STT_AMDGPU_HSA_KERNEL);
1792   Lex();
1793   return false;
1794 }
1795 
1796 bool AMDGPUAsmParser::ParseDirectiveAMDGPUHsaModuleGlobal() {
1797   if (getLexer().isNot(AsmToken::Identifier))
1798     return TokError("expected symbol name");
1799 
1800   StringRef GlobalName = Parser.getTok().getIdentifier();
1801 
1802   getTargetStreamer().EmitAMDGPUHsaModuleScopeGlobal(GlobalName);
1803   Lex();
1804   return false;
1805 }
1806 
1807 bool AMDGPUAsmParser::ParseDirectiveAMDGPUHsaProgramGlobal() {
1808   if (getLexer().isNot(AsmToken::Identifier))
1809     return TokError("expected symbol name");
1810 
1811   StringRef GlobalName = Parser.getTok().getIdentifier();
1812 
1813   getTargetStreamer().EmitAMDGPUHsaProgramScopeGlobal(GlobalName);
1814   Lex();
1815   return false;
1816 }
1817 
1818 bool AMDGPUAsmParser::ParseSectionDirectiveHSADataGlobalAgent() {
1819   getParser().getStreamer().SwitchSection(
1820       AMDGPU::getHSADataGlobalAgentSection(getContext()));
1821   return false;
1822 }
1823 
1824 bool AMDGPUAsmParser::ParseSectionDirectiveHSADataGlobalProgram() {
1825   getParser().getStreamer().SwitchSection(
1826       AMDGPU::getHSADataGlobalProgramSection(getContext()));
1827   return false;
1828 }
1829 
1830 bool AMDGPUAsmParser::ParseSectionDirectiveHSARodataReadonlyAgent() {
1831   getParser().getStreamer().SwitchSection(
1832       AMDGPU::getHSARodataReadonlyAgentSection(getContext()));
1833   return false;
1834 }
1835 
1836 bool AMDGPUAsmParser::ParseDirective(AsmToken DirectiveID) {
1837   StringRef IDVal = DirectiveID.getString();
1838 
1839   if (IDVal == ".hsa_code_object_version")
1840     return ParseDirectiveHSACodeObjectVersion();
1841 
1842   if (IDVal == ".hsa_code_object_isa")
1843     return ParseDirectiveHSACodeObjectISA();
1844 
1845   if (IDVal == ".amd_kernel_code_t")
1846     return ParseDirectiveAMDKernelCodeT();
1847 
1848   if (IDVal == ".hsatext")
1849     return ParseSectionDirectiveHSAText();
1850 
1851   if (IDVal == ".amdgpu_hsa_kernel")
1852     return ParseDirectiveAMDGPUHsaKernel();
1853 
1854   if (IDVal == ".amdgpu_hsa_module_global")
1855     return ParseDirectiveAMDGPUHsaModuleGlobal();
1856 
1857   if (IDVal == ".amdgpu_hsa_program_global")
1858     return ParseDirectiveAMDGPUHsaProgramGlobal();
1859 
1860   if (IDVal == ".hsadata_global_agent")
1861     return ParseSectionDirectiveHSADataGlobalAgent();
1862 
1863   if (IDVal == ".hsadata_global_program")
1864     return ParseSectionDirectiveHSADataGlobalProgram();
1865 
1866   if (IDVal == ".hsarodata_readonly_agent")
1867     return ParseSectionDirectiveHSARodataReadonlyAgent();
1868 
1869   return true;
1870 }
1871 
1872 bool AMDGPUAsmParser::subtargetHasRegister(const MCRegisterInfo &MRI,
1873                                            unsigned RegNo) const {
1874   if (isCI())
1875     return true;
1876 
1877   if (isSI()) {
1878     // No flat_scr
1879     switch (RegNo) {
1880     case AMDGPU::FLAT_SCR:
1881     case AMDGPU::FLAT_SCR_LO:
1882     case AMDGPU::FLAT_SCR_HI:
1883       return false;
1884     default:
1885       return true;
1886     }
1887   }
1888 
1889   // VI only has 102 SGPRs, so make sure we aren't trying to use the 2 more that
1890   // SI/CI have.
1891   for (MCRegAliasIterator R(AMDGPU::SGPR102_SGPR103, &MRI, true);
1892        R.isValid(); ++R) {
1893     if (*R == RegNo)
1894       return false;
1895   }
1896 
1897   return true;
1898 }
1899 
1900 OperandMatchResultTy
1901 AMDGPUAsmParser::parseOperand(OperandVector &Operands, StringRef Mnemonic) {
1902 
1903   // Try to parse with a custom parser
1904   OperandMatchResultTy ResTy = MatchOperandParserImpl(Operands, Mnemonic);
1905 
1906   // If we successfully parsed the operand or if there as an error parsing,
1907   // we are done.
1908   //
1909   // If we are parsing after we reach EndOfStatement then this means we
1910   // are appending default values to the Operands list.  This is only done
1911   // by custom parser, so we shouldn't continue on to the generic parsing.
1912   if (ResTy == MatchOperand_Success || ResTy == MatchOperand_ParseFail ||
1913       getLexer().is(AsmToken::EndOfStatement))
1914     return ResTy;
1915 
1916   ResTy = parseRegOrImm(Operands);
1917 
1918   if (ResTy == MatchOperand_Success)
1919     return ResTy;
1920 
1921   if (getLexer().getKind() == AsmToken::Identifier) {
1922     // If this identifier is a symbol, we want to create an expression for it.
1923     // It is a little difficult to distinguish between a symbol name, and
1924     // an instruction flag like 'gds'.  In order to do this, we parse
1925     // all tokens as expressions and then treate the symbol name as the token
1926     // string when we want to interpret the operand as a token.
1927     const auto &Tok = Parser.getTok();
1928     SMLoc S = Tok.getLoc();
1929     const MCExpr *Expr = nullptr;
1930     if (!Parser.parseExpression(Expr)) {
1931       Operands.push_back(AMDGPUOperand::CreateExpr(this, Expr, S));
1932       return MatchOperand_Success;
1933     }
1934 
1935     Operands.push_back(AMDGPUOperand::CreateToken(this, Tok.getString(), Tok.getLoc()));
1936     Parser.Lex();
1937     return MatchOperand_Success;
1938   }
1939   return MatchOperand_NoMatch;
1940 }
1941 
1942 StringRef AMDGPUAsmParser::parseMnemonicSuffix(StringRef Name) {
1943   // Clear any forced encodings from the previous instruction.
1944   setForcedEncodingSize(0);
1945   setForcedDPP(false);
1946   setForcedSDWA(false);
1947 
1948   if (Name.endswith("_e64")) {
1949     setForcedEncodingSize(64);
1950     return Name.substr(0, Name.size() - 4);
1951   } else if (Name.endswith("_e32")) {
1952     setForcedEncodingSize(32);
1953     return Name.substr(0, Name.size() - 4);
1954   } else if (Name.endswith("_dpp")) {
1955     setForcedDPP(true);
1956     return Name.substr(0, Name.size() - 4);
1957   } else if (Name.endswith("_sdwa")) {
1958     setForcedSDWA(true);
1959     return Name.substr(0, Name.size() - 5);
1960   }
1961   return Name;
1962 }
1963 
1964 bool AMDGPUAsmParser::ParseInstruction(ParseInstructionInfo &Info,
1965                                        StringRef Name,
1966                                        SMLoc NameLoc, OperandVector &Operands) {
1967   // Add the instruction mnemonic
1968   Name = parseMnemonicSuffix(Name);
1969   Operands.push_back(AMDGPUOperand::CreateToken(this, Name, NameLoc));
1970 
1971   while (!getLexer().is(AsmToken::EndOfStatement)) {
1972     OperandMatchResultTy Res = parseOperand(Operands, Name);
1973 
1974     // Eat the comma or space if there is one.
1975     if (getLexer().is(AsmToken::Comma))
1976       Parser.Lex();
1977 
1978     switch (Res) {
1979       case MatchOperand_Success: break;
1980       case MatchOperand_ParseFail:
1981         Error(getLexer().getLoc(), "failed parsing operand.");
1982         while (!getLexer().is(AsmToken::EndOfStatement)) {
1983           Parser.Lex();
1984         }
1985         return true;
1986       case MatchOperand_NoMatch:
1987         Error(getLexer().getLoc(), "not a valid operand.");
1988         while (!getLexer().is(AsmToken::EndOfStatement)) {
1989           Parser.Lex();
1990         }
1991         return true;
1992     }
1993   }
1994 
1995   return false;
1996 }
1997 
1998 //===----------------------------------------------------------------------===//
1999 // Utility functions
2000 //===----------------------------------------------------------------------===//
2001 
2002 OperandMatchResultTy
2003 AMDGPUAsmParser::parseIntWithPrefix(const char *Prefix, int64_t &Int) {
2004   switch(getLexer().getKind()) {
2005     default: return MatchOperand_NoMatch;
2006     case AsmToken::Identifier: {
2007       StringRef Name = Parser.getTok().getString();
2008       if (!Name.equals(Prefix)) {
2009         return MatchOperand_NoMatch;
2010       }
2011 
2012       Parser.Lex();
2013       if (getLexer().isNot(AsmToken::Colon))
2014         return MatchOperand_ParseFail;
2015 
2016       Parser.Lex();
2017       if (getLexer().isNot(AsmToken::Integer))
2018         return MatchOperand_ParseFail;
2019 
2020       if (getParser().parseAbsoluteExpression(Int))
2021         return MatchOperand_ParseFail;
2022       break;
2023     }
2024   }
2025   return MatchOperand_Success;
2026 }
2027 
2028 OperandMatchResultTy
2029 AMDGPUAsmParser::parseIntWithPrefix(const char *Prefix, OperandVector &Operands,
2030                                     enum AMDGPUOperand::ImmTy ImmTy,
2031                                     bool (*ConvertResult)(int64_t&)) {
2032   SMLoc S = Parser.getTok().getLoc();
2033   int64_t Value = 0;
2034 
2035   OperandMatchResultTy Res = parseIntWithPrefix(Prefix, Value);
2036   if (Res != MatchOperand_Success)
2037     return Res;
2038 
2039   if (ConvertResult && !ConvertResult(Value)) {
2040     return MatchOperand_ParseFail;
2041   }
2042 
2043   Operands.push_back(AMDGPUOperand::CreateImm(this, Value, S, ImmTy));
2044   return MatchOperand_Success;
2045 }
2046 
2047 OperandMatchResultTy
2048 AMDGPUAsmParser::parseNamedBit(const char *Name, OperandVector &Operands,
2049                                enum AMDGPUOperand::ImmTy ImmTy) {
2050   int64_t Bit = 0;
2051   SMLoc S = Parser.getTok().getLoc();
2052 
2053   // We are at the end of the statement, and this is a default argument, so
2054   // use a default value.
2055   if (getLexer().isNot(AsmToken::EndOfStatement)) {
2056     switch(getLexer().getKind()) {
2057       case AsmToken::Identifier: {
2058         StringRef Tok = Parser.getTok().getString();
2059         if (Tok == Name) {
2060           Bit = 1;
2061           Parser.Lex();
2062         } else if (Tok.startswith("no") && Tok.endswith(Name)) {
2063           Bit = 0;
2064           Parser.Lex();
2065         } else {
2066           return MatchOperand_NoMatch;
2067         }
2068         break;
2069       }
2070       default:
2071         return MatchOperand_NoMatch;
2072     }
2073   }
2074 
2075   Operands.push_back(AMDGPUOperand::CreateImm(this, Bit, S, ImmTy));
2076   return MatchOperand_Success;
2077 }
2078 
2079 typedef std::map<enum AMDGPUOperand::ImmTy, unsigned> OptionalImmIndexMap;
2080 
2081 void addOptionalImmOperand(MCInst& Inst, const OperandVector& Operands,
2082                            OptionalImmIndexMap& OptionalIdx,
2083                            enum AMDGPUOperand::ImmTy ImmT, int64_t Default = 0) {
2084   auto i = OptionalIdx.find(ImmT);
2085   if (i != OptionalIdx.end()) {
2086     unsigned Idx = i->second;
2087     ((AMDGPUOperand &)*Operands[Idx]).addImmOperands(Inst, 1);
2088   } else {
2089     Inst.addOperand(MCOperand::createImm(Default));
2090   }
2091 }
2092 
2093 OperandMatchResultTy
2094 AMDGPUAsmParser::parseStringWithPrefix(StringRef Prefix, StringRef &Value) {
2095   if (getLexer().isNot(AsmToken::Identifier)) {
2096     return MatchOperand_NoMatch;
2097   }
2098   StringRef Tok = Parser.getTok().getString();
2099   if (Tok != Prefix) {
2100     return MatchOperand_NoMatch;
2101   }
2102 
2103   Parser.Lex();
2104   if (getLexer().isNot(AsmToken::Colon)) {
2105     return MatchOperand_ParseFail;
2106   }
2107 
2108   Parser.Lex();
2109   if (getLexer().isNot(AsmToken::Identifier)) {
2110     return MatchOperand_ParseFail;
2111   }
2112 
2113   Value = Parser.getTok().getString();
2114   return MatchOperand_Success;
2115 }
2116 
2117 //===----------------------------------------------------------------------===//
2118 // ds
2119 //===----------------------------------------------------------------------===//
2120 
2121 void AMDGPUAsmParser::cvtDSOffset01(MCInst &Inst,
2122                                     const OperandVector &Operands) {
2123   OptionalImmIndexMap OptionalIdx;
2124 
2125   for (unsigned i = 1, e = Operands.size(); i != e; ++i) {
2126     AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[i]);
2127 
2128     // Add the register arguments
2129     if (Op.isReg()) {
2130       Op.addRegOperands(Inst, 1);
2131       continue;
2132     }
2133 
2134     // Handle optional arguments
2135     OptionalIdx[Op.getImmTy()] = i;
2136   }
2137 
2138   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyOffset0);
2139   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyOffset1);
2140   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyGDS);
2141 
2142   Inst.addOperand(MCOperand::createReg(AMDGPU::M0)); // m0
2143 }
2144 
2145 void AMDGPUAsmParser::cvtDS(MCInst &Inst, const OperandVector &Operands) {
2146   std::map<enum AMDGPUOperand::ImmTy, unsigned> OptionalIdx;
2147   bool GDSOnly = false;
2148 
2149   for (unsigned i = 1, e = Operands.size(); i != e; ++i) {
2150     AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[i]);
2151 
2152     // Add the register arguments
2153     if (Op.isReg()) {
2154       Op.addRegOperands(Inst, 1);
2155       continue;
2156     }
2157 
2158     if (Op.isToken() && Op.getToken() == "gds") {
2159       GDSOnly = true;
2160       continue;
2161     }
2162 
2163     // Handle optional arguments
2164     OptionalIdx[Op.getImmTy()] = i;
2165   }
2166 
2167   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyOffset);
2168   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyGDS);
2169 
2170   if (!GDSOnly) {
2171     addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyGDS);
2172   }
2173   Inst.addOperand(MCOperand::createReg(AMDGPU::M0)); // m0
2174 }
2175 
2176 void AMDGPUAsmParser::cvtExp(MCInst &Inst, const OperandVector &Operands) {
2177   OptionalImmIndexMap OptionalIdx;
2178 
2179   unsigned EnMask = 0;
2180   int SrcIdx = 0;
2181 
2182   for (unsigned i = 1, e = Operands.size(); i != e; ++i) {
2183     AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[i]);
2184 
2185     // Add the register arguments
2186     if (Op.isReg()) {
2187       EnMask |= (1 << SrcIdx);
2188       Op.addRegOperands(Inst, 1);
2189       ++SrcIdx;
2190       continue;
2191     }
2192 
2193     if (Op.isOff()) {
2194       ++SrcIdx;
2195       Inst.addOperand(MCOperand::createReg(AMDGPU::NoRegister));
2196       continue;
2197     }
2198 
2199     if (Op.isImm() && Op.getImmTy() == AMDGPUOperand::ImmTyExpTgt) {
2200       Op.addImmOperands(Inst, 1);
2201       continue;
2202     }
2203 
2204     if (Op.isToken() && Op.getToken() == "done")
2205       continue;
2206 
2207     // Handle optional arguments
2208     OptionalIdx[Op.getImmTy()] = i;
2209   }
2210 
2211   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyExpVM);
2212   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyExpCompr);
2213 
2214   Inst.addOperand(MCOperand::createImm(EnMask));
2215 }
2216 
2217 //===----------------------------------------------------------------------===//
2218 // s_waitcnt
2219 //===----------------------------------------------------------------------===//
2220 
2221 bool AMDGPUAsmParser::parseCnt(int64_t &IntVal) {
2222   StringRef CntName = Parser.getTok().getString();
2223   int64_t CntVal;
2224 
2225   Parser.Lex();
2226   if (getLexer().isNot(AsmToken::LParen))
2227     return true;
2228 
2229   Parser.Lex();
2230   if (getLexer().isNot(AsmToken::Integer))
2231     return true;
2232 
2233   if (getParser().parseAbsoluteExpression(CntVal))
2234     return true;
2235 
2236   if (getLexer().isNot(AsmToken::RParen))
2237     return true;
2238 
2239   Parser.Lex();
2240   if (getLexer().is(AsmToken::Amp) || getLexer().is(AsmToken::Comma))
2241     Parser.Lex();
2242 
2243   IsaVersion IV = getIsaVersion(getSTI().getFeatureBits());
2244   if (CntName == "vmcnt")
2245     IntVal = encodeVmcnt(IV, IntVal, CntVal);
2246   else if (CntName == "expcnt")
2247     IntVal = encodeExpcnt(IV, IntVal, CntVal);
2248   else if (CntName == "lgkmcnt")
2249     IntVal = encodeLgkmcnt(IV, IntVal, CntVal);
2250   else
2251     return true;
2252 
2253   return false;
2254 }
2255 
2256 OperandMatchResultTy
2257 AMDGPUAsmParser::parseSWaitCntOps(OperandVector &Operands) {
2258   IsaVersion IV = getIsaVersion(getSTI().getFeatureBits());
2259   int64_t Waitcnt = getWaitcntBitMask(IV);
2260   SMLoc S = Parser.getTok().getLoc();
2261 
2262   switch(getLexer().getKind()) {
2263     default: return MatchOperand_ParseFail;
2264     case AsmToken::Integer:
2265       // The operand can be an integer value.
2266       if (getParser().parseAbsoluteExpression(Waitcnt))
2267         return MatchOperand_ParseFail;
2268       break;
2269 
2270     case AsmToken::Identifier:
2271       do {
2272         if (parseCnt(Waitcnt))
2273           return MatchOperand_ParseFail;
2274       } while(getLexer().isNot(AsmToken::EndOfStatement));
2275       break;
2276   }
2277   Operands.push_back(AMDGPUOperand::CreateImm(this, Waitcnt, S));
2278   return MatchOperand_Success;
2279 }
2280 
2281 bool AMDGPUAsmParser::parseHwregConstruct(OperandInfoTy &HwReg, int64_t &Offset, int64_t &Width) {
2282   using namespace llvm::AMDGPU::Hwreg;
2283 
2284   if (Parser.getTok().getString() != "hwreg")
2285     return true;
2286   Parser.Lex();
2287 
2288   if (getLexer().isNot(AsmToken::LParen))
2289     return true;
2290   Parser.Lex();
2291 
2292   if (getLexer().is(AsmToken::Identifier)) {
2293     HwReg.IsSymbolic = true;
2294     HwReg.Id = ID_UNKNOWN_;
2295     const StringRef tok = Parser.getTok().getString();
2296     for (int i = ID_SYMBOLIC_FIRST_; i < ID_SYMBOLIC_LAST_; ++i) {
2297       if (tok == IdSymbolic[i]) {
2298         HwReg.Id = i;
2299         break;
2300       }
2301     }
2302     Parser.Lex();
2303   } else {
2304     HwReg.IsSymbolic = false;
2305     if (getLexer().isNot(AsmToken::Integer))
2306       return true;
2307     if (getParser().parseAbsoluteExpression(HwReg.Id))
2308       return true;
2309   }
2310 
2311   if (getLexer().is(AsmToken::RParen)) {
2312     Parser.Lex();
2313     return false;
2314   }
2315 
2316   // optional params
2317   if (getLexer().isNot(AsmToken::Comma))
2318     return true;
2319   Parser.Lex();
2320 
2321   if (getLexer().isNot(AsmToken::Integer))
2322     return true;
2323   if (getParser().parseAbsoluteExpression(Offset))
2324     return true;
2325 
2326   if (getLexer().isNot(AsmToken::Comma))
2327     return true;
2328   Parser.Lex();
2329 
2330   if (getLexer().isNot(AsmToken::Integer))
2331     return true;
2332   if (getParser().parseAbsoluteExpression(Width))
2333     return true;
2334 
2335   if (getLexer().isNot(AsmToken::RParen))
2336     return true;
2337   Parser.Lex();
2338 
2339   return false;
2340 }
2341 
2342 OperandMatchResultTy
2343 AMDGPUAsmParser::parseHwreg(OperandVector &Operands) {
2344   using namespace llvm::AMDGPU::Hwreg;
2345 
2346   int64_t Imm16Val = 0;
2347   SMLoc S = Parser.getTok().getLoc();
2348 
2349   switch(getLexer().getKind()) {
2350     default: return MatchOperand_NoMatch;
2351     case AsmToken::Integer:
2352       // The operand can be an integer value.
2353       if (getParser().parseAbsoluteExpression(Imm16Val))
2354         return MatchOperand_NoMatch;
2355       if (Imm16Val < 0 || !isUInt<16>(Imm16Val)) {
2356         Error(S, "invalid immediate: only 16-bit values are legal");
2357         // Do not return error code, but create an imm operand anyway and proceed
2358         // to the next operand, if any. That avoids unneccessary error messages.
2359       }
2360       break;
2361 
2362     case AsmToken::Identifier: {
2363         OperandInfoTy HwReg(ID_UNKNOWN_);
2364         int64_t Offset = OFFSET_DEFAULT_;
2365         int64_t Width = WIDTH_M1_DEFAULT_ + 1;
2366         if (parseHwregConstruct(HwReg, Offset, Width))
2367           return MatchOperand_ParseFail;
2368         if (HwReg.Id < 0 || !isUInt<ID_WIDTH_>(HwReg.Id)) {
2369           if (HwReg.IsSymbolic)
2370             Error(S, "invalid symbolic name of hardware register");
2371           else
2372             Error(S, "invalid code of hardware register: only 6-bit values are legal");
2373         }
2374         if (Offset < 0 || !isUInt<OFFSET_WIDTH_>(Offset))
2375           Error(S, "invalid bit offset: only 5-bit values are legal");
2376         if ((Width-1) < 0 || !isUInt<WIDTH_M1_WIDTH_>(Width-1))
2377           Error(S, "invalid bitfield width: only values from 1 to 32 are legal");
2378         Imm16Val = (HwReg.Id << ID_SHIFT_) | (Offset << OFFSET_SHIFT_) | ((Width-1) << WIDTH_M1_SHIFT_);
2379       }
2380       break;
2381   }
2382   Operands.push_back(AMDGPUOperand::CreateImm(this, Imm16Val, S, AMDGPUOperand::ImmTyHwreg));
2383   return MatchOperand_Success;
2384 }
2385 
2386 bool AMDGPUOperand::isSWaitCnt() const {
2387   return isImm();
2388 }
2389 
2390 bool AMDGPUOperand::isHwreg() const {
2391   return isImmTy(ImmTyHwreg);
2392 }
2393 
2394 bool AMDGPUAsmParser::parseSendMsgConstruct(OperandInfoTy &Msg, OperandInfoTy &Operation, int64_t &StreamId) {
2395   using namespace llvm::AMDGPU::SendMsg;
2396 
2397   if (Parser.getTok().getString() != "sendmsg")
2398     return true;
2399   Parser.Lex();
2400 
2401   if (getLexer().isNot(AsmToken::LParen))
2402     return true;
2403   Parser.Lex();
2404 
2405   if (getLexer().is(AsmToken::Identifier)) {
2406     Msg.IsSymbolic = true;
2407     Msg.Id = ID_UNKNOWN_;
2408     const std::string tok = Parser.getTok().getString();
2409     for (int i = ID_GAPS_FIRST_; i < ID_GAPS_LAST_; ++i) {
2410       switch(i) {
2411         default: continue; // Omit gaps.
2412         case ID_INTERRUPT: case ID_GS: case ID_GS_DONE:  case ID_SYSMSG: break;
2413       }
2414       if (tok == IdSymbolic[i]) {
2415         Msg.Id = i;
2416         break;
2417       }
2418     }
2419     Parser.Lex();
2420   } else {
2421     Msg.IsSymbolic = false;
2422     if (getLexer().isNot(AsmToken::Integer))
2423       return true;
2424     if (getParser().parseAbsoluteExpression(Msg.Id))
2425       return true;
2426     if (getLexer().is(AsmToken::Integer))
2427       if (getParser().parseAbsoluteExpression(Msg.Id))
2428         Msg.Id = ID_UNKNOWN_;
2429   }
2430   if (Msg.Id == ID_UNKNOWN_) // Don't know how to parse the rest.
2431     return false;
2432 
2433   if (!(Msg.Id == ID_GS || Msg.Id == ID_GS_DONE || Msg.Id == ID_SYSMSG)) {
2434     if (getLexer().isNot(AsmToken::RParen))
2435       return true;
2436     Parser.Lex();
2437     return false;
2438   }
2439 
2440   if (getLexer().isNot(AsmToken::Comma))
2441     return true;
2442   Parser.Lex();
2443 
2444   assert(Msg.Id == ID_GS || Msg.Id == ID_GS_DONE || Msg.Id == ID_SYSMSG);
2445   Operation.Id = ID_UNKNOWN_;
2446   if (getLexer().is(AsmToken::Identifier)) {
2447     Operation.IsSymbolic = true;
2448     const char* const *S = (Msg.Id == ID_SYSMSG) ? OpSysSymbolic : OpGsSymbolic;
2449     const int F = (Msg.Id == ID_SYSMSG) ? OP_SYS_FIRST_ : OP_GS_FIRST_;
2450     const int L = (Msg.Id == ID_SYSMSG) ? OP_SYS_LAST_ : OP_GS_LAST_;
2451     const StringRef Tok = Parser.getTok().getString();
2452     for (int i = F; i < L; ++i) {
2453       if (Tok == S[i]) {
2454         Operation.Id = i;
2455         break;
2456       }
2457     }
2458     Parser.Lex();
2459   } else {
2460     Operation.IsSymbolic = false;
2461     if (getLexer().isNot(AsmToken::Integer))
2462       return true;
2463     if (getParser().parseAbsoluteExpression(Operation.Id))
2464       return true;
2465   }
2466 
2467   if ((Msg.Id == ID_GS || Msg.Id == ID_GS_DONE) && Operation.Id != OP_GS_NOP) {
2468     // Stream id is optional.
2469     if (getLexer().is(AsmToken::RParen)) {
2470       Parser.Lex();
2471       return false;
2472     }
2473 
2474     if (getLexer().isNot(AsmToken::Comma))
2475       return true;
2476     Parser.Lex();
2477 
2478     if (getLexer().isNot(AsmToken::Integer))
2479       return true;
2480     if (getParser().parseAbsoluteExpression(StreamId))
2481       return true;
2482   }
2483 
2484   if (getLexer().isNot(AsmToken::RParen))
2485     return true;
2486   Parser.Lex();
2487   return false;
2488 }
2489 
2490 void AMDGPUAsmParser::errorExpTgt() {
2491   Error(Parser.getTok().getLoc(), "invalid exp target");
2492 }
2493 
2494 OperandMatchResultTy AMDGPUAsmParser::parseExpTgtImpl(StringRef Str,
2495                                                       uint8_t &Val) {
2496   if (Str == "null") {
2497     Val = 9;
2498     return MatchOperand_Success;
2499   }
2500 
2501   if (Str.startswith("mrt")) {
2502     Str = Str.drop_front(3);
2503     if (Str == "z") { // == mrtz
2504       Val = 8;
2505       return MatchOperand_Success;
2506     }
2507 
2508     if (Str.getAsInteger(10, Val))
2509       return MatchOperand_ParseFail;
2510 
2511     if (Val > 7)
2512       errorExpTgt();
2513 
2514     return MatchOperand_Success;
2515   }
2516 
2517   if (Str.startswith("pos")) {
2518     Str = Str.drop_front(3);
2519     if (Str.getAsInteger(10, Val))
2520       return MatchOperand_ParseFail;
2521 
2522     if (Val > 3)
2523       errorExpTgt();
2524 
2525     Val += 12;
2526     return MatchOperand_Success;
2527   }
2528 
2529   if (Str.startswith("param")) {
2530     Str = Str.drop_front(5);
2531     if (Str.getAsInteger(10, Val))
2532       return MatchOperand_ParseFail;
2533 
2534     if (Val >= 32)
2535       errorExpTgt();
2536 
2537     Val += 32;
2538     return MatchOperand_Success;
2539   }
2540 
2541   if (Str.startswith("invalid_target_")) {
2542     Str = Str.drop_front(15);
2543     if (Str.getAsInteger(10, Val))
2544       return MatchOperand_ParseFail;
2545 
2546     errorExpTgt();
2547     return MatchOperand_Success;
2548   }
2549 
2550   return MatchOperand_NoMatch;
2551 }
2552 
2553 OperandMatchResultTy AMDGPUAsmParser::parseExpTgt(OperandVector &Operands) {
2554   uint8_t Val;
2555   StringRef Str = Parser.getTok().getString();
2556 
2557   auto Res = parseExpTgtImpl(Str, Val);
2558   if (Res != MatchOperand_Success)
2559     return Res;
2560 
2561   SMLoc S = Parser.getTok().getLoc();
2562   Parser.Lex();
2563 
2564   Operands.push_back(AMDGPUOperand::CreateImm(this, Val, S,
2565                                               AMDGPUOperand::ImmTyExpTgt));
2566   return MatchOperand_Success;
2567 }
2568 
2569 OperandMatchResultTy
2570 AMDGPUAsmParser::parseSendMsgOp(OperandVector &Operands) {
2571   using namespace llvm::AMDGPU::SendMsg;
2572 
2573   int64_t Imm16Val = 0;
2574   SMLoc S = Parser.getTok().getLoc();
2575 
2576   switch(getLexer().getKind()) {
2577   default:
2578     return MatchOperand_NoMatch;
2579   case AsmToken::Integer:
2580     // The operand can be an integer value.
2581     if (getParser().parseAbsoluteExpression(Imm16Val))
2582       return MatchOperand_NoMatch;
2583     if (Imm16Val < 0 || !isUInt<16>(Imm16Val)) {
2584       Error(S, "invalid immediate: only 16-bit values are legal");
2585       // Do not return error code, but create an imm operand anyway and proceed
2586       // to the next operand, if any. That avoids unneccessary error messages.
2587     }
2588     break;
2589   case AsmToken::Identifier: {
2590       OperandInfoTy Msg(ID_UNKNOWN_);
2591       OperandInfoTy Operation(OP_UNKNOWN_);
2592       int64_t StreamId = STREAM_ID_DEFAULT_;
2593       if (parseSendMsgConstruct(Msg, Operation, StreamId))
2594         return MatchOperand_ParseFail;
2595       do {
2596         // Validate and encode message ID.
2597         if (! ((ID_INTERRUPT <= Msg.Id && Msg.Id <= ID_GS_DONE)
2598                 || Msg.Id == ID_SYSMSG)) {
2599           if (Msg.IsSymbolic)
2600             Error(S, "invalid/unsupported symbolic name of message");
2601           else
2602             Error(S, "invalid/unsupported code of message");
2603           break;
2604         }
2605         Imm16Val = (Msg.Id << ID_SHIFT_);
2606         // Validate and encode operation ID.
2607         if (Msg.Id == ID_GS || Msg.Id == ID_GS_DONE) {
2608           if (! (OP_GS_FIRST_ <= Operation.Id && Operation.Id < OP_GS_LAST_)) {
2609             if (Operation.IsSymbolic)
2610               Error(S, "invalid symbolic name of GS_OP");
2611             else
2612               Error(S, "invalid code of GS_OP: only 2-bit values are legal");
2613             break;
2614           }
2615           if (Operation.Id == OP_GS_NOP
2616               && Msg.Id != ID_GS_DONE) {
2617             Error(S, "invalid GS_OP: NOP is for GS_DONE only");
2618             break;
2619           }
2620           Imm16Val |= (Operation.Id << OP_SHIFT_);
2621         }
2622         if (Msg.Id == ID_SYSMSG) {
2623           if (! (OP_SYS_FIRST_ <= Operation.Id && Operation.Id < OP_SYS_LAST_)) {
2624             if (Operation.IsSymbolic)
2625               Error(S, "invalid/unsupported symbolic name of SYSMSG_OP");
2626             else
2627               Error(S, "invalid/unsupported code of SYSMSG_OP");
2628             break;
2629           }
2630           Imm16Val |= (Operation.Id << OP_SHIFT_);
2631         }
2632         // Validate and encode stream ID.
2633         if ((Msg.Id == ID_GS || Msg.Id == ID_GS_DONE) && Operation.Id != OP_GS_NOP) {
2634           if (! (STREAM_ID_FIRST_ <= StreamId && StreamId < STREAM_ID_LAST_)) {
2635             Error(S, "invalid stream id: only 2-bit values are legal");
2636             break;
2637           }
2638           Imm16Val |= (StreamId << STREAM_ID_SHIFT_);
2639         }
2640       } while (false);
2641     }
2642     break;
2643   }
2644   Operands.push_back(AMDGPUOperand::CreateImm(this, Imm16Val, S, AMDGPUOperand::ImmTySendMsg));
2645   return MatchOperand_Success;
2646 }
2647 
2648 bool AMDGPUOperand::isSendMsg() const {
2649   return isImmTy(ImmTySendMsg);
2650 }
2651 
2652 //===----------------------------------------------------------------------===//
2653 // sopp branch targets
2654 //===----------------------------------------------------------------------===//
2655 
2656 OperandMatchResultTy
2657 AMDGPUAsmParser::parseSOppBrTarget(OperandVector &Operands) {
2658   SMLoc S = Parser.getTok().getLoc();
2659 
2660   switch (getLexer().getKind()) {
2661     default: return MatchOperand_ParseFail;
2662     case AsmToken::Integer: {
2663       int64_t Imm;
2664       if (getParser().parseAbsoluteExpression(Imm))
2665         return MatchOperand_ParseFail;
2666       Operands.push_back(AMDGPUOperand::CreateImm(this, Imm, S));
2667       return MatchOperand_Success;
2668     }
2669 
2670     case AsmToken::Identifier:
2671       Operands.push_back(AMDGPUOperand::CreateExpr(this,
2672           MCSymbolRefExpr::create(getContext().getOrCreateSymbol(
2673                                   Parser.getTok().getString()), getContext()), S));
2674       Parser.Lex();
2675       return MatchOperand_Success;
2676   }
2677 }
2678 
2679 //===----------------------------------------------------------------------===//
2680 // mubuf
2681 //===----------------------------------------------------------------------===//
2682 
2683 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultGLC() const {
2684   return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTyGLC);
2685 }
2686 
2687 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultSLC() const {
2688   return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTySLC);
2689 }
2690 
2691 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultTFE() const {
2692   return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTyTFE);
2693 }
2694 
2695 void AMDGPUAsmParser::cvtMubufImpl(MCInst &Inst,
2696                                const OperandVector &Operands,
2697                                bool IsAtomic, bool IsAtomicReturn) {
2698   OptionalImmIndexMap OptionalIdx;
2699   assert(IsAtomicReturn ? IsAtomic : true);
2700 
2701   for (unsigned i = 1, e = Operands.size(); i != e; ++i) {
2702     AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[i]);
2703 
2704     // Add the register arguments
2705     if (Op.isReg()) {
2706       Op.addRegOperands(Inst, 1);
2707       continue;
2708     }
2709 
2710     // Handle the case where soffset is an immediate
2711     if (Op.isImm() && Op.getImmTy() == AMDGPUOperand::ImmTyNone) {
2712       Op.addImmOperands(Inst, 1);
2713       continue;
2714     }
2715 
2716     // Handle tokens like 'offen' which are sometimes hard-coded into the
2717     // asm string.  There are no MCInst operands for these.
2718     if (Op.isToken()) {
2719       continue;
2720     }
2721     assert(Op.isImm());
2722 
2723     // Handle optional arguments
2724     OptionalIdx[Op.getImmTy()] = i;
2725   }
2726 
2727   // Copy $vdata_in operand and insert as $vdata for MUBUF_Atomic RTN insns.
2728   if (IsAtomicReturn) {
2729     MCInst::iterator I = Inst.begin(); // $vdata_in is always at the beginning.
2730     Inst.insert(I, *I);
2731   }
2732 
2733   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyOffset);
2734   if (!IsAtomic) { // glc is hard-coded.
2735     addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyGLC);
2736   }
2737   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySLC);
2738   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyTFE);
2739 }
2740 
2741 //===----------------------------------------------------------------------===//
2742 // mimg
2743 //===----------------------------------------------------------------------===//
2744 
2745 void AMDGPUAsmParser::cvtMIMG(MCInst &Inst, const OperandVector &Operands) {
2746   unsigned I = 1;
2747   const MCInstrDesc &Desc = MII.get(Inst.getOpcode());
2748   for (unsigned J = 0; J < Desc.getNumDefs(); ++J) {
2749     ((AMDGPUOperand &)*Operands[I++]).addRegOperands(Inst, 1);
2750   }
2751 
2752   OptionalImmIndexMap OptionalIdx;
2753 
2754   for (unsigned E = Operands.size(); I != E; ++I) {
2755     AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[I]);
2756 
2757     // Add the register arguments
2758     if (Op.isRegOrImm()) {
2759       Op.addRegOrImmOperands(Inst, 1);
2760       continue;
2761     } else if (Op.isImmModifier()) {
2762       OptionalIdx[Op.getImmTy()] = I;
2763     } else {
2764       llvm_unreachable("unexpected operand type");
2765     }
2766   }
2767 
2768   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyDMask);
2769   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyUNorm);
2770   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyGLC);
2771   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyDA);
2772   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyR128);
2773   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyTFE);
2774   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyLWE);
2775   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySLC);
2776 }
2777 
2778 void AMDGPUAsmParser::cvtMIMGAtomic(MCInst &Inst, const OperandVector &Operands) {
2779   unsigned I = 1;
2780   const MCInstrDesc &Desc = MII.get(Inst.getOpcode());
2781   for (unsigned J = 0; J < Desc.getNumDefs(); ++J) {
2782     ((AMDGPUOperand &)*Operands[I++]).addRegOperands(Inst, 1);
2783   }
2784 
2785   // Add src, same as dst
2786   ((AMDGPUOperand &)*Operands[I]).addRegOperands(Inst, 1);
2787 
2788   OptionalImmIndexMap OptionalIdx;
2789 
2790   for (unsigned E = Operands.size(); I != E; ++I) {
2791     AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[I]);
2792 
2793     // Add the register arguments
2794     if (Op.isRegOrImm()) {
2795       Op.addRegOrImmOperands(Inst, 1);
2796       continue;
2797     } else if (Op.isImmModifier()) {
2798       OptionalIdx[Op.getImmTy()] = I;
2799     } else {
2800       llvm_unreachable("unexpected operand type");
2801     }
2802   }
2803 
2804   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyDMask);
2805   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyUNorm);
2806   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyGLC);
2807   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyDA);
2808   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyR128);
2809   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyTFE);
2810   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyLWE);
2811   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySLC);
2812 }
2813 
2814 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultDMask() const {
2815   return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTyDMask);
2816 }
2817 
2818 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultUNorm() const {
2819   return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTyUNorm);
2820 }
2821 
2822 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultDA() const {
2823   return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTyDA);
2824 }
2825 
2826 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultR128() const {
2827   return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTyR128);
2828 }
2829 
2830 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultLWE() const {
2831   return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTyLWE);
2832 }
2833 
2834 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultExpTgt() const {
2835   return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTyExpTgt);
2836 }
2837 
2838 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultExpCompr() const {
2839   return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTyExpCompr);
2840 }
2841 
2842 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultExpVM() const {
2843   return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTyExpVM);
2844 }
2845 
2846 //===----------------------------------------------------------------------===//
2847 // smrd
2848 //===----------------------------------------------------------------------===//
2849 
2850 bool AMDGPUOperand::isSMRDOffset8() const {
2851   return isImm() && isUInt<8>(getImm());
2852 }
2853 
2854 bool AMDGPUOperand::isSMRDOffset20() const {
2855   return isImm() && isUInt<20>(getImm());
2856 }
2857 
2858 bool AMDGPUOperand::isSMRDLiteralOffset() const {
2859   // 32-bit literals are only supported on CI and we only want to use them
2860   // when the offset is > 8-bits.
2861   return isImm() && !isUInt<8>(getImm()) && isUInt<32>(getImm());
2862 }
2863 
2864 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultSMRDOffset8() const {
2865   return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTyOffset);
2866 }
2867 
2868 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultSMRDOffset20() const {
2869   return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTyOffset);
2870 }
2871 
2872 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultSMRDLiteralOffset() const {
2873   return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTyOffset);
2874 }
2875 
2876 //===----------------------------------------------------------------------===//
2877 // vop3
2878 //===----------------------------------------------------------------------===//
2879 
2880 static bool ConvertOmodMul(int64_t &Mul) {
2881   if (Mul != 1 && Mul != 2 && Mul != 4)
2882     return false;
2883 
2884   Mul >>= 1;
2885   return true;
2886 }
2887 
2888 static bool ConvertOmodDiv(int64_t &Div) {
2889   if (Div == 1) {
2890     Div = 0;
2891     return true;
2892   }
2893 
2894   if (Div == 2) {
2895     Div = 3;
2896     return true;
2897   }
2898 
2899   return false;
2900 }
2901 
2902 static bool ConvertBoundCtrl(int64_t &BoundCtrl) {
2903   if (BoundCtrl == 0) {
2904     BoundCtrl = 1;
2905     return true;
2906   }
2907 
2908   if (BoundCtrl == -1) {
2909     BoundCtrl = 0;
2910     return true;
2911   }
2912 
2913   return false;
2914 }
2915 
2916 // Note: the order in this table matches the order of operands in AsmString.
2917 static const OptionalOperand AMDGPUOptionalOperandTable[] = {
2918   {"offen",   AMDGPUOperand::ImmTyOffen, true, nullptr},
2919   {"idxen",   AMDGPUOperand::ImmTyIdxen, true, nullptr},
2920   {"addr64",  AMDGPUOperand::ImmTyAddr64, true, nullptr},
2921   {"offset0", AMDGPUOperand::ImmTyOffset0, false, nullptr},
2922   {"offset1", AMDGPUOperand::ImmTyOffset1, false, nullptr},
2923   {"gds",     AMDGPUOperand::ImmTyGDS, true, nullptr},
2924   {"offset",  AMDGPUOperand::ImmTyOffset, false, nullptr},
2925   {"glc",     AMDGPUOperand::ImmTyGLC, true, nullptr},
2926   {"slc",     AMDGPUOperand::ImmTySLC, true, nullptr},
2927   {"tfe",     AMDGPUOperand::ImmTyTFE, true, nullptr},
2928   {"clamp",   AMDGPUOperand::ImmTyClampSI, true, nullptr},
2929   {"omod",    AMDGPUOperand::ImmTyOModSI, false, ConvertOmodMul},
2930   {"unorm",   AMDGPUOperand::ImmTyUNorm, true, nullptr},
2931   {"da",      AMDGPUOperand::ImmTyDA,    true, nullptr},
2932   {"r128",    AMDGPUOperand::ImmTyR128,  true, nullptr},
2933   {"lwe",     AMDGPUOperand::ImmTyLWE,   true, nullptr},
2934   {"dmask",   AMDGPUOperand::ImmTyDMask, false, nullptr},
2935   {"row_mask",   AMDGPUOperand::ImmTyDppRowMask, false, nullptr},
2936   {"bank_mask",  AMDGPUOperand::ImmTyDppBankMask, false, nullptr},
2937   {"bound_ctrl", AMDGPUOperand::ImmTyDppBoundCtrl, false, ConvertBoundCtrl},
2938   {"dst_sel",    AMDGPUOperand::ImmTySdwaDstSel, false, nullptr},
2939   {"src0_sel",   AMDGPUOperand::ImmTySdwaSrc0Sel, false, nullptr},
2940   {"src1_sel",   AMDGPUOperand::ImmTySdwaSrc1Sel, false, nullptr},
2941   {"dst_unused", AMDGPUOperand::ImmTySdwaDstUnused, false, nullptr},
2942   {"vm", AMDGPUOperand::ImmTyExpVM, true, nullptr},
2943 };
2944 
2945 OperandMatchResultTy AMDGPUAsmParser::parseOptionalOperand(OperandVector &Operands) {
2946   OperandMatchResultTy res;
2947   for (const OptionalOperand &Op : AMDGPUOptionalOperandTable) {
2948     // try to parse any optional operand here
2949     if (Op.IsBit) {
2950       res = parseNamedBit(Op.Name, Operands, Op.Type);
2951     } else if (Op.Type == AMDGPUOperand::ImmTyOModSI) {
2952       res = parseOModOperand(Operands);
2953     } else if (Op.Type == AMDGPUOperand::ImmTySdwaDstSel ||
2954                Op.Type == AMDGPUOperand::ImmTySdwaSrc0Sel ||
2955                Op.Type == AMDGPUOperand::ImmTySdwaSrc1Sel) {
2956       res = parseSDWASel(Operands, Op.Name, Op.Type);
2957     } else if (Op.Type == AMDGPUOperand::ImmTySdwaDstUnused) {
2958       res = parseSDWADstUnused(Operands);
2959     } else {
2960       res = parseIntWithPrefix(Op.Name, Operands, Op.Type, Op.ConvertResult);
2961     }
2962     if (res != MatchOperand_NoMatch) {
2963       return res;
2964     }
2965   }
2966   return MatchOperand_NoMatch;
2967 }
2968 
2969 OperandMatchResultTy AMDGPUAsmParser::parseOModOperand(OperandVector &Operands) {
2970   StringRef Name = Parser.getTok().getString();
2971   if (Name == "mul") {
2972     return parseIntWithPrefix("mul", Operands,
2973                               AMDGPUOperand::ImmTyOModSI, ConvertOmodMul);
2974   }
2975 
2976   if (Name == "div") {
2977     return parseIntWithPrefix("div", Operands,
2978                               AMDGPUOperand::ImmTyOModSI, ConvertOmodDiv);
2979   }
2980 
2981   return MatchOperand_NoMatch;
2982 }
2983 
2984 void AMDGPUAsmParser::cvtId(MCInst &Inst, const OperandVector &Operands) {
2985   unsigned I = 1;
2986   const MCInstrDesc &Desc = MII.get(Inst.getOpcode());
2987   for (unsigned J = 0; J < Desc.getNumDefs(); ++J) {
2988     ((AMDGPUOperand &)*Operands[I++]).addRegOperands(Inst, 1);
2989   }
2990   for (unsigned E = Operands.size(); I != E; ++I)
2991     ((AMDGPUOperand &)*Operands[I]).addRegOrImmOperands(Inst, 1);
2992 }
2993 
2994 void AMDGPUAsmParser::cvtVOP3_2_mod(MCInst &Inst, const OperandVector &Operands) {
2995   uint64_t TSFlags = MII.get(Inst.getOpcode()).TSFlags;
2996   if (TSFlags & SIInstrFlags::VOP3) {
2997     cvtVOP3(Inst, Operands);
2998   } else {
2999     cvtId(Inst, Operands);
3000   }
3001 }
3002 
3003 static bool isRegOrImmWithInputMods(const MCInstrDesc &Desc, unsigned OpNum) {
3004       // 1. This operand is input modifiers
3005   return Desc.OpInfo[OpNum].OperandType == AMDGPU::OPERAND_INPUT_MODS
3006       // 2. This is not last operand
3007       && Desc.NumOperands > (OpNum + 1)
3008       // 3. Next operand is register class
3009       && Desc.OpInfo[OpNum + 1].RegClass != -1
3010       // 4. Next register is not tied to any other operand
3011       && Desc.getOperandConstraint(OpNum + 1, MCOI::OperandConstraint::TIED_TO) == -1;
3012 }
3013 
3014 void AMDGPUAsmParser::cvtVOP3(MCInst &Inst, const OperandVector &Operands) {
3015   OptionalImmIndexMap OptionalIdx;
3016   unsigned I = 1;
3017   const MCInstrDesc &Desc = MII.get(Inst.getOpcode());
3018   for (unsigned J = 0; J < Desc.getNumDefs(); ++J) {
3019     ((AMDGPUOperand &)*Operands[I++]).addRegOperands(Inst, 1);
3020   }
3021 
3022   for (unsigned E = Operands.size(); I != E; ++I) {
3023     AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[I]);
3024     if (isRegOrImmWithInputMods(Desc, Inst.getNumOperands())) {
3025       Op.addRegOrImmWithFPInputModsOperands(Inst, 2);
3026     } else if (Op.isImm()) {
3027       OptionalIdx[Op.getImmTy()] = I;
3028     } else {
3029       llvm_unreachable("unhandled operand type");
3030     }
3031   }
3032 
3033   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyClampSI);
3034   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyOModSI);
3035 
3036   // special case v_mac_{f16, f32}:
3037   // it has src2 register operand that is tied to dst operand
3038   // we don't allow modifiers for this operand in assembler so src2_modifiers
3039   // should be 0
3040   if (Inst.getOpcode() == AMDGPU::V_MAC_F32_e64_si ||
3041       Inst.getOpcode() == AMDGPU::V_MAC_F32_e64_vi ||
3042       Inst.getOpcode() == AMDGPU::V_MAC_F16_e64_vi) {
3043     auto it = Inst.begin();
3044     std::advance(
3045       it,
3046       AMDGPU::getNamedOperandIdx(Inst.getOpcode() == AMDGPU::V_MAC_F16_e64_vi ?
3047                                      AMDGPU::V_MAC_F16_e64 :
3048                                      AMDGPU::V_MAC_F32_e64,
3049                                  AMDGPU::OpName::src2_modifiers));
3050     it = Inst.insert(it, MCOperand::createImm(0)); // no modifiers for src2
3051     ++it;
3052     Inst.insert(it, Inst.getOperand(0)); // src2 = dst
3053   }
3054 }
3055 
3056 //===----------------------------------------------------------------------===//
3057 // dpp
3058 //===----------------------------------------------------------------------===//
3059 
3060 bool AMDGPUOperand::isDPPCtrl() const {
3061   bool result = isImm() && getImmTy() == ImmTyDppCtrl && isUInt<9>(getImm());
3062   if (result) {
3063     int64_t Imm = getImm();
3064     return ((Imm >= 0x000) && (Imm <= 0x0ff)) ||
3065            ((Imm >= 0x101) && (Imm <= 0x10f)) ||
3066            ((Imm >= 0x111) && (Imm <= 0x11f)) ||
3067            ((Imm >= 0x121) && (Imm <= 0x12f)) ||
3068            (Imm == 0x130) ||
3069            (Imm == 0x134) ||
3070            (Imm == 0x138) ||
3071            (Imm == 0x13c) ||
3072            (Imm == 0x140) ||
3073            (Imm == 0x141) ||
3074            (Imm == 0x142) ||
3075            (Imm == 0x143);
3076   }
3077   return false;
3078 }
3079 
3080 bool AMDGPUOperand::isGPRIdxMode() const {
3081   return isImm() && isUInt<4>(getImm());
3082 }
3083 
3084 OperandMatchResultTy
3085 AMDGPUAsmParser::parseDPPCtrl(OperandVector &Operands) {
3086   SMLoc S = Parser.getTok().getLoc();
3087   StringRef Prefix;
3088   int64_t Int;
3089 
3090   if (getLexer().getKind() == AsmToken::Identifier) {
3091     Prefix = Parser.getTok().getString();
3092   } else {
3093     return MatchOperand_NoMatch;
3094   }
3095 
3096   if (Prefix == "row_mirror") {
3097     Int = 0x140;
3098     Parser.Lex();
3099   } else if (Prefix == "row_half_mirror") {
3100     Int = 0x141;
3101     Parser.Lex();
3102   } else {
3103     // Check to prevent parseDPPCtrlOps from eating invalid tokens
3104     if (Prefix != "quad_perm"
3105         && Prefix != "row_shl"
3106         && Prefix != "row_shr"
3107         && Prefix != "row_ror"
3108         && Prefix != "wave_shl"
3109         && Prefix != "wave_rol"
3110         && Prefix != "wave_shr"
3111         && Prefix != "wave_ror"
3112         && Prefix != "row_bcast") {
3113       return MatchOperand_NoMatch;
3114     }
3115 
3116     Parser.Lex();
3117     if (getLexer().isNot(AsmToken::Colon))
3118       return MatchOperand_ParseFail;
3119 
3120     if (Prefix == "quad_perm") {
3121       // quad_perm:[%d,%d,%d,%d]
3122       Parser.Lex();
3123       if (getLexer().isNot(AsmToken::LBrac))
3124         return MatchOperand_ParseFail;
3125       Parser.Lex();
3126 
3127       if (getParser().parseAbsoluteExpression(Int) || !(0 <= Int && Int <=3))
3128         return MatchOperand_ParseFail;
3129 
3130       for (int i = 0; i < 3; ++i) {
3131         if (getLexer().isNot(AsmToken::Comma))
3132           return MatchOperand_ParseFail;
3133         Parser.Lex();
3134 
3135         int64_t Temp;
3136         if (getParser().parseAbsoluteExpression(Temp) || !(0 <= Temp && Temp <=3))
3137           return MatchOperand_ParseFail;
3138         const int shift = i*2 + 2;
3139         Int += (Temp << shift);
3140       }
3141 
3142       if (getLexer().isNot(AsmToken::RBrac))
3143         return MatchOperand_ParseFail;
3144       Parser.Lex();
3145 
3146     } else {
3147       // sel:%d
3148       Parser.Lex();
3149       if (getParser().parseAbsoluteExpression(Int))
3150         return MatchOperand_ParseFail;
3151 
3152       if (Prefix == "row_shl" && 1 <= Int && Int <= 15) {
3153         Int |= 0x100;
3154       } else if (Prefix == "row_shr" && 1 <= Int && Int <= 15) {
3155         Int |= 0x110;
3156       } else if (Prefix == "row_ror" && 1 <= Int && Int <= 15) {
3157         Int |= 0x120;
3158       } else if (Prefix == "wave_shl" && 1 == Int) {
3159         Int = 0x130;
3160       } else if (Prefix == "wave_rol" && 1 == Int) {
3161         Int = 0x134;
3162       } else if (Prefix == "wave_shr" && 1 == Int) {
3163         Int = 0x138;
3164       } else if (Prefix == "wave_ror" && 1 == Int) {
3165         Int = 0x13C;
3166       } else if (Prefix == "row_bcast") {
3167         if (Int == 15) {
3168           Int = 0x142;
3169         } else if (Int == 31) {
3170           Int = 0x143;
3171         } else {
3172           return MatchOperand_ParseFail;
3173         }
3174       } else {
3175         return MatchOperand_ParseFail;
3176       }
3177     }
3178   }
3179 
3180   Operands.push_back(AMDGPUOperand::CreateImm(this, Int, S, AMDGPUOperand::ImmTyDppCtrl));
3181   return MatchOperand_Success;
3182 }
3183 
3184 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultRowMask() const {
3185   return AMDGPUOperand::CreateImm(this, 0xf, SMLoc(), AMDGPUOperand::ImmTyDppRowMask);
3186 }
3187 
3188 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultBankMask() const {
3189   return AMDGPUOperand::CreateImm(this, 0xf, SMLoc(), AMDGPUOperand::ImmTyDppBankMask);
3190 }
3191 
3192 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultBoundCtrl() const {
3193   return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTyDppBoundCtrl);
3194 }
3195 
3196 void AMDGPUAsmParser::cvtDPP(MCInst &Inst, const OperandVector &Operands) {
3197   OptionalImmIndexMap OptionalIdx;
3198 
3199   unsigned I = 1;
3200   const MCInstrDesc &Desc = MII.get(Inst.getOpcode());
3201   for (unsigned J = 0; J < Desc.getNumDefs(); ++J) {
3202     ((AMDGPUOperand &)*Operands[I++]).addRegOperands(Inst, 1);
3203   }
3204 
3205   for (unsigned E = Operands.size(); I != E; ++I) {
3206     AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[I]);
3207     // Add the register arguments
3208     if (isRegOrImmWithInputMods(Desc, Inst.getNumOperands())) {
3209       Op.addRegOrImmWithFPInputModsOperands(Inst, 2);
3210     } else if (Op.isDPPCtrl()) {
3211       Op.addImmOperands(Inst, 1);
3212     } else if (Op.isImm()) {
3213       // Handle optional arguments
3214       OptionalIdx[Op.getImmTy()] = I;
3215     } else {
3216       llvm_unreachable("Invalid operand type");
3217     }
3218   }
3219 
3220   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyDppRowMask, 0xf);
3221   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyDppBankMask, 0xf);
3222   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyDppBoundCtrl);
3223 
3224   // special case v_mac_{f16, f32}:
3225   // it has src2 register operand that is tied to dst operand
3226   if (Inst.getOpcode() == AMDGPU::V_MAC_F32_dpp ||
3227       Inst.getOpcode() == AMDGPU::V_MAC_F16_dpp) {
3228     auto it = Inst.begin();
3229     std::advance(
3230         it, AMDGPU::getNamedOperandIdx(Inst.getOpcode(), AMDGPU::OpName::src2));
3231     Inst.insert(it, Inst.getOperand(0)); // src2 = dst
3232   }
3233 }
3234 
3235 //===----------------------------------------------------------------------===//
3236 // sdwa
3237 //===----------------------------------------------------------------------===//
3238 
3239 OperandMatchResultTy
3240 AMDGPUAsmParser::parseSDWASel(OperandVector &Operands, StringRef Prefix,
3241                               AMDGPUOperand::ImmTy Type) {
3242   using namespace llvm::AMDGPU::SDWA;
3243 
3244   SMLoc S = Parser.getTok().getLoc();
3245   StringRef Value;
3246   OperandMatchResultTy res;
3247 
3248   res = parseStringWithPrefix(Prefix, Value);
3249   if (res != MatchOperand_Success) {
3250     return res;
3251   }
3252 
3253   int64_t Int;
3254   Int = StringSwitch<int64_t>(Value)
3255         .Case("BYTE_0", SdwaSel::BYTE_0)
3256         .Case("BYTE_1", SdwaSel::BYTE_1)
3257         .Case("BYTE_2", SdwaSel::BYTE_2)
3258         .Case("BYTE_3", SdwaSel::BYTE_3)
3259         .Case("WORD_0", SdwaSel::WORD_0)
3260         .Case("WORD_1", SdwaSel::WORD_1)
3261         .Case("DWORD", SdwaSel::DWORD)
3262         .Default(0xffffffff);
3263   Parser.Lex(); // eat last token
3264 
3265   if (Int == 0xffffffff) {
3266     return MatchOperand_ParseFail;
3267   }
3268 
3269   Operands.push_back(AMDGPUOperand::CreateImm(this, Int, S, Type));
3270   return MatchOperand_Success;
3271 }
3272 
3273 OperandMatchResultTy
3274 AMDGPUAsmParser::parseSDWADstUnused(OperandVector &Operands) {
3275   using namespace llvm::AMDGPU::SDWA;
3276 
3277   SMLoc S = Parser.getTok().getLoc();
3278   StringRef Value;
3279   OperandMatchResultTy res;
3280 
3281   res = parseStringWithPrefix("dst_unused", Value);
3282   if (res != MatchOperand_Success) {
3283     return res;
3284   }
3285 
3286   int64_t Int;
3287   Int = StringSwitch<int64_t>(Value)
3288         .Case("UNUSED_PAD", DstUnused::UNUSED_PAD)
3289         .Case("UNUSED_SEXT", DstUnused::UNUSED_SEXT)
3290         .Case("UNUSED_PRESERVE", DstUnused::UNUSED_PRESERVE)
3291         .Default(0xffffffff);
3292   Parser.Lex(); // eat last token
3293 
3294   if (Int == 0xffffffff) {
3295     return MatchOperand_ParseFail;
3296   }
3297 
3298   Operands.push_back(AMDGPUOperand::CreateImm(this, Int, S, AMDGPUOperand::ImmTySdwaDstUnused));
3299   return MatchOperand_Success;
3300 }
3301 
3302 void AMDGPUAsmParser::cvtSdwaVOP1(MCInst &Inst, const OperandVector &Operands) {
3303   cvtSDWA(Inst, Operands, SIInstrFlags::VOP1);
3304 }
3305 
3306 void AMDGPUAsmParser::cvtSdwaVOP2(MCInst &Inst, const OperandVector &Operands) {
3307   cvtSDWA(Inst, Operands, SIInstrFlags::VOP2);
3308 }
3309 
3310 void AMDGPUAsmParser::cvtSdwaVOPC(MCInst &Inst, const OperandVector &Operands) {
3311   cvtSDWA(Inst, Operands, SIInstrFlags::VOPC);
3312 }
3313 
3314 void AMDGPUAsmParser::cvtSDWA(MCInst &Inst, const OperandVector &Operands,
3315                               uint64_t BasicInstType) {
3316   OptionalImmIndexMap OptionalIdx;
3317 
3318   unsigned I = 1;
3319   const MCInstrDesc &Desc = MII.get(Inst.getOpcode());
3320   for (unsigned J = 0; J < Desc.getNumDefs(); ++J) {
3321     ((AMDGPUOperand &)*Operands[I++]).addRegOperands(Inst, 1);
3322   }
3323 
3324   for (unsigned E = Operands.size(); I != E; ++I) {
3325     AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[I]);
3326     // Add the register arguments
3327     if (BasicInstType == SIInstrFlags::VOPC &&
3328         Op.isReg() &&
3329         Op.Reg.RegNo == AMDGPU::VCC) {
3330       // VOPC sdwa use "vcc" token as dst. Skip it.
3331       continue;
3332     } else if (isRegOrImmWithInputMods(Desc, Inst.getNumOperands())) {
3333       Op.addRegOrImmWithInputModsOperands(Inst, 2);
3334     } else if (Op.isImm()) {
3335       // Handle optional arguments
3336       OptionalIdx[Op.getImmTy()] = I;
3337     } else {
3338       llvm_unreachable("Invalid operand type");
3339     }
3340   }
3341 
3342   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyClampSI, 0);
3343 
3344   if (Inst.getOpcode() != AMDGPU::V_NOP_sdwa) {
3345     // V_NOP_sdwa has no optional sdwa arguments
3346     switch (BasicInstType) {
3347     case SIInstrFlags::VOP1:
3348       addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySdwaDstSel, 6);
3349       addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySdwaDstUnused, 2);
3350       addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySdwaSrc0Sel, 6);
3351       break;
3352 
3353     case SIInstrFlags::VOP2:
3354       addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySdwaDstSel, 6);
3355       addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySdwaDstUnused, 2);
3356       addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySdwaSrc0Sel, 6);
3357       addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySdwaSrc1Sel, 6);
3358       break;
3359 
3360     case SIInstrFlags::VOPC:
3361       addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySdwaSrc0Sel, 6);
3362       addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySdwaSrc1Sel, 6);
3363       break;
3364 
3365     default:
3366       llvm_unreachable("Invalid instruction type. Only VOP1, VOP2 and VOPC allowed");
3367     }
3368   }
3369 
3370   // special case v_mac_{f16, f32}:
3371   // it has src2 register operand that is tied to dst operand
3372   if (Inst.getOpcode() == AMDGPU::V_MAC_F32_sdwa ||
3373       Inst.getOpcode() == AMDGPU::V_MAC_F16_sdwa)  {
3374     auto it = Inst.begin();
3375     std::advance(
3376         it, AMDGPU::getNamedOperandIdx(Inst.getOpcode(), AMDGPU::OpName::src2));
3377     Inst.insert(it, Inst.getOperand(0)); // src2 = dst
3378   }
3379 
3380 }
3381 
3382 /// Force static initialization.
3383 extern "C" void LLVMInitializeAMDGPUAsmParser() {
3384   RegisterMCAsmParser<AMDGPUAsmParser> A(getTheAMDGPUTarget());
3385   RegisterMCAsmParser<AMDGPUAsmParser> B(getTheGCNTarget());
3386 }
3387 
3388 #define GET_REGISTER_MATCHER
3389 #define GET_MATCHER_IMPLEMENTATION
3390 #include "AMDGPUGenAsmMatcher.inc"
3391 
3392 // This fuction should be defined after auto-generated include so that we have
3393 // MatchClassKind enum defined
3394 unsigned AMDGPUAsmParser::validateTargetOperandClass(MCParsedAsmOperand &Op,
3395                                                      unsigned Kind) {
3396   // Tokens like "glc" would be parsed as immediate operands in ParseOperand().
3397   // But MatchInstructionImpl() expects to meet token and fails to validate
3398   // operand. This method checks if we are given immediate operand but expect to
3399   // get corresponding token.
3400   AMDGPUOperand &Operand = (AMDGPUOperand&)Op;
3401   switch (Kind) {
3402   case MCK_addr64:
3403     return Operand.isAddr64() ? Match_Success : Match_InvalidOperand;
3404   case MCK_gds:
3405     return Operand.isGDS() ? Match_Success : Match_InvalidOperand;
3406   case MCK_glc:
3407     return Operand.isGLC() ? Match_Success : Match_InvalidOperand;
3408   case MCK_idxen:
3409     return Operand.isIdxen() ? Match_Success : Match_InvalidOperand;
3410   case MCK_offen:
3411     return Operand.isOffen() ? Match_Success : Match_InvalidOperand;
3412   case MCK_SSrcB32:
3413     // When operands have expression values, they will return true for isToken,
3414     // because it is not possible to distinguish between a token and an
3415     // expression at parse time. MatchInstructionImpl() will always try to
3416     // match an operand as a token, when isToken returns true, and when the
3417     // name of the expression is not a valid token, the match will fail,
3418     // so we need to handle it here.
3419     return Operand.isSSrcB32() ? Match_Success : Match_InvalidOperand;
3420   case MCK_SSrcF32:
3421     return Operand.isSSrcF32() ? Match_Success : Match_InvalidOperand;
3422   case MCK_SoppBrTarget:
3423     return Operand.isSoppBrTarget() ? Match_Success : Match_InvalidOperand;
3424   case MCK_VReg32OrOff:
3425     return Operand.isVReg32OrOff() ? Match_Success : Match_InvalidOperand;
3426   default:
3427     return Match_InvalidOperand;
3428   }
3429 }
3430