1 //===-- AMDGPUAsmParser.cpp - Parse SI asm to MCInst instructions ---------===//
2 //
3 //                     The LLVM Compiler Infrastructure
4 //
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
7 //
8 //===----------------------------------------------------------------------===//
9 
10 #include "AMDKernelCodeT.h"
11 #include "MCTargetDesc/AMDGPUMCTargetDesc.h"
12 #include "MCTargetDesc/AMDGPUTargetStreamer.h"
13 #include "SIDefines.h"
14 #include "Utils/AMDGPUBaseInfo.h"
15 #include "Utils/AMDKernelCodeTUtils.h"
16 #include "Utils/AMDGPUAsmUtils.h"
17 #include "llvm/ADT/APFloat.h"
18 #include "llvm/ADT/APInt.h"
19 #include "llvm/ADT/SmallBitVector.h"
20 #include "llvm/ADT/SmallString.h"
21 #include "llvm/ADT/STLExtras.h"
22 #include "llvm/ADT/StringRef.h"
23 #include "llvm/ADT/StringSwitch.h"
24 #include "llvm/ADT/Twine.h"
25 #include "llvm/CodeGen/MachineValueType.h"
26 #include "llvm/MC/MCAsmInfo.h"
27 #include "llvm/MC/MCContext.h"
28 #include "llvm/MC/MCExpr.h"
29 #include "llvm/MC/MCInst.h"
30 #include "llvm/MC/MCInstrDesc.h"
31 #include "llvm/MC/MCInstrInfo.h"
32 #include "llvm/MC/MCParser/MCAsmLexer.h"
33 #include "llvm/MC/MCParser/MCAsmParser.h"
34 #include "llvm/MC/MCParser/MCAsmParserExtension.h"
35 #include "llvm/MC/MCParser/MCParsedAsmOperand.h"
36 #include "llvm/MC/MCParser/MCTargetAsmParser.h"
37 #include "llvm/MC/MCRegisterInfo.h"
38 #include "llvm/MC/MCStreamer.h"
39 #include "llvm/MC/MCSubtargetInfo.h"
40 #include "llvm/MC/MCSymbol.h"
41 #include "llvm/Support/Casting.h"
42 #include "llvm/Support/Debug.h"
43 #include "llvm/Support/ELF.h"
44 #include "llvm/Support/ErrorHandling.h"
45 #include "llvm/Support/MathExtras.h"
46 #include "llvm/Support/raw_ostream.h"
47 #include "llvm/Support/SMLoc.h"
48 #include "llvm/Support/TargetRegistry.h"
49 #include "llvm/Support/raw_ostream.h"
50 #include "llvm/Support/MathExtras.h"
51 #include <algorithm>
52 #include <cassert>
53 #include <cstdint>
54 #include <cstring>
55 #include <iterator>
56 #include <map>
57 #include <memory>
58 #include <string>
59 #include <vector>
60 
61 using namespace llvm;
62 using namespace llvm::AMDGPU;
63 
64 namespace {
65 
66 class AMDGPUAsmParser;
67 
68 enum RegisterKind { IS_UNKNOWN, IS_VGPR, IS_SGPR, IS_TTMP, IS_SPECIAL };
69 
70 //===----------------------------------------------------------------------===//
71 // Operand
72 //===----------------------------------------------------------------------===//
73 
74 class AMDGPUOperand : public MCParsedAsmOperand {
75   enum KindTy {
76     Token,
77     Immediate,
78     Register,
79     Expression
80   } Kind;
81 
82   SMLoc StartLoc, EndLoc;
83   const AMDGPUAsmParser *AsmParser;
84 
85 public:
86   AMDGPUOperand(enum KindTy Kind_, const AMDGPUAsmParser *AsmParser_)
87     : MCParsedAsmOperand(), Kind(Kind_), AsmParser(AsmParser_) {}
88 
89   typedef std::unique_ptr<AMDGPUOperand> Ptr;
90 
91   struct Modifiers {
92     bool Abs = false;
93     bool Neg = false;
94     bool Sext = false;
95 
96     bool hasFPModifiers() const { return Abs || Neg; }
97     bool hasIntModifiers() const { return Sext; }
98     bool hasModifiers() const { return hasFPModifiers() || hasIntModifiers(); }
99 
100     int64_t getFPModifiersOperand() const {
101       int64_t Operand = 0;
102       Operand |= Abs ? SISrcMods::ABS : 0;
103       Operand |= Neg ? SISrcMods::NEG : 0;
104       return Operand;
105     }
106 
107     int64_t getIntModifiersOperand() const {
108       int64_t Operand = 0;
109       Operand |= Sext ? SISrcMods::SEXT : 0;
110       return Operand;
111     }
112 
113     int64_t getModifiersOperand() const {
114       assert(!(hasFPModifiers() && hasIntModifiers())
115            && "fp and int modifiers should not be used simultaneously");
116       if (hasFPModifiers()) {
117         return getFPModifiersOperand();
118       } else if (hasIntModifiers()) {
119         return getIntModifiersOperand();
120       } else {
121         return 0;
122       }
123     }
124 
125     friend raw_ostream &operator <<(raw_ostream &OS, AMDGPUOperand::Modifiers Mods);
126   };
127 
128   enum ImmTy {
129     ImmTyNone,
130     ImmTyGDS,
131     ImmTyOffen,
132     ImmTyIdxen,
133     ImmTyAddr64,
134     ImmTyOffset,
135     ImmTyOffset0,
136     ImmTyOffset1,
137     ImmTyGLC,
138     ImmTySLC,
139     ImmTyTFE,
140     ImmTyClampSI,
141     ImmTyOModSI,
142     ImmTyDppCtrl,
143     ImmTyDppRowMask,
144     ImmTyDppBankMask,
145     ImmTyDppBoundCtrl,
146     ImmTySdwaDstSel,
147     ImmTySdwaSrc0Sel,
148     ImmTySdwaSrc1Sel,
149     ImmTySdwaDstUnused,
150     ImmTyDMask,
151     ImmTyUNorm,
152     ImmTyDA,
153     ImmTyR128,
154     ImmTyLWE,
155     ImmTyExpTgt,
156     ImmTyExpCompr,
157     ImmTyExpVM,
158     ImmTyHwreg,
159     ImmTyOff,
160     ImmTySendMsg,
161     ImmTyInterpSlot,
162     ImmTyInterpAttr,
163     ImmTyAttrChan
164   };
165 
166   struct TokOp {
167     const char *Data;
168     unsigned Length;
169   };
170 
171   struct ImmOp {
172     int64_t Val;
173     ImmTy Type;
174     bool IsFPImm;
175     Modifiers Mods;
176   };
177 
178   struct RegOp {
179     unsigned RegNo;
180     bool IsForcedVOP3;
181     Modifiers Mods;
182   };
183 
184   union {
185     TokOp Tok;
186     ImmOp Imm;
187     RegOp Reg;
188     const MCExpr *Expr;
189   };
190 
191   bool isToken() const override {
192     if (Kind == Token)
193       return true;
194 
195     if (Kind != Expression || !Expr)
196       return false;
197 
198     // When parsing operands, we can't always tell if something was meant to be
199     // a token, like 'gds', or an expression that references a global variable.
200     // In this case, we assume the string is an expression, and if we need to
201     // interpret is a token, then we treat the symbol name as the token.
202     return isa<MCSymbolRefExpr>(Expr);
203   }
204 
205   bool isImm() const override {
206     return Kind == Immediate;
207   }
208 
209   bool isInlinableImm(MVT type) const;
210   bool isLiteralImm(MVT type) const;
211 
212   bool isRegKind() const {
213     return Kind == Register;
214   }
215 
216   bool isReg() const override {
217     return isRegKind() && !Reg.Mods.hasModifiers();
218   }
219 
220   bool isRegOrImmWithInputMods(MVT type) const {
221     return isRegKind() || isInlinableImm(type);
222   }
223 
224   bool isRegOrImmWithInt16InputMods() const {
225     return isRegOrImmWithInputMods(MVT::i16);
226   }
227 
228   bool isRegOrImmWithInt32InputMods() const {
229     return isRegOrImmWithInputMods(MVT::i32);
230   }
231 
232   bool isRegOrImmWithInt64InputMods() const {
233     return isRegOrImmWithInputMods(MVT::i64);
234   }
235 
236   bool isRegOrImmWithFP16InputMods() const {
237     return isRegOrImmWithInputMods(MVT::f16);
238   }
239 
240   bool isRegOrImmWithFP32InputMods() const {
241     return isRegOrImmWithInputMods(MVT::f32);
242   }
243 
244   bool isRegOrImmWithFP64InputMods() const {
245     return isRegOrImmWithInputMods(MVT::f64);
246   }
247 
248   bool isVReg32OrOff() const {
249     return isOff() || isRegClass(AMDGPU::VGPR_32RegClassID);
250   }
251 
252   bool isImmTy(ImmTy ImmT) const {
253     return isImm() && Imm.Type == ImmT;
254   }
255 
256   bool isImmModifier() const {
257     return isImm() && Imm.Type != ImmTyNone;
258   }
259 
260   bool isClampSI() const { return isImmTy(ImmTyClampSI); }
261   bool isOModSI() const { return isImmTy(ImmTyOModSI); }
262   bool isDMask() const { return isImmTy(ImmTyDMask); }
263   bool isUNorm() const { return isImmTy(ImmTyUNorm); }
264   bool isDA() const { return isImmTy(ImmTyDA); }
265   bool isR128() const { return isImmTy(ImmTyUNorm); }
266   bool isLWE() const { return isImmTy(ImmTyLWE); }
267   bool isOff() const { return isImmTy(ImmTyOff); }
268   bool isExpTgt() const { return isImmTy(ImmTyExpTgt); }
269   bool isExpVM() const { return isImmTy(ImmTyExpVM); }
270   bool isExpCompr() const { return isImmTy(ImmTyExpCompr); }
271   bool isOffen() const { return isImmTy(ImmTyOffen); }
272   bool isIdxen() const { return isImmTy(ImmTyIdxen); }
273   bool isAddr64() const { return isImmTy(ImmTyAddr64); }
274   bool isOffset() const { return isImmTy(ImmTyOffset) && isUInt<16>(getImm()); }
275   bool isOffset0() const { return isImmTy(ImmTyOffset0) && isUInt<16>(getImm()); }
276   bool isOffset1() const { return isImmTy(ImmTyOffset1) && isUInt<8>(getImm()); }
277   bool isGDS() const { return isImmTy(ImmTyGDS); }
278   bool isGLC() const { return isImmTy(ImmTyGLC); }
279   bool isSLC() const { return isImmTy(ImmTySLC); }
280   bool isTFE() const { return isImmTy(ImmTyTFE); }
281   bool isBankMask() const { return isImmTy(ImmTyDppBankMask); }
282   bool isRowMask() const { return isImmTy(ImmTyDppRowMask); }
283   bool isBoundCtrl() const { return isImmTy(ImmTyDppBoundCtrl); }
284   bool isSDWADstSel() const { return isImmTy(ImmTySdwaDstSel); }
285   bool isSDWASrc0Sel() const { return isImmTy(ImmTySdwaSrc0Sel); }
286   bool isSDWASrc1Sel() const { return isImmTy(ImmTySdwaSrc1Sel); }
287   bool isSDWADstUnused() const { return isImmTy(ImmTySdwaDstUnused); }
288   bool isInterpSlot() const { return isImmTy(ImmTyInterpSlot); }
289   bool isInterpAttr() const { return isImmTy(ImmTyInterpAttr); }
290   bool isAttrChan() const { return isImmTy(ImmTyAttrChan); }
291 
292   bool isMod() const {
293     return isClampSI() || isOModSI();
294   }
295 
296   bool isRegOrImm() const {
297     return isReg() || isImm();
298   }
299 
300   bool isRegClass(unsigned RCID) const;
301 
302   bool isSCSrcB16() const {
303     return isRegClass(AMDGPU::SReg_32RegClassID) || isInlinableImm(MVT::i16);
304   }
305 
306   bool isSCSrcB32() const {
307     return isRegClass(AMDGPU::SReg_32RegClassID) || isInlinableImm(MVT::i32);
308   }
309 
310   bool isSCSrcB64() const {
311     return isRegClass(AMDGPU::SReg_64RegClassID) || isInlinableImm(MVT::i64);
312   }
313 
314   bool isSCSrcF16() const {
315     return isRegClass(AMDGPU::SReg_32RegClassID) || isInlinableImm(MVT::f16);
316   }
317 
318   bool isSCSrcF32() const {
319     return isRegClass(AMDGPU::SReg_32RegClassID) || isInlinableImm(MVT::f32);
320   }
321 
322   bool isSCSrcF64() const {
323     return isRegClass(AMDGPU::SReg_64RegClassID) || isInlinableImm(MVT::f64);
324   }
325 
326   bool isSSrcB32() const {
327     return isSCSrcB32() || isLiteralImm(MVT::i32) || isExpr();
328   }
329 
330   bool isSSrcB16() const {
331     return isSCSrcB16() || isLiteralImm(MVT::i16);
332   }
333 
334   bool isSSrcB64() const {
335     // TODO: Find out how SALU supports extension of 32-bit literals to 64 bits.
336     // See isVSrc64().
337     return isSCSrcB64() || isLiteralImm(MVT::i64);
338   }
339 
340   bool isSSrcF32() const {
341     return isSCSrcB32() || isLiteralImm(MVT::f32) || isExpr();
342   }
343 
344   bool isSSrcF64() const {
345     return isSCSrcB64() || isLiteralImm(MVT::f64);
346   }
347 
348   bool isSSrcF16() const {
349     return isSCSrcB16() || isLiteralImm(MVT::f16);
350   }
351 
352   bool isVCSrcB32() const {
353     return isRegClass(AMDGPU::VS_32RegClassID) || isInlinableImm(MVT::i32);
354   }
355 
356   bool isVCSrcB64() const {
357     return isRegClass(AMDGPU::VS_64RegClassID) || isInlinableImm(MVT::i64);
358   }
359 
360   bool isVCSrcB16() const {
361     return isRegClass(AMDGPU::VS_32RegClassID) || isInlinableImm(MVT::i16);
362   }
363 
364   bool isVCSrcF32() const {
365     return isRegClass(AMDGPU::VS_32RegClassID) || isInlinableImm(MVT::f32);
366   }
367 
368   bool isVCSrcF64() const {
369     return isRegClass(AMDGPU::VS_64RegClassID) || isInlinableImm(MVT::f64);
370   }
371 
372   bool isVCSrcF16() const {
373     return isRegClass(AMDGPU::VS_32RegClassID) || isInlinableImm(MVT::f16);
374   }
375 
376   bool isVSrcB32() const {
377     return isVCSrcF32() || isLiteralImm(MVT::i32);
378   }
379 
380   bool isVSrcB64() const {
381     return isVCSrcF64() || isLiteralImm(MVT::i64);
382   }
383 
384   bool isVSrcB16() const {
385     return isVCSrcF16() || isLiteralImm(MVT::i16);
386   }
387 
388   bool isVSrcF32() const {
389     return isVCSrcF32() || isLiteralImm(MVT::f32);
390   }
391 
392   bool isVSrcF64() const {
393     return isVCSrcF64() || isLiteralImm(MVT::f64);
394   }
395 
396   bool isVSrcF16() const {
397     return isVCSrcF16() || isLiteralImm(MVT::f16);
398   }
399 
400   bool isKImmFP32() const {
401     return isLiteralImm(MVT::f32);
402   }
403 
404   bool isKImmFP16() const {
405     return isLiteralImm(MVT::f16);
406   }
407 
408   bool isMem() const override {
409     return false;
410   }
411 
412   bool isExpr() const {
413     return Kind == Expression;
414   }
415 
416   bool isSoppBrTarget() const {
417     return isExpr() || isImm();
418   }
419 
420   bool isSWaitCnt() const;
421   bool isHwreg() const;
422   bool isSendMsg() const;
423   bool isSMRDOffset8() const;
424   bool isSMRDOffset20() const;
425   bool isSMRDLiteralOffset() const;
426   bool isDPPCtrl() const;
427   bool isGPRIdxMode() const;
428 
429   StringRef getExpressionAsToken() const {
430     assert(isExpr());
431     const MCSymbolRefExpr *S = cast<MCSymbolRefExpr>(Expr);
432     return S->getSymbol().getName();
433   }
434 
435   StringRef getToken() const {
436     assert(isToken());
437 
438     if (Kind == Expression)
439       return getExpressionAsToken();
440 
441     return StringRef(Tok.Data, Tok.Length);
442   }
443 
444   int64_t getImm() const {
445     assert(isImm());
446     return Imm.Val;
447   }
448 
449   enum ImmTy getImmTy() const {
450     assert(isImm());
451     return Imm.Type;
452   }
453 
454   unsigned getReg() const override {
455     return Reg.RegNo;
456   }
457 
458   SMLoc getStartLoc() const override {
459     return StartLoc;
460   }
461 
462   SMLoc getEndLoc() const override {
463     return EndLoc;
464   }
465 
466   Modifiers getModifiers() const {
467     assert(isRegKind() || isImmTy(ImmTyNone));
468     return isRegKind() ? Reg.Mods : Imm.Mods;
469   }
470 
471   void setModifiers(Modifiers Mods) {
472     assert(isRegKind() || isImmTy(ImmTyNone));
473     if (isRegKind())
474       Reg.Mods = Mods;
475     else
476       Imm.Mods = Mods;
477   }
478 
479   bool hasModifiers() const {
480     return getModifiers().hasModifiers();
481   }
482 
483   bool hasFPModifiers() const {
484     return getModifiers().hasFPModifiers();
485   }
486 
487   bool hasIntModifiers() const {
488     return getModifiers().hasIntModifiers();
489   }
490 
491   void addImmOperands(MCInst &Inst, unsigned N, bool ApplyModifiers = true) const;
492 
493   void addLiteralImmOperand(MCInst &Inst, int64_t Val) const;
494 
495   template <unsigned Bitwidth>
496   void addKImmFPOperands(MCInst &Inst, unsigned N) const;
497 
498   void addKImmFP16Operands(MCInst &Inst, unsigned N) const {
499     addKImmFPOperands<16>(Inst, N);
500   }
501 
502   void addKImmFP32Operands(MCInst &Inst, unsigned N) const {
503     addKImmFPOperands<32>(Inst, N);
504   }
505 
506   void addRegOperands(MCInst &Inst, unsigned N) const;
507 
508   void addRegOrImmOperands(MCInst &Inst, unsigned N) const {
509     if (isRegKind())
510       addRegOperands(Inst, N);
511     else if (isExpr())
512       Inst.addOperand(MCOperand::createExpr(Expr));
513     else
514       addImmOperands(Inst, N);
515   }
516 
517   void addRegOrImmWithInputModsOperands(MCInst &Inst, unsigned N) const {
518     Modifiers Mods = getModifiers();
519     Inst.addOperand(MCOperand::createImm(Mods.getModifiersOperand()));
520     if (isRegKind()) {
521       addRegOperands(Inst, N);
522     } else {
523       addImmOperands(Inst, N, false);
524     }
525   }
526 
527   void addRegOrImmWithFPInputModsOperands(MCInst &Inst, unsigned N) const {
528     assert(!hasIntModifiers());
529     addRegOrImmWithInputModsOperands(Inst, N);
530   }
531 
532   void addRegOrImmWithIntInputModsOperands(MCInst &Inst, unsigned N) const {
533     assert(!hasFPModifiers());
534     addRegOrImmWithInputModsOperands(Inst, N);
535   }
536 
537   void addSoppBrTargetOperands(MCInst &Inst, unsigned N) const {
538     if (isImm())
539       addImmOperands(Inst, N);
540     else {
541       assert(isExpr());
542       Inst.addOperand(MCOperand::createExpr(Expr));
543     }
544   }
545 
546   static void printImmTy(raw_ostream& OS, ImmTy Type) {
547     switch (Type) {
548     case ImmTyNone: OS << "None"; break;
549     case ImmTyGDS: OS << "GDS"; break;
550     case ImmTyOffen: OS << "Offen"; break;
551     case ImmTyIdxen: OS << "Idxen"; break;
552     case ImmTyAddr64: OS << "Addr64"; break;
553     case ImmTyOffset: OS << "Offset"; break;
554     case ImmTyOffset0: OS << "Offset0"; break;
555     case ImmTyOffset1: OS << "Offset1"; break;
556     case ImmTyGLC: OS << "GLC"; break;
557     case ImmTySLC: OS << "SLC"; break;
558     case ImmTyTFE: OS << "TFE"; break;
559     case ImmTyClampSI: OS << "ClampSI"; break;
560     case ImmTyOModSI: OS << "OModSI"; break;
561     case ImmTyDppCtrl: OS << "DppCtrl"; break;
562     case ImmTyDppRowMask: OS << "DppRowMask"; break;
563     case ImmTyDppBankMask: OS << "DppBankMask"; break;
564     case ImmTyDppBoundCtrl: OS << "DppBoundCtrl"; break;
565     case ImmTySdwaDstSel: OS << "SdwaDstSel"; break;
566     case ImmTySdwaSrc0Sel: OS << "SdwaSrc0Sel"; break;
567     case ImmTySdwaSrc1Sel: OS << "SdwaSrc1Sel"; break;
568     case ImmTySdwaDstUnused: OS << "SdwaDstUnused"; break;
569     case ImmTyDMask: OS << "DMask"; break;
570     case ImmTyUNorm: OS << "UNorm"; break;
571     case ImmTyDA: OS << "DA"; break;
572     case ImmTyR128: OS << "R128"; break;
573     case ImmTyLWE: OS << "LWE"; break;
574     case ImmTyOff: OS << "Off"; break;
575     case ImmTyExpTgt: OS << "ExpTgt"; break;
576     case ImmTyExpCompr: OS << "ExpCompr"; break;
577     case ImmTyExpVM: OS << "ExpVM"; break;
578     case ImmTyHwreg: OS << "Hwreg"; break;
579     case ImmTySendMsg: OS << "SendMsg"; break;
580     case ImmTyInterpSlot: OS << "InterpSlot"; break;
581     case ImmTyInterpAttr: OS << "InterpAttr"; break;
582     case ImmTyAttrChan: OS << "AttrChan"; break;
583     }
584   }
585 
586   void print(raw_ostream &OS) const override {
587     switch (Kind) {
588     case Register:
589       OS << "<register " << getReg() << " mods: " << Reg.Mods << '>';
590       break;
591     case Immediate:
592       OS << '<' << getImm();
593       if (getImmTy() != ImmTyNone) {
594         OS << " type: "; printImmTy(OS, getImmTy());
595       }
596       OS << " mods: " << Imm.Mods << '>';
597       break;
598     case Token:
599       OS << '\'' << getToken() << '\'';
600       break;
601     case Expression:
602       OS << "<expr " << *Expr << '>';
603       break;
604     }
605   }
606 
607   static AMDGPUOperand::Ptr CreateImm(const AMDGPUAsmParser *AsmParser,
608                                       int64_t Val, SMLoc Loc,
609                                       enum ImmTy Type = ImmTyNone,
610                                       bool IsFPImm = false) {
611     auto Op = llvm::make_unique<AMDGPUOperand>(Immediate, AsmParser);
612     Op->Imm.Val = Val;
613     Op->Imm.IsFPImm = IsFPImm;
614     Op->Imm.Type = Type;
615     Op->Imm.Mods = Modifiers();
616     Op->StartLoc = Loc;
617     Op->EndLoc = Loc;
618     return Op;
619   }
620 
621   static AMDGPUOperand::Ptr CreateToken(const AMDGPUAsmParser *AsmParser,
622                                         StringRef Str, SMLoc Loc,
623                                         bool HasExplicitEncodingSize = true) {
624     auto Res = llvm::make_unique<AMDGPUOperand>(Token, AsmParser);
625     Res->Tok.Data = Str.data();
626     Res->Tok.Length = Str.size();
627     Res->StartLoc = Loc;
628     Res->EndLoc = Loc;
629     return Res;
630   }
631 
632   static AMDGPUOperand::Ptr CreateReg(const AMDGPUAsmParser *AsmParser,
633                                       unsigned RegNo, SMLoc S,
634                                       SMLoc E,
635                                       bool ForceVOP3) {
636     auto Op = llvm::make_unique<AMDGPUOperand>(Register, AsmParser);
637     Op->Reg.RegNo = RegNo;
638     Op->Reg.Mods = Modifiers();
639     Op->Reg.IsForcedVOP3 = ForceVOP3;
640     Op->StartLoc = S;
641     Op->EndLoc = E;
642     return Op;
643   }
644 
645   static AMDGPUOperand::Ptr CreateExpr(const AMDGPUAsmParser *AsmParser,
646                                        const class MCExpr *Expr, SMLoc S) {
647     auto Op = llvm::make_unique<AMDGPUOperand>(Expression, AsmParser);
648     Op->Expr = Expr;
649     Op->StartLoc = S;
650     Op->EndLoc = S;
651     return Op;
652   }
653 };
654 
655 raw_ostream &operator <<(raw_ostream &OS, AMDGPUOperand::Modifiers Mods) {
656   OS << "abs:" << Mods.Abs << " neg: " << Mods.Neg << " sext:" << Mods.Sext;
657   return OS;
658 }
659 
660 //===----------------------------------------------------------------------===//
661 // AsmParser
662 //===----------------------------------------------------------------------===//
663 
664 // Holds info related to the current kernel, e.g. count of SGPRs used.
665 // Kernel scope begins at .amdgpu_hsa_kernel directive, ends at next
666 // .amdgpu_hsa_kernel or at EOF.
667 class KernelScopeInfo {
668   int SgprIndexUnusedMin;
669   int VgprIndexUnusedMin;
670   MCContext *Ctx;
671 
672   void usesSgprAt(int i) {
673     if (i >= SgprIndexUnusedMin) {
674       SgprIndexUnusedMin = ++i;
675       if (Ctx) {
676         MCSymbol * const Sym = Ctx->getOrCreateSymbol(Twine(".kernel.sgpr_count"));
677         Sym->setVariableValue(MCConstantExpr::create(SgprIndexUnusedMin, *Ctx));
678       }
679     }
680   }
681   void usesVgprAt(int i) {
682     if (i >= VgprIndexUnusedMin) {
683       VgprIndexUnusedMin = ++i;
684       if (Ctx) {
685         MCSymbol * const Sym = Ctx->getOrCreateSymbol(Twine(".kernel.vgpr_count"));
686         Sym->setVariableValue(MCConstantExpr::create(VgprIndexUnusedMin, *Ctx));
687       }
688     }
689   }
690 public:
691   KernelScopeInfo() : SgprIndexUnusedMin(-1), VgprIndexUnusedMin(-1), Ctx(nullptr)
692   {}
693   void initialize(MCContext &Context) {
694     Ctx = &Context;
695     usesSgprAt(SgprIndexUnusedMin = -1);
696     usesVgprAt(VgprIndexUnusedMin = -1);
697   }
698   void usesRegister(RegisterKind RegKind, unsigned DwordRegIndex, unsigned RegWidth) {
699     switch (RegKind) {
700       case IS_SGPR: usesSgprAt(DwordRegIndex + RegWidth - 1); break;
701       case IS_VGPR: usesVgprAt(DwordRegIndex + RegWidth - 1); break;
702       default: break;
703     }
704   }
705 };
706 
707 class AMDGPUAsmParser : public MCTargetAsmParser {
708   const MCInstrInfo &MII;
709   MCAsmParser &Parser;
710 
711   unsigned ForcedEncodingSize;
712   bool ForcedDPP;
713   bool ForcedSDWA;
714   KernelScopeInfo KernelScope;
715 
716   /// @name Auto-generated Match Functions
717   /// {
718 
719 #define GET_ASSEMBLER_HEADER
720 #include "AMDGPUGenAsmMatcher.inc"
721 
722   /// }
723 
724 private:
725   bool ParseAsAbsoluteExpression(uint32_t &Ret);
726   bool ParseDirectiveMajorMinor(uint32_t &Major, uint32_t &Minor);
727   bool ParseDirectiveHSACodeObjectVersion();
728   bool ParseDirectiveHSACodeObjectISA();
729   bool ParseDirectiveRuntimeMetadata();
730   bool ParseAMDKernelCodeTValue(StringRef ID, amd_kernel_code_t &Header);
731   bool ParseDirectiveAMDKernelCodeT();
732   bool ParseSectionDirectiveHSAText();
733   bool subtargetHasRegister(const MCRegisterInfo &MRI, unsigned RegNo) const;
734   bool ParseDirectiveAMDGPUHsaKernel();
735   bool ParseDirectiveAMDGPUHsaModuleGlobal();
736   bool ParseDirectiveAMDGPUHsaProgramGlobal();
737   bool ParseSectionDirectiveHSADataGlobalAgent();
738   bool ParseSectionDirectiveHSADataGlobalProgram();
739   bool ParseSectionDirectiveHSARodataReadonlyAgent();
740   bool AddNextRegisterToList(unsigned& Reg, unsigned& RegWidth, RegisterKind RegKind, unsigned Reg1, unsigned RegNum);
741   bool ParseAMDGPURegister(RegisterKind& RegKind, unsigned& Reg, unsigned& RegNum, unsigned& RegWidth, unsigned *DwordRegIndex);
742   void cvtMubufImpl(MCInst &Inst, const OperandVector &Operands, bool IsAtomic, bool IsAtomicReturn);
743 
744 public:
745   enum AMDGPUMatchResultTy {
746     Match_PreferE32 = FIRST_TARGET_MATCH_RESULT_TY
747   };
748 
749   AMDGPUAsmParser(const MCSubtargetInfo &STI, MCAsmParser &_Parser,
750                const MCInstrInfo &MII,
751                const MCTargetOptions &Options)
752       : MCTargetAsmParser(Options, STI), MII(MII), Parser(_Parser),
753         ForcedEncodingSize(0),
754         ForcedDPP(false),
755         ForcedSDWA(false) {
756     MCAsmParserExtension::Initialize(Parser);
757 
758     if (getSTI().getFeatureBits().none()) {
759       // Set default features.
760       copySTI().ToggleFeature("SOUTHERN_ISLANDS");
761     }
762 
763     setAvailableFeatures(ComputeAvailableFeatures(getSTI().getFeatureBits()));
764 
765     {
766       // TODO: make those pre-defined variables read-only.
767       // Currently there is none suitable machinery in the core llvm-mc for this.
768       // MCSymbol::isRedefinable is intended for another purpose, and
769       // AsmParser::parseDirectiveSet() cannot be specialized for specific target.
770       AMDGPU::IsaVersion Isa = AMDGPU::getIsaVersion(getSTI().getFeatureBits());
771       MCContext &Ctx = getContext();
772       MCSymbol *Sym = Ctx.getOrCreateSymbol(Twine(".option.machine_version_major"));
773       Sym->setVariableValue(MCConstantExpr::create(Isa.Major, Ctx));
774       Sym = Ctx.getOrCreateSymbol(Twine(".option.machine_version_minor"));
775       Sym->setVariableValue(MCConstantExpr::create(Isa.Minor, Ctx));
776       Sym = Ctx.getOrCreateSymbol(Twine(".option.machine_version_stepping"));
777       Sym->setVariableValue(MCConstantExpr::create(Isa.Stepping, Ctx));
778     }
779     KernelScope.initialize(getContext());
780   }
781 
782   bool isSI() const {
783     return AMDGPU::isSI(getSTI());
784   }
785 
786   bool isCI() const {
787     return AMDGPU::isCI(getSTI());
788   }
789 
790   bool isVI() const {
791     return AMDGPU::isVI(getSTI());
792   }
793 
794   bool hasInv2PiInlineImm() const {
795     return getSTI().getFeatureBits()[AMDGPU::FeatureInv2PiInlineImm];
796   }
797 
798   bool hasSGPR102_SGPR103() const {
799     return !isVI();
800   }
801 
802   AMDGPUTargetStreamer &getTargetStreamer() {
803     MCTargetStreamer &TS = *getParser().getStreamer().getTargetStreamer();
804     return static_cast<AMDGPUTargetStreamer &>(TS);
805   }
806 
807   const MCRegisterInfo *getMRI() const {
808     // We need this const_cast because for some reason getContext() is not const
809     // in MCAsmParser.
810     return const_cast<AMDGPUAsmParser*>(this)->getContext().getRegisterInfo();
811   }
812 
813   const MCInstrInfo *getMII() const {
814     return &MII;
815   }
816 
817   void setForcedEncodingSize(unsigned Size) { ForcedEncodingSize = Size; }
818   void setForcedDPP(bool ForceDPP_) { ForcedDPP = ForceDPP_; }
819   void setForcedSDWA(bool ForceSDWA_) { ForcedSDWA = ForceSDWA_; }
820 
821   unsigned getForcedEncodingSize() const { return ForcedEncodingSize; }
822   bool isForcedVOP3() const { return ForcedEncodingSize == 64; }
823   bool isForcedDPP() const { return ForcedDPP; }
824   bool isForcedSDWA() const { return ForcedSDWA; }
825   ArrayRef<unsigned> getMatchedVariants() const;
826 
827   std::unique_ptr<AMDGPUOperand> parseRegister();
828   bool ParseRegister(unsigned &RegNo, SMLoc &StartLoc, SMLoc &EndLoc) override;
829   unsigned checkTargetMatchPredicate(MCInst &Inst) override;
830   unsigned validateTargetOperandClass(MCParsedAsmOperand &Op,
831                                       unsigned Kind) override;
832   bool MatchAndEmitInstruction(SMLoc IDLoc, unsigned &Opcode,
833                                OperandVector &Operands, MCStreamer &Out,
834                                uint64_t &ErrorInfo,
835                                bool MatchingInlineAsm) override;
836   bool ParseDirective(AsmToken DirectiveID) override;
837   OperandMatchResultTy parseOperand(OperandVector &Operands, StringRef Mnemonic);
838   StringRef parseMnemonicSuffix(StringRef Name);
839   bool ParseInstruction(ParseInstructionInfo &Info, StringRef Name,
840                         SMLoc NameLoc, OperandVector &Operands) override;
841   //bool ProcessInstruction(MCInst &Inst);
842 
843   OperandMatchResultTy parseIntWithPrefix(const char *Prefix, int64_t &Int);
844   OperandMatchResultTy
845   parseIntWithPrefix(const char *Prefix, OperandVector &Operands,
846                      enum AMDGPUOperand::ImmTy ImmTy = AMDGPUOperand::ImmTyNone,
847                      bool (*ConvertResult)(int64_t &) = nullptr);
848   OperandMatchResultTy
849   parseNamedBit(const char *Name, OperandVector &Operands,
850                 enum AMDGPUOperand::ImmTy ImmTy = AMDGPUOperand::ImmTyNone);
851   OperandMatchResultTy parseStringWithPrefix(StringRef Prefix,
852                                              StringRef &Value);
853 
854   OperandMatchResultTy parseImm(OperandVector &Operands);
855   OperandMatchResultTy parseRegOrImm(OperandVector &Operands);
856   OperandMatchResultTy parseRegOrImmWithFPInputMods(OperandVector &Operands);
857   OperandMatchResultTy parseRegOrImmWithIntInputMods(OperandVector &Operands);
858   OperandMatchResultTy parseVReg32OrOff(OperandVector &Operands);
859 
860   void cvtDSOffset01(MCInst &Inst, const OperandVector &Operands);
861   void cvtDS(MCInst &Inst, const OperandVector &Operands);
862   void cvtExp(MCInst &Inst, const OperandVector &Operands);
863 
864   bool parseCnt(int64_t &IntVal);
865   OperandMatchResultTy parseSWaitCntOps(OperandVector &Operands);
866   OperandMatchResultTy parseHwreg(OperandVector &Operands);
867 
868 private:
869   struct OperandInfoTy {
870     int64_t Id;
871     bool IsSymbolic;
872     OperandInfoTy(int64_t Id_) : Id(Id_), IsSymbolic(false) { }
873   };
874 
875   bool parseSendMsgConstruct(OperandInfoTy &Msg, OperandInfoTy &Operation, int64_t &StreamId);
876   bool parseHwregConstruct(OperandInfoTy &HwReg, int64_t &Offset, int64_t &Width);
877 
878   void errorExpTgt();
879   OperandMatchResultTy parseExpTgtImpl(StringRef Str, uint8_t &Val);
880 
881 public:
882   OperandMatchResultTy parseOptionalOperand(OperandVector &Operands);
883 
884   OperandMatchResultTy parseExpTgt(OperandVector &Operands);
885   OperandMatchResultTy parseSendMsgOp(OperandVector &Operands);
886   OperandMatchResultTy parseInterpSlot(OperandVector &Operands);
887   OperandMatchResultTy parseInterpAttr(OperandVector &Operands);
888   OperandMatchResultTy parseSOppBrTarget(OperandVector &Operands);
889 
890   void cvtMubuf(MCInst &Inst, const OperandVector &Operands) { cvtMubufImpl(Inst, Operands, false, false); }
891   void cvtMubufAtomic(MCInst &Inst, const OperandVector &Operands) { cvtMubufImpl(Inst, Operands, true, false); }
892   void cvtMubufAtomicReturn(MCInst &Inst, const OperandVector &Operands) { cvtMubufImpl(Inst, Operands, true, true); }
893   AMDGPUOperand::Ptr defaultGLC() const;
894   AMDGPUOperand::Ptr defaultSLC() const;
895   AMDGPUOperand::Ptr defaultTFE() const;
896 
897   AMDGPUOperand::Ptr defaultDMask() const;
898   AMDGPUOperand::Ptr defaultUNorm() const;
899   AMDGPUOperand::Ptr defaultDA() const;
900   AMDGPUOperand::Ptr defaultR128() const;
901   AMDGPUOperand::Ptr defaultLWE() const;
902   AMDGPUOperand::Ptr defaultSMRDOffset8() const;
903   AMDGPUOperand::Ptr defaultSMRDOffset20() const;
904   AMDGPUOperand::Ptr defaultSMRDLiteralOffset() const;
905 
906   OperandMatchResultTy parseOModOperand(OperandVector &Operands);
907 
908   void cvtId(MCInst &Inst, const OperandVector &Operands);
909   void cvtVOP3_2_mod(MCInst &Inst, const OperandVector &Operands);
910   void cvtVOP3(MCInst &Inst, const OperandVector &Operands);
911 
912   void cvtMIMG(MCInst &Inst, const OperandVector &Operands);
913   void cvtMIMGAtomic(MCInst &Inst, const OperandVector &Operands);
914 
915   OperandMatchResultTy parseDPPCtrl(OperandVector &Operands);
916   AMDGPUOperand::Ptr defaultRowMask() const;
917   AMDGPUOperand::Ptr defaultBankMask() const;
918   AMDGPUOperand::Ptr defaultBoundCtrl() const;
919   void cvtDPP(MCInst &Inst, const OperandVector &Operands);
920 
921   OperandMatchResultTy parseSDWASel(OperandVector &Operands, StringRef Prefix,
922                                     AMDGPUOperand::ImmTy Type);
923   OperandMatchResultTy parseSDWADstUnused(OperandVector &Operands);
924   void cvtSdwaVOP1(MCInst &Inst, const OperandVector &Operands);
925   void cvtSdwaVOP2(MCInst &Inst, const OperandVector &Operands);
926   void cvtSdwaVOPC(MCInst &Inst, const OperandVector &Operands);
927   void cvtSDWA(MCInst &Inst, const OperandVector &Operands,
928                uint64_t BasicInstType);
929 };
930 
931 struct OptionalOperand {
932   const char *Name;
933   AMDGPUOperand::ImmTy Type;
934   bool IsBit;
935   bool (*ConvertResult)(int64_t&);
936 };
937 
938 } // end anonymous namespace
939 
940 // May be called with integer type with equivalent bitwidth.
941 static const fltSemantics *getFltSemantics(unsigned Size) {
942   switch (Size) {
943   case 4:
944     return &APFloat::IEEEsingle();
945   case 8:
946     return &APFloat::IEEEdouble();
947   case 2:
948     return &APFloat::IEEEhalf();
949   default:
950     llvm_unreachable("unsupported fp type");
951   }
952 }
953 
954 static const fltSemantics *getFltSemantics(MVT VT) {
955   return getFltSemantics(VT.getSizeInBits() / 8);
956 }
957 
958 //===----------------------------------------------------------------------===//
959 // Operand
960 //===----------------------------------------------------------------------===//
961 
962 static bool canLosslesslyConvertToFPType(APFloat &FPLiteral, MVT VT) {
963   bool Lost;
964 
965   // Convert literal to single precision
966   APFloat::opStatus Status = FPLiteral.convert(*getFltSemantics(VT),
967                                                APFloat::rmNearestTiesToEven,
968                                                &Lost);
969   // We allow precision lost but not overflow or underflow
970   if (Status != APFloat::opOK &&
971       Lost &&
972       ((Status & APFloat::opOverflow)  != 0 ||
973        (Status & APFloat::opUnderflow) != 0)) {
974     return false;
975   }
976 
977   return true;
978 }
979 
980 bool AMDGPUOperand::isInlinableImm(MVT type) const {
981   if (!isImmTy(ImmTyNone)) {
982     // Only plain immediates are inlinable (e.g. "clamp" attribute is not)
983     return false;
984   }
985   // TODO: We should avoid using host float here. It would be better to
986   // check the float bit values which is what a few other places do.
987   // We've had bot failures before due to weird NaN support on mips hosts.
988 
989   APInt Literal(64, Imm.Val);
990 
991   if (Imm.IsFPImm) { // We got fp literal token
992     if (type == MVT::f64 || type == MVT::i64) { // Expected 64-bit operand
993       return AMDGPU::isInlinableLiteral64(Imm.Val,
994                                           AsmParser->hasInv2PiInlineImm());
995     }
996 
997     APFloat FPLiteral(APFloat::IEEEdouble(), APInt(64, Imm.Val));
998     if (!canLosslesslyConvertToFPType(FPLiteral, type))
999       return false;
1000 
1001     // Check if single precision literal is inlinable
1002     return AMDGPU::isInlinableLiteral32(
1003       static_cast<int32_t>(FPLiteral.bitcastToAPInt().getZExtValue()),
1004       AsmParser->hasInv2PiInlineImm());
1005   }
1006 
1007 
1008   // We got int literal token.
1009   if (type == MVT::f64 || type == MVT::i64) { // Expected 64-bit operand
1010     return AMDGPU::isInlinableLiteral64(Imm.Val,
1011                                         AsmParser->hasInv2PiInlineImm());
1012   }
1013 
1014   if (type.getScalarSizeInBits() == 16) {
1015     return AMDGPU::isInlinableLiteral16(
1016       static_cast<int16_t>(Literal.getLoBits(16).getSExtValue()),
1017       AsmParser->hasInv2PiInlineImm());
1018   }
1019 
1020   return AMDGPU::isInlinableLiteral32(
1021     static_cast<int32_t>(Literal.getLoBits(32).getZExtValue()),
1022     AsmParser->hasInv2PiInlineImm());
1023 }
1024 
1025 bool AMDGPUOperand::isLiteralImm(MVT type) const {
1026   // Check that this imediate can be added as literal
1027   if (!isImmTy(ImmTyNone)) {
1028     return false;
1029   }
1030 
1031   if (!Imm.IsFPImm) {
1032     // We got int literal token.
1033 
1034     unsigned Size = type.getSizeInBits();
1035     if (Size == 64)
1036       Size = 32;
1037 
1038     // FIXME: 64-bit operands can zero extend, sign extend, or pad zeroes for FP
1039     // types.
1040     return isUIntN(Size, Imm.Val) || isIntN(Size, Imm.Val);
1041   }
1042 
1043   // We got fp literal token
1044   if (type == MVT::f64) { // Expected 64-bit fp operand
1045     // We would set low 64-bits of literal to zeroes but we accept this literals
1046     return true;
1047   }
1048 
1049   if (type == MVT::i64) { // Expected 64-bit int operand
1050     // We don't allow fp literals in 64-bit integer instructions. It is
1051     // unclear how we should encode them.
1052     return false;
1053   }
1054 
1055   APFloat FPLiteral(APFloat::IEEEdouble(), APInt(64, Imm.Val));
1056   return canLosslesslyConvertToFPType(FPLiteral, type);
1057 }
1058 
1059 bool AMDGPUOperand::isRegClass(unsigned RCID) const {
1060   return isReg() && AsmParser->getMRI()->getRegClass(RCID).contains(getReg());
1061 }
1062 
1063 void AMDGPUOperand::addImmOperands(MCInst &Inst, unsigned N, bool ApplyModifiers) const {
1064   int64_t Val = Imm.Val;
1065   if (isImmTy(ImmTyNone) && ApplyModifiers && Imm.Mods.hasFPModifiers() && Imm.Mods.Neg) {
1066     // Apply modifiers to immediate value. Only negate can get here
1067     if (Imm.IsFPImm) {
1068       APFloat F(BitsToDouble(Val));
1069       F.changeSign();
1070       Val = F.bitcastToAPInt().getZExtValue();
1071     } else {
1072       Val = -Val;
1073     }
1074   }
1075 
1076   if (AMDGPU::isSISrcOperand(AsmParser->getMII()->get(Inst.getOpcode()),
1077                              Inst.getNumOperands())) {
1078     addLiteralImmOperand(Inst, Val);
1079   } else {
1080     Inst.addOperand(MCOperand::createImm(Val));
1081   }
1082 }
1083 
1084 void AMDGPUOperand::addLiteralImmOperand(MCInst &Inst, int64_t Val) const {
1085   const auto& InstDesc = AsmParser->getMII()->get(Inst.getOpcode());
1086   auto OpNum = Inst.getNumOperands();
1087   // Check that this operand accepts literals
1088   assert(AMDGPU::isSISrcOperand(InstDesc, OpNum));
1089 
1090   auto OpSize = AMDGPU::getOperandSize(InstDesc, OpNum); // expected operand size
1091 
1092   if (Imm.IsFPImm) { // We got fp literal token
1093     APInt Literal(64, Val);
1094 
1095     switch (OpSize) {
1096     case 8: {
1097       if (AMDGPU::isInlinableLiteral64(Literal.getZExtValue(),
1098                                        AsmParser->hasInv2PiInlineImm())) {
1099         Inst.addOperand(MCOperand::createImm(Literal.getZExtValue()));
1100         return;
1101       }
1102 
1103       // Non-inlineable
1104       if (AMDGPU::isSISrcFPOperand(InstDesc, OpNum)) { // Expected 64-bit fp operand
1105         // For fp operands we check if low 32 bits are zeros
1106         if (Literal.getLoBits(32) != 0) {
1107           const_cast<AMDGPUAsmParser *>(AsmParser)->Warning(Inst.getLoc(),
1108           "Can't encode literal as exact 64-bit floating-point operand. "
1109           "Low 32-bits will be set to zero");
1110         }
1111 
1112         Inst.addOperand(MCOperand::createImm(Literal.lshr(32).getZExtValue()));
1113         return;
1114       }
1115 
1116       // We don't allow fp literals in 64-bit integer instructions. It is
1117       // unclear how we should encode them. This case should be checked earlier
1118       // in predicate methods (isLiteralImm())
1119       llvm_unreachable("fp literal in 64-bit integer instruction.");
1120     }
1121     case 4:
1122     case 2: {
1123       bool lost;
1124       APFloat FPLiteral(APFloat::IEEEdouble(), Literal);
1125       // Convert literal to single precision
1126       FPLiteral.convert(*getFltSemantics(OpSize),
1127                         APFloat::rmNearestTiesToEven, &lost);
1128       // We allow precision lost but not overflow or underflow. This should be
1129       // checked earlier in isLiteralImm()
1130       Inst.addOperand(MCOperand::createImm(FPLiteral.bitcastToAPInt().getZExtValue()));
1131       return;
1132     }
1133     default:
1134       llvm_unreachable("invalid operand size");
1135     }
1136 
1137     return;
1138   }
1139 
1140    // We got int literal token.
1141   // Only sign extend inline immediates.
1142   // FIXME: No errors on truncation
1143   switch (OpSize) {
1144   case 4: {
1145     if (isInt<32>(Val) &&
1146         AMDGPU::isInlinableLiteral32(static_cast<int32_t>(Val),
1147                                      AsmParser->hasInv2PiInlineImm())) {
1148       Inst.addOperand(MCOperand::createImm(Val));
1149       return;
1150     }
1151 
1152     Inst.addOperand(MCOperand::createImm(Val & 0xffffffff));
1153     return;
1154   }
1155   case 8: {
1156     if (AMDGPU::isInlinableLiteral64(Val,
1157                                      AsmParser->hasInv2PiInlineImm())) {
1158       Inst.addOperand(MCOperand::createImm(Val));
1159       return;
1160     }
1161 
1162     Inst.addOperand(MCOperand::createImm(Lo_32(Val)));
1163     return;
1164   }
1165   case 2: {
1166     if (isInt<16>(Val) &&
1167         AMDGPU::isInlinableLiteral16(static_cast<int16_t>(Val),
1168                                      AsmParser->hasInv2PiInlineImm())) {
1169       Inst.addOperand(MCOperand::createImm(Val));
1170       return;
1171     }
1172 
1173     Inst.addOperand(MCOperand::createImm(Val & 0xffff));
1174     return;
1175   }
1176   default:
1177     llvm_unreachable("invalid operand size");
1178   }
1179 }
1180 
1181 template <unsigned Bitwidth>
1182 void AMDGPUOperand::addKImmFPOperands(MCInst &Inst, unsigned N) const {
1183   APInt Literal(64, Imm.Val);
1184 
1185   if (!Imm.IsFPImm) {
1186     // We got int literal token.
1187     Inst.addOperand(MCOperand::createImm(Literal.getLoBits(Bitwidth).getZExtValue()));
1188     return;
1189   }
1190 
1191   bool Lost;
1192   APFloat FPLiteral(APFloat::IEEEdouble(), Literal);
1193   FPLiteral.convert(*getFltSemantics(Bitwidth / 8),
1194                     APFloat::rmNearestTiesToEven, &Lost);
1195   Inst.addOperand(MCOperand::createImm(FPLiteral.bitcastToAPInt().getZExtValue()));
1196 }
1197 
1198 void AMDGPUOperand::addRegOperands(MCInst &Inst, unsigned N) const {
1199   Inst.addOperand(MCOperand::createReg(AMDGPU::getMCReg(getReg(), AsmParser->getSTI())));
1200 }
1201 
1202 //===----------------------------------------------------------------------===//
1203 // AsmParser
1204 //===----------------------------------------------------------------------===//
1205 
1206 static int getRegClass(RegisterKind Is, unsigned RegWidth) {
1207   if (Is == IS_VGPR) {
1208     switch (RegWidth) {
1209       default: return -1;
1210       case 1: return AMDGPU::VGPR_32RegClassID;
1211       case 2: return AMDGPU::VReg_64RegClassID;
1212       case 3: return AMDGPU::VReg_96RegClassID;
1213       case 4: return AMDGPU::VReg_128RegClassID;
1214       case 8: return AMDGPU::VReg_256RegClassID;
1215       case 16: return AMDGPU::VReg_512RegClassID;
1216     }
1217   } else if (Is == IS_TTMP) {
1218     switch (RegWidth) {
1219       default: return -1;
1220       case 1: return AMDGPU::TTMP_32RegClassID;
1221       case 2: return AMDGPU::TTMP_64RegClassID;
1222       case 4: return AMDGPU::TTMP_128RegClassID;
1223     }
1224   } else if (Is == IS_SGPR) {
1225     switch (RegWidth) {
1226       default: return -1;
1227       case 1: return AMDGPU::SGPR_32RegClassID;
1228       case 2: return AMDGPU::SGPR_64RegClassID;
1229       case 4: return AMDGPU::SGPR_128RegClassID;
1230       case 8: return AMDGPU::SReg_256RegClassID;
1231       case 16: return AMDGPU::SReg_512RegClassID;
1232     }
1233   }
1234   return -1;
1235 }
1236 
1237 static unsigned getSpecialRegForName(StringRef RegName) {
1238   return StringSwitch<unsigned>(RegName)
1239     .Case("exec", AMDGPU::EXEC)
1240     .Case("vcc", AMDGPU::VCC)
1241     .Case("flat_scratch", AMDGPU::FLAT_SCR)
1242     .Case("m0", AMDGPU::M0)
1243     .Case("scc", AMDGPU::SCC)
1244     .Case("tba", AMDGPU::TBA)
1245     .Case("tma", AMDGPU::TMA)
1246     .Case("flat_scratch_lo", AMDGPU::FLAT_SCR_LO)
1247     .Case("flat_scratch_hi", AMDGPU::FLAT_SCR_HI)
1248     .Case("vcc_lo", AMDGPU::VCC_LO)
1249     .Case("vcc_hi", AMDGPU::VCC_HI)
1250     .Case("exec_lo", AMDGPU::EXEC_LO)
1251     .Case("exec_hi", AMDGPU::EXEC_HI)
1252     .Case("tma_lo", AMDGPU::TMA_LO)
1253     .Case("tma_hi", AMDGPU::TMA_HI)
1254     .Case("tba_lo", AMDGPU::TBA_LO)
1255     .Case("tba_hi", AMDGPU::TBA_HI)
1256     .Default(0);
1257 }
1258 
1259 bool AMDGPUAsmParser::ParseRegister(unsigned &RegNo, SMLoc &StartLoc, SMLoc &EndLoc) {
1260   auto R = parseRegister();
1261   if (!R) return true;
1262   assert(R->isReg());
1263   RegNo = R->getReg();
1264   StartLoc = R->getStartLoc();
1265   EndLoc = R->getEndLoc();
1266   return false;
1267 }
1268 
1269 bool AMDGPUAsmParser::AddNextRegisterToList(unsigned& Reg, unsigned& RegWidth, RegisterKind RegKind, unsigned Reg1, unsigned RegNum)
1270 {
1271   switch (RegKind) {
1272   case IS_SPECIAL:
1273     if (Reg == AMDGPU::EXEC_LO && Reg1 == AMDGPU::EXEC_HI) { Reg = AMDGPU::EXEC; RegWidth = 2; return true; }
1274     if (Reg == AMDGPU::FLAT_SCR_LO && Reg1 == AMDGPU::FLAT_SCR_HI) { Reg = AMDGPU::FLAT_SCR; RegWidth = 2; return true; }
1275     if (Reg == AMDGPU::VCC_LO && Reg1 == AMDGPU::VCC_HI) { Reg = AMDGPU::VCC; RegWidth = 2; return true; }
1276     if (Reg == AMDGPU::TBA_LO && Reg1 == AMDGPU::TBA_HI) { Reg = AMDGPU::TBA; RegWidth = 2; return true; }
1277     if (Reg == AMDGPU::TMA_LO && Reg1 == AMDGPU::TMA_HI) { Reg = AMDGPU::TMA; RegWidth = 2; return true; }
1278     return false;
1279   case IS_VGPR:
1280   case IS_SGPR:
1281   case IS_TTMP:
1282     if (Reg1 != Reg + RegWidth) { return false; }
1283     RegWidth++;
1284     return true;
1285   default:
1286     llvm_unreachable("unexpected register kind");
1287   }
1288 }
1289 
1290 bool AMDGPUAsmParser::ParseAMDGPURegister(RegisterKind& RegKind, unsigned& Reg, unsigned& RegNum, unsigned& RegWidth, unsigned *DwordRegIndex)
1291 {
1292   if (DwordRegIndex) { *DwordRegIndex = 0; }
1293   const MCRegisterInfo *TRI = getContext().getRegisterInfo();
1294   if (getLexer().is(AsmToken::Identifier)) {
1295     StringRef RegName = Parser.getTok().getString();
1296     if ((Reg = getSpecialRegForName(RegName))) {
1297       Parser.Lex();
1298       RegKind = IS_SPECIAL;
1299     } else {
1300       unsigned RegNumIndex = 0;
1301       if (RegName[0] == 'v') {
1302         RegNumIndex = 1;
1303         RegKind = IS_VGPR;
1304       } else if (RegName[0] == 's') {
1305         RegNumIndex = 1;
1306         RegKind = IS_SGPR;
1307       } else if (RegName.startswith("ttmp")) {
1308         RegNumIndex = strlen("ttmp");
1309         RegKind = IS_TTMP;
1310       } else {
1311         return false;
1312       }
1313       if (RegName.size() > RegNumIndex) {
1314         // Single 32-bit register: vXX.
1315         if (RegName.substr(RegNumIndex).getAsInteger(10, RegNum))
1316           return false;
1317         Parser.Lex();
1318         RegWidth = 1;
1319       } else {
1320         // Range of registers: v[XX:YY]. ":YY" is optional.
1321         Parser.Lex();
1322         int64_t RegLo, RegHi;
1323         if (getLexer().isNot(AsmToken::LBrac))
1324           return false;
1325         Parser.Lex();
1326 
1327         if (getParser().parseAbsoluteExpression(RegLo))
1328           return false;
1329 
1330         const bool isRBrace = getLexer().is(AsmToken::RBrac);
1331         if (!isRBrace && getLexer().isNot(AsmToken::Colon))
1332           return false;
1333         Parser.Lex();
1334 
1335         if (isRBrace) {
1336           RegHi = RegLo;
1337         } else {
1338           if (getParser().parseAbsoluteExpression(RegHi))
1339             return false;
1340 
1341           if (getLexer().isNot(AsmToken::RBrac))
1342             return false;
1343           Parser.Lex();
1344         }
1345         RegNum = (unsigned) RegLo;
1346         RegWidth = (RegHi - RegLo) + 1;
1347       }
1348     }
1349   } else if (getLexer().is(AsmToken::LBrac)) {
1350     // List of consecutive registers: [s0,s1,s2,s3]
1351     Parser.Lex();
1352     if (!ParseAMDGPURegister(RegKind, Reg, RegNum, RegWidth, nullptr))
1353       return false;
1354     if (RegWidth != 1)
1355       return false;
1356     RegisterKind RegKind1;
1357     unsigned Reg1, RegNum1, RegWidth1;
1358     do {
1359       if (getLexer().is(AsmToken::Comma)) {
1360         Parser.Lex();
1361       } else if (getLexer().is(AsmToken::RBrac)) {
1362         Parser.Lex();
1363         break;
1364       } else if (ParseAMDGPURegister(RegKind1, Reg1, RegNum1, RegWidth1, nullptr)) {
1365         if (RegWidth1 != 1) {
1366           return false;
1367         }
1368         if (RegKind1 != RegKind) {
1369           return false;
1370         }
1371         if (!AddNextRegisterToList(Reg, RegWidth, RegKind1, Reg1, RegNum1)) {
1372           return false;
1373         }
1374       } else {
1375         return false;
1376       }
1377     } while (true);
1378   } else {
1379     return false;
1380   }
1381   switch (RegKind) {
1382   case IS_SPECIAL:
1383     RegNum = 0;
1384     RegWidth = 1;
1385     break;
1386   case IS_VGPR:
1387   case IS_SGPR:
1388   case IS_TTMP:
1389   {
1390     unsigned Size = 1;
1391     if (RegKind == IS_SGPR || RegKind == IS_TTMP) {
1392       // SGPR and TTMP registers must be aligned. Max required alignment is 4 dwords.
1393       Size = std::min(RegWidth, 4u);
1394     }
1395     if (RegNum % Size != 0)
1396       return false;
1397     if (DwordRegIndex) { *DwordRegIndex = RegNum; }
1398     RegNum = RegNum / Size;
1399     int RCID = getRegClass(RegKind, RegWidth);
1400     if (RCID == -1)
1401       return false;
1402     const MCRegisterClass RC = TRI->getRegClass(RCID);
1403     if (RegNum >= RC.getNumRegs())
1404       return false;
1405     Reg = RC.getRegister(RegNum);
1406     break;
1407   }
1408 
1409   default:
1410     llvm_unreachable("unexpected register kind");
1411   }
1412 
1413   if (!subtargetHasRegister(*TRI, Reg))
1414     return false;
1415   return true;
1416 }
1417 
1418 std::unique_ptr<AMDGPUOperand> AMDGPUAsmParser::parseRegister() {
1419   const auto &Tok = Parser.getTok();
1420   SMLoc StartLoc = Tok.getLoc();
1421   SMLoc EndLoc = Tok.getEndLoc();
1422   RegisterKind RegKind;
1423   unsigned Reg, RegNum, RegWidth, DwordRegIndex;
1424 
1425   if (!ParseAMDGPURegister(RegKind, Reg, RegNum, RegWidth, &DwordRegIndex)) {
1426     return nullptr;
1427   }
1428   KernelScope.usesRegister(RegKind, DwordRegIndex, RegWidth);
1429   return AMDGPUOperand::CreateReg(this, Reg, StartLoc, EndLoc, false);
1430 }
1431 
1432 OperandMatchResultTy
1433 AMDGPUAsmParser::parseImm(OperandVector &Operands) {
1434   // TODO: add syntactic sugar for 1/(2*PI)
1435   bool Minus = false;
1436   if (getLexer().getKind() == AsmToken::Minus) {
1437     Minus = true;
1438     Parser.Lex();
1439   }
1440 
1441   SMLoc S = Parser.getTok().getLoc();
1442   switch(getLexer().getKind()) {
1443   case AsmToken::Integer: {
1444     int64_t IntVal;
1445     if (getParser().parseAbsoluteExpression(IntVal))
1446       return MatchOperand_ParseFail;
1447     if (Minus)
1448       IntVal *= -1;
1449     Operands.push_back(AMDGPUOperand::CreateImm(this, IntVal, S));
1450     return MatchOperand_Success;
1451   }
1452   case AsmToken::Real: {
1453     int64_t IntVal;
1454     if (getParser().parseAbsoluteExpression(IntVal))
1455       return MatchOperand_ParseFail;
1456 
1457     APFloat F(BitsToDouble(IntVal));
1458     if (Minus)
1459       F.changeSign();
1460     Operands.push_back(
1461         AMDGPUOperand::CreateImm(this, F.bitcastToAPInt().getZExtValue(), S,
1462                                  AMDGPUOperand::ImmTyNone, true));
1463     return MatchOperand_Success;
1464   }
1465   default:
1466     return Minus ? MatchOperand_ParseFail : MatchOperand_NoMatch;
1467   }
1468 }
1469 
1470 OperandMatchResultTy
1471 AMDGPUAsmParser::parseRegOrImm(OperandVector &Operands) {
1472   auto res = parseImm(Operands);
1473   if (res != MatchOperand_NoMatch) {
1474     return res;
1475   }
1476 
1477   if (auto R = parseRegister()) {
1478     assert(R->isReg());
1479     R->Reg.IsForcedVOP3 = isForcedVOP3();
1480     Operands.push_back(std::move(R));
1481     return MatchOperand_Success;
1482   }
1483   return MatchOperand_ParseFail;
1484 }
1485 
1486 OperandMatchResultTy
1487 AMDGPUAsmParser::parseRegOrImmWithFPInputMods(OperandVector &Operands) {
1488   // XXX: During parsing we can't determine if minus sign means
1489   // negate-modifier or negative immediate value.
1490   // By default we suppose it is modifier.
1491   bool Negate = false, Abs = false, Abs2 = false;
1492 
1493   if (getLexer().getKind()== AsmToken::Minus) {
1494     Parser.Lex();
1495     Negate = true;
1496   }
1497 
1498   if (getLexer().getKind() == AsmToken::Identifier && Parser.getTok().getString() == "abs") {
1499     Parser.Lex();
1500     Abs2 = true;
1501     if (getLexer().isNot(AsmToken::LParen)) {
1502       Error(Parser.getTok().getLoc(), "expected left paren after abs");
1503       return MatchOperand_ParseFail;
1504     }
1505     Parser.Lex();
1506   }
1507 
1508   if (getLexer().getKind() == AsmToken::Pipe) {
1509     if (Abs2) {
1510       Error(Parser.getTok().getLoc(), "expected register or immediate");
1511       return MatchOperand_ParseFail;
1512     }
1513     Parser.Lex();
1514     Abs = true;
1515   }
1516 
1517   auto Res = parseRegOrImm(Operands);
1518   if (Res != MatchOperand_Success) {
1519     return Res;
1520   }
1521 
1522   AMDGPUOperand::Modifiers Mods;
1523   if (Negate) {
1524     Mods.Neg = true;
1525   }
1526   if (Abs) {
1527     if (getLexer().getKind() != AsmToken::Pipe) {
1528       Error(Parser.getTok().getLoc(), "expected vertical bar");
1529       return MatchOperand_ParseFail;
1530     }
1531     Parser.Lex();
1532     Mods.Abs = true;
1533   }
1534   if (Abs2) {
1535     if (getLexer().isNot(AsmToken::RParen)) {
1536       Error(Parser.getTok().getLoc(), "expected closing parentheses");
1537       return MatchOperand_ParseFail;
1538     }
1539     Parser.Lex();
1540     Mods.Abs = true;
1541   }
1542 
1543   if (Mods.hasFPModifiers()) {
1544     AMDGPUOperand &Op = static_cast<AMDGPUOperand &>(*Operands.back());
1545     Op.setModifiers(Mods);
1546   }
1547   return MatchOperand_Success;
1548 }
1549 
1550 OperandMatchResultTy
1551 AMDGPUAsmParser::parseRegOrImmWithIntInputMods(OperandVector &Operands) {
1552   bool Sext = false;
1553 
1554   if (getLexer().getKind() == AsmToken::Identifier && Parser.getTok().getString() == "sext") {
1555     Parser.Lex();
1556     Sext = true;
1557     if (getLexer().isNot(AsmToken::LParen)) {
1558       Error(Parser.getTok().getLoc(), "expected left paren after sext");
1559       return MatchOperand_ParseFail;
1560     }
1561     Parser.Lex();
1562   }
1563 
1564   auto Res = parseRegOrImm(Operands);
1565   if (Res != MatchOperand_Success) {
1566     return Res;
1567   }
1568 
1569   AMDGPUOperand::Modifiers Mods;
1570   if (Sext) {
1571     if (getLexer().isNot(AsmToken::RParen)) {
1572       Error(Parser.getTok().getLoc(), "expected closing parentheses");
1573       return MatchOperand_ParseFail;
1574     }
1575     Parser.Lex();
1576     Mods.Sext = true;
1577   }
1578 
1579   if (Mods.hasIntModifiers()) {
1580     AMDGPUOperand &Op = static_cast<AMDGPUOperand &>(*Operands.back());
1581     Op.setModifiers(Mods);
1582   }
1583 
1584   return MatchOperand_Success;
1585 }
1586 
1587 OperandMatchResultTy AMDGPUAsmParser::parseVReg32OrOff(OperandVector &Operands) {
1588   std::unique_ptr<AMDGPUOperand> Reg = parseRegister();
1589   if (Reg) {
1590     Operands.push_back(std::move(Reg));
1591     return MatchOperand_Success;
1592   }
1593 
1594   const AsmToken &Tok = Parser.getTok();
1595   if (Tok.getString() == "off") {
1596     Operands.push_back(AMDGPUOperand::CreateImm(this, 0, Tok.getLoc(),
1597                                                 AMDGPUOperand::ImmTyOff, false));
1598     Parser.Lex();
1599     return MatchOperand_Success;
1600   }
1601 
1602   return MatchOperand_NoMatch;
1603 }
1604 
1605 unsigned AMDGPUAsmParser::checkTargetMatchPredicate(MCInst &Inst) {
1606 
1607   uint64_t TSFlags = MII.get(Inst.getOpcode()).TSFlags;
1608 
1609   if ((getForcedEncodingSize() == 32 && (TSFlags & SIInstrFlags::VOP3)) ||
1610       (getForcedEncodingSize() == 64 && !(TSFlags & SIInstrFlags::VOP3)) ||
1611       (isForcedDPP() && !(TSFlags & SIInstrFlags::DPP)) ||
1612       (isForcedSDWA() && !(TSFlags & SIInstrFlags::SDWA)) )
1613     return Match_InvalidOperand;
1614 
1615   if ((TSFlags & SIInstrFlags::VOP3) &&
1616       (TSFlags & SIInstrFlags::VOPAsmPrefer32Bit) &&
1617       getForcedEncodingSize() != 64)
1618     return Match_PreferE32;
1619 
1620   if (Inst.getOpcode() == AMDGPU::V_MAC_F32_sdwa_vi ||
1621       Inst.getOpcode() == AMDGPU::V_MAC_F16_sdwa_vi) {
1622     // v_mac_f32/16 allow only dst_sel == DWORD;
1623     auto OpNum =
1624         AMDGPU::getNamedOperandIdx(Inst.getOpcode(), AMDGPU::OpName::dst_sel);
1625     const auto &Op = Inst.getOperand(OpNum);
1626     if (!Op.isImm() || Op.getImm() != AMDGPU::SDWA::SdwaSel::DWORD) {
1627       return Match_InvalidOperand;
1628     }
1629   }
1630 
1631   return Match_Success;
1632 }
1633 
1634 // What asm variants we should check
1635 ArrayRef<unsigned> AMDGPUAsmParser::getMatchedVariants() const {
1636   if (getForcedEncodingSize() == 32) {
1637     static const unsigned Variants[] = {AMDGPUAsmVariants::DEFAULT};
1638     return makeArrayRef(Variants);
1639   }
1640 
1641   if (isForcedVOP3()) {
1642     static const unsigned Variants[] = {AMDGPUAsmVariants::VOP3};
1643     return makeArrayRef(Variants);
1644   }
1645 
1646   if (isForcedSDWA()) {
1647     static const unsigned Variants[] = {AMDGPUAsmVariants::SDWA};
1648     return makeArrayRef(Variants);
1649   }
1650 
1651   if (isForcedDPP()) {
1652     static const unsigned Variants[] = {AMDGPUAsmVariants::DPP};
1653     return makeArrayRef(Variants);
1654   }
1655 
1656   static const unsigned Variants[] = {
1657     AMDGPUAsmVariants::DEFAULT, AMDGPUAsmVariants::VOP3,
1658     AMDGPUAsmVariants::SDWA, AMDGPUAsmVariants::DPP
1659   };
1660 
1661   return makeArrayRef(Variants);
1662 }
1663 
1664 bool AMDGPUAsmParser::MatchAndEmitInstruction(SMLoc IDLoc, unsigned &Opcode,
1665                                               OperandVector &Operands,
1666                                               MCStreamer &Out,
1667                                               uint64_t &ErrorInfo,
1668                                               bool MatchingInlineAsm) {
1669   MCInst Inst;
1670   unsigned Result = Match_Success;
1671   for (auto Variant : getMatchedVariants()) {
1672     uint64_t EI;
1673     auto R = MatchInstructionImpl(Operands, Inst, EI, MatchingInlineAsm,
1674                                   Variant);
1675     // We order match statuses from least to most specific. We use most specific
1676     // status as resulting
1677     // Match_MnemonicFail < Match_InvalidOperand < Match_MissingFeature < Match_PreferE32
1678     if ((R == Match_Success) ||
1679         (R == Match_PreferE32) ||
1680         (R == Match_MissingFeature && Result != Match_PreferE32) ||
1681         (R == Match_InvalidOperand && Result != Match_MissingFeature
1682                                    && Result != Match_PreferE32) ||
1683         (R == Match_MnemonicFail   && Result != Match_InvalidOperand
1684                                    && Result != Match_MissingFeature
1685                                    && Result != Match_PreferE32)) {
1686       Result = R;
1687       ErrorInfo = EI;
1688     }
1689     if (R == Match_Success)
1690       break;
1691   }
1692 
1693   switch (Result) {
1694   default: break;
1695   case Match_Success:
1696     Inst.setLoc(IDLoc);
1697     Out.EmitInstruction(Inst, getSTI());
1698     return false;
1699 
1700   case Match_MissingFeature:
1701     return Error(IDLoc, "instruction not supported on this GPU");
1702 
1703   case Match_MnemonicFail:
1704     return Error(IDLoc, "unrecognized instruction mnemonic");
1705 
1706   case Match_InvalidOperand: {
1707     SMLoc ErrorLoc = IDLoc;
1708     if (ErrorInfo != ~0ULL) {
1709       if (ErrorInfo >= Operands.size()) {
1710         return Error(IDLoc, "too few operands for instruction");
1711       }
1712       ErrorLoc = ((AMDGPUOperand &)*Operands[ErrorInfo]).getStartLoc();
1713       if (ErrorLoc == SMLoc())
1714         ErrorLoc = IDLoc;
1715     }
1716     return Error(ErrorLoc, "invalid operand for instruction");
1717   }
1718 
1719   case Match_PreferE32:
1720     return Error(IDLoc, "internal error: instruction without _e64 suffix "
1721                         "should be encoded as e32");
1722   }
1723   llvm_unreachable("Implement any new match types added!");
1724 }
1725 
1726 bool AMDGPUAsmParser::ParseAsAbsoluteExpression(uint32_t &Ret) {
1727   int64_t Tmp = -1;
1728   if (getLexer().isNot(AsmToken::Integer) && getLexer().isNot(AsmToken::Identifier)) {
1729     return true;
1730   }
1731   if (getParser().parseAbsoluteExpression(Tmp)) {
1732     return true;
1733   }
1734   Ret = static_cast<uint32_t>(Tmp);
1735   return false;
1736 }
1737 
1738 
1739 bool AMDGPUAsmParser::ParseDirectiveMajorMinor(uint32_t &Major,
1740                                                uint32_t &Minor) {
1741   if (ParseAsAbsoluteExpression(Major))
1742     return TokError("invalid major version");
1743 
1744   if (getLexer().isNot(AsmToken::Comma))
1745     return TokError("minor version number required, comma expected");
1746   Lex();
1747 
1748   if (ParseAsAbsoluteExpression(Minor))
1749     return TokError("invalid minor version");
1750 
1751   return false;
1752 }
1753 
1754 bool AMDGPUAsmParser::ParseDirectiveHSACodeObjectVersion() {
1755 
1756   uint32_t Major;
1757   uint32_t Minor;
1758 
1759   if (ParseDirectiveMajorMinor(Major, Minor))
1760     return true;
1761 
1762   getTargetStreamer().EmitDirectiveHSACodeObjectVersion(Major, Minor);
1763   return false;
1764 }
1765 
1766 bool AMDGPUAsmParser::ParseDirectiveHSACodeObjectISA() {
1767   uint32_t Major;
1768   uint32_t Minor;
1769   uint32_t Stepping;
1770   StringRef VendorName;
1771   StringRef ArchName;
1772 
1773   // If this directive has no arguments, then use the ISA version for the
1774   // targeted GPU.
1775   if (getLexer().is(AsmToken::EndOfStatement)) {
1776     AMDGPU::IsaVersion Isa = AMDGPU::getIsaVersion(getSTI().getFeatureBits());
1777     getTargetStreamer().EmitDirectiveHSACodeObjectISA(Isa.Major, Isa.Minor,
1778                                                       Isa.Stepping,
1779                                                       "AMD", "AMDGPU");
1780     return false;
1781   }
1782 
1783   if (ParseDirectiveMajorMinor(Major, Minor))
1784     return true;
1785 
1786   if (getLexer().isNot(AsmToken::Comma))
1787     return TokError("stepping version number required, comma expected");
1788   Lex();
1789 
1790   if (ParseAsAbsoluteExpression(Stepping))
1791     return TokError("invalid stepping version");
1792 
1793   if (getLexer().isNot(AsmToken::Comma))
1794     return TokError("vendor name required, comma expected");
1795   Lex();
1796 
1797   if (getLexer().isNot(AsmToken::String))
1798     return TokError("invalid vendor name");
1799 
1800   VendorName = getLexer().getTok().getStringContents();
1801   Lex();
1802 
1803   if (getLexer().isNot(AsmToken::Comma))
1804     return TokError("arch name required, comma expected");
1805   Lex();
1806 
1807   if (getLexer().isNot(AsmToken::String))
1808     return TokError("invalid arch name");
1809 
1810   ArchName = getLexer().getTok().getStringContents();
1811   Lex();
1812 
1813   getTargetStreamer().EmitDirectiveHSACodeObjectISA(Major, Minor, Stepping,
1814                                                     VendorName, ArchName);
1815   return false;
1816 }
1817 
1818 bool AMDGPUAsmParser::ParseDirectiveRuntimeMetadata() {
1819   std::string Metadata;
1820   raw_string_ostream MS(Metadata);
1821 
1822   getLexer().setSkipSpace(false);
1823 
1824   bool FoundEnd = false;
1825   while (!getLexer().is(AsmToken::Eof)) {
1826     while (getLexer().is(AsmToken::Space)) {
1827       MS << ' ';
1828       Lex();
1829     }
1830 
1831     if (getLexer().is(AsmToken::Identifier)) {
1832       StringRef ID = getLexer().getTok().getIdentifier();
1833       if (ID == ".end_amdgpu_runtime_metadata") {
1834         Lex();
1835         FoundEnd = true;
1836         break;
1837       }
1838     }
1839 
1840     MS << Parser.parseStringToEndOfStatement()
1841        << getContext().getAsmInfo()->getSeparatorString();
1842 
1843     Parser.eatToEndOfStatement();
1844   }
1845 
1846   getLexer().setSkipSpace(true);
1847 
1848   if (getLexer().is(AsmToken::Eof) && !FoundEnd)
1849     return TokError("expected directive .end_amdgpu_runtime_metadata not found");
1850 
1851   MS.flush();
1852 
1853   getTargetStreamer().EmitRuntimeMetadata(Metadata);
1854 
1855   return false;
1856 }
1857 
1858 bool AMDGPUAsmParser::ParseAMDKernelCodeTValue(StringRef ID,
1859                                                amd_kernel_code_t &Header) {
1860   SmallString<40> ErrStr;
1861   raw_svector_ostream Err(ErrStr);
1862   if (!parseAmdKernelCodeField(ID, getParser(), Header, Err)) {
1863     return TokError(Err.str());
1864   }
1865   Lex();
1866   return false;
1867 }
1868 
1869 bool AMDGPUAsmParser::ParseDirectiveAMDKernelCodeT() {
1870   amd_kernel_code_t Header;
1871   AMDGPU::initDefaultAMDKernelCodeT(Header, getSTI().getFeatureBits());
1872 
1873   while (true) {
1874     // Lex EndOfStatement.  This is in a while loop, because lexing a comment
1875     // will set the current token to EndOfStatement.
1876     while(getLexer().is(AsmToken::EndOfStatement))
1877       Lex();
1878 
1879     if (getLexer().isNot(AsmToken::Identifier))
1880       return TokError("expected value identifier or .end_amd_kernel_code_t");
1881 
1882     StringRef ID = getLexer().getTok().getIdentifier();
1883     Lex();
1884 
1885     if (ID == ".end_amd_kernel_code_t")
1886       break;
1887 
1888     if (ParseAMDKernelCodeTValue(ID, Header))
1889       return true;
1890   }
1891 
1892   getTargetStreamer().EmitAMDKernelCodeT(Header);
1893 
1894   return false;
1895 }
1896 
1897 bool AMDGPUAsmParser::ParseSectionDirectiveHSAText() {
1898   getParser().getStreamer().SwitchSection(
1899       AMDGPU::getHSATextSection(getContext()));
1900   return false;
1901 }
1902 
1903 bool AMDGPUAsmParser::ParseDirectiveAMDGPUHsaKernel() {
1904   if (getLexer().isNot(AsmToken::Identifier))
1905     return TokError("expected symbol name");
1906 
1907   StringRef KernelName = Parser.getTok().getString();
1908 
1909   getTargetStreamer().EmitAMDGPUSymbolType(KernelName,
1910                                            ELF::STT_AMDGPU_HSA_KERNEL);
1911   Lex();
1912   KernelScope.initialize(getContext());
1913   return false;
1914 }
1915 
1916 bool AMDGPUAsmParser::ParseDirectiveAMDGPUHsaModuleGlobal() {
1917   if (getLexer().isNot(AsmToken::Identifier))
1918     return TokError("expected symbol name");
1919 
1920   StringRef GlobalName = Parser.getTok().getIdentifier();
1921 
1922   getTargetStreamer().EmitAMDGPUHsaModuleScopeGlobal(GlobalName);
1923   Lex();
1924   return false;
1925 }
1926 
1927 bool AMDGPUAsmParser::ParseDirectiveAMDGPUHsaProgramGlobal() {
1928   if (getLexer().isNot(AsmToken::Identifier))
1929     return TokError("expected symbol name");
1930 
1931   StringRef GlobalName = Parser.getTok().getIdentifier();
1932 
1933   getTargetStreamer().EmitAMDGPUHsaProgramScopeGlobal(GlobalName);
1934   Lex();
1935   return false;
1936 }
1937 
1938 bool AMDGPUAsmParser::ParseSectionDirectiveHSADataGlobalAgent() {
1939   getParser().getStreamer().SwitchSection(
1940       AMDGPU::getHSADataGlobalAgentSection(getContext()));
1941   return false;
1942 }
1943 
1944 bool AMDGPUAsmParser::ParseSectionDirectiveHSADataGlobalProgram() {
1945   getParser().getStreamer().SwitchSection(
1946       AMDGPU::getHSADataGlobalProgramSection(getContext()));
1947   return false;
1948 }
1949 
1950 bool AMDGPUAsmParser::ParseSectionDirectiveHSARodataReadonlyAgent() {
1951   getParser().getStreamer().SwitchSection(
1952       AMDGPU::getHSARodataReadonlyAgentSection(getContext()));
1953   return false;
1954 }
1955 
1956 bool AMDGPUAsmParser::ParseDirective(AsmToken DirectiveID) {
1957   StringRef IDVal = DirectiveID.getString();
1958 
1959   if (IDVal == ".hsa_code_object_version")
1960     return ParseDirectiveHSACodeObjectVersion();
1961 
1962   if (IDVal == ".hsa_code_object_isa")
1963     return ParseDirectiveHSACodeObjectISA();
1964 
1965   if (IDVal == ".amdgpu_runtime_metadata")
1966     return ParseDirectiveRuntimeMetadata();
1967 
1968   if (IDVal == ".amd_kernel_code_t")
1969     return ParseDirectiveAMDKernelCodeT();
1970 
1971   if (IDVal == ".hsatext")
1972     return ParseSectionDirectiveHSAText();
1973 
1974   if (IDVal == ".amdgpu_hsa_kernel")
1975     return ParseDirectiveAMDGPUHsaKernel();
1976 
1977   if (IDVal == ".amdgpu_hsa_module_global")
1978     return ParseDirectiveAMDGPUHsaModuleGlobal();
1979 
1980   if (IDVal == ".amdgpu_hsa_program_global")
1981     return ParseDirectiveAMDGPUHsaProgramGlobal();
1982 
1983   if (IDVal == ".hsadata_global_agent")
1984     return ParseSectionDirectiveHSADataGlobalAgent();
1985 
1986   if (IDVal == ".hsadata_global_program")
1987     return ParseSectionDirectiveHSADataGlobalProgram();
1988 
1989   if (IDVal == ".hsarodata_readonly_agent")
1990     return ParseSectionDirectiveHSARodataReadonlyAgent();
1991 
1992   return true;
1993 }
1994 
1995 bool AMDGPUAsmParser::subtargetHasRegister(const MCRegisterInfo &MRI,
1996                                            unsigned RegNo) const {
1997   if (isCI())
1998     return true;
1999 
2000   if (isSI()) {
2001     // No flat_scr
2002     switch (RegNo) {
2003     case AMDGPU::FLAT_SCR:
2004     case AMDGPU::FLAT_SCR_LO:
2005     case AMDGPU::FLAT_SCR_HI:
2006       return false;
2007     default:
2008       return true;
2009     }
2010   }
2011 
2012   // VI only has 102 SGPRs, so make sure we aren't trying to use the 2 more that
2013   // SI/CI have.
2014   for (MCRegAliasIterator R(AMDGPU::SGPR102_SGPR103, &MRI, true);
2015        R.isValid(); ++R) {
2016     if (*R == RegNo)
2017       return false;
2018   }
2019 
2020   return true;
2021 }
2022 
2023 OperandMatchResultTy
2024 AMDGPUAsmParser::parseOperand(OperandVector &Operands, StringRef Mnemonic) {
2025 
2026   // Try to parse with a custom parser
2027   OperandMatchResultTy ResTy = MatchOperandParserImpl(Operands, Mnemonic);
2028 
2029   // If we successfully parsed the operand or if there as an error parsing,
2030   // we are done.
2031   //
2032   // If we are parsing after we reach EndOfStatement then this means we
2033   // are appending default values to the Operands list.  This is only done
2034   // by custom parser, so we shouldn't continue on to the generic parsing.
2035   if (ResTy == MatchOperand_Success || ResTy == MatchOperand_ParseFail ||
2036       getLexer().is(AsmToken::EndOfStatement))
2037     return ResTy;
2038 
2039   ResTy = parseRegOrImm(Operands);
2040 
2041   if (ResTy == MatchOperand_Success)
2042     return ResTy;
2043 
2044   if (getLexer().getKind() == AsmToken::Identifier) {
2045     // If this identifier is a symbol, we want to create an expression for it.
2046     // It is a little difficult to distinguish between a symbol name, and
2047     // an instruction flag like 'gds'.  In order to do this, we parse
2048     // all tokens as expressions and then treate the symbol name as the token
2049     // string when we want to interpret the operand as a token.
2050     const auto &Tok = Parser.getTok();
2051     SMLoc S = Tok.getLoc();
2052     const MCExpr *Expr = nullptr;
2053     if (!Parser.parseExpression(Expr)) {
2054       Operands.push_back(AMDGPUOperand::CreateExpr(this, Expr, S));
2055       return MatchOperand_Success;
2056     }
2057 
2058     Operands.push_back(AMDGPUOperand::CreateToken(this, Tok.getString(), Tok.getLoc()));
2059     Parser.Lex();
2060     return MatchOperand_Success;
2061   }
2062   return MatchOperand_NoMatch;
2063 }
2064 
2065 StringRef AMDGPUAsmParser::parseMnemonicSuffix(StringRef Name) {
2066   // Clear any forced encodings from the previous instruction.
2067   setForcedEncodingSize(0);
2068   setForcedDPP(false);
2069   setForcedSDWA(false);
2070 
2071   if (Name.endswith("_e64")) {
2072     setForcedEncodingSize(64);
2073     return Name.substr(0, Name.size() - 4);
2074   } else if (Name.endswith("_e32")) {
2075     setForcedEncodingSize(32);
2076     return Name.substr(0, Name.size() - 4);
2077   } else if (Name.endswith("_dpp")) {
2078     setForcedDPP(true);
2079     return Name.substr(0, Name.size() - 4);
2080   } else if (Name.endswith("_sdwa")) {
2081     setForcedSDWA(true);
2082     return Name.substr(0, Name.size() - 5);
2083   }
2084   return Name;
2085 }
2086 
2087 bool AMDGPUAsmParser::ParseInstruction(ParseInstructionInfo &Info,
2088                                        StringRef Name,
2089                                        SMLoc NameLoc, OperandVector &Operands) {
2090   // Add the instruction mnemonic
2091   Name = parseMnemonicSuffix(Name);
2092   Operands.push_back(AMDGPUOperand::CreateToken(this, Name, NameLoc));
2093 
2094   while (!getLexer().is(AsmToken::EndOfStatement)) {
2095     OperandMatchResultTy Res = parseOperand(Operands, Name);
2096 
2097     // Eat the comma or space if there is one.
2098     if (getLexer().is(AsmToken::Comma))
2099       Parser.Lex();
2100 
2101     switch (Res) {
2102       case MatchOperand_Success: break;
2103       case MatchOperand_ParseFail:
2104         Error(getLexer().getLoc(), "failed parsing operand.");
2105         while (!getLexer().is(AsmToken::EndOfStatement)) {
2106           Parser.Lex();
2107         }
2108         return true;
2109       case MatchOperand_NoMatch:
2110         Error(getLexer().getLoc(), "not a valid operand.");
2111         while (!getLexer().is(AsmToken::EndOfStatement)) {
2112           Parser.Lex();
2113         }
2114         return true;
2115     }
2116   }
2117 
2118   return false;
2119 }
2120 
2121 //===----------------------------------------------------------------------===//
2122 // Utility functions
2123 //===----------------------------------------------------------------------===//
2124 
2125 OperandMatchResultTy
2126 AMDGPUAsmParser::parseIntWithPrefix(const char *Prefix, int64_t &Int) {
2127   switch(getLexer().getKind()) {
2128     default: return MatchOperand_NoMatch;
2129     case AsmToken::Identifier: {
2130       StringRef Name = Parser.getTok().getString();
2131       if (!Name.equals(Prefix)) {
2132         return MatchOperand_NoMatch;
2133       }
2134 
2135       Parser.Lex();
2136       if (getLexer().isNot(AsmToken::Colon))
2137         return MatchOperand_ParseFail;
2138 
2139       Parser.Lex();
2140       if (getLexer().isNot(AsmToken::Integer))
2141         return MatchOperand_ParseFail;
2142 
2143       if (getParser().parseAbsoluteExpression(Int))
2144         return MatchOperand_ParseFail;
2145       break;
2146     }
2147   }
2148   return MatchOperand_Success;
2149 }
2150 
2151 OperandMatchResultTy
2152 AMDGPUAsmParser::parseIntWithPrefix(const char *Prefix, OperandVector &Operands,
2153                                     enum AMDGPUOperand::ImmTy ImmTy,
2154                                     bool (*ConvertResult)(int64_t&)) {
2155   SMLoc S = Parser.getTok().getLoc();
2156   int64_t Value = 0;
2157 
2158   OperandMatchResultTy Res = parseIntWithPrefix(Prefix, Value);
2159   if (Res != MatchOperand_Success)
2160     return Res;
2161 
2162   if (ConvertResult && !ConvertResult(Value)) {
2163     return MatchOperand_ParseFail;
2164   }
2165 
2166   Operands.push_back(AMDGPUOperand::CreateImm(this, Value, S, ImmTy));
2167   return MatchOperand_Success;
2168 }
2169 
2170 OperandMatchResultTy
2171 AMDGPUAsmParser::parseNamedBit(const char *Name, OperandVector &Operands,
2172                                enum AMDGPUOperand::ImmTy ImmTy) {
2173   int64_t Bit = 0;
2174   SMLoc S = Parser.getTok().getLoc();
2175 
2176   // We are at the end of the statement, and this is a default argument, so
2177   // use a default value.
2178   if (getLexer().isNot(AsmToken::EndOfStatement)) {
2179     switch(getLexer().getKind()) {
2180       case AsmToken::Identifier: {
2181         StringRef Tok = Parser.getTok().getString();
2182         if (Tok == Name) {
2183           Bit = 1;
2184           Parser.Lex();
2185         } else if (Tok.startswith("no") && Tok.endswith(Name)) {
2186           Bit = 0;
2187           Parser.Lex();
2188         } else {
2189           return MatchOperand_NoMatch;
2190         }
2191         break;
2192       }
2193       default:
2194         return MatchOperand_NoMatch;
2195     }
2196   }
2197 
2198   Operands.push_back(AMDGPUOperand::CreateImm(this, Bit, S, ImmTy));
2199   return MatchOperand_Success;
2200 }
2201 
2202 typedef std::map<enum AMDGPUOperand::ImmTy, unsigned> OptionalImmIndexMap;
2203 
2204 void addOptionalImmOperand(MCInst& Inst, const OperandVector& Operands,
2205                            OptionalImmIndexMap& OptionalIdx,
2206                            enum AMDGPUOperand::ImmTy ImmT, int64_t Default = 0) {
2207   auto i = OptionalIdx.find(ImmT);
2208   if (i != OptionalIdx.end()) {
2209     unsigned Idx = i->second;
2210     ((AMDGPUOperand &)*Operands[Idx]).addImmOperands(Inst, 1);
2211   } else {
2212     Inst.addOperand(MCOperand::createImm(Default));
2213   }
2214 }
2215 
2216 OperandMatchResultTy
2217 AMDGPUAsmParser::parseStringWithPrefix(StringRef Prefix, StringRef &Value) {
2218   if (getLexer().isNot(AsmToken::Identifier)) {
2219     return MatchOperand_NoMatch;
2220   }
2221   StringRef Tok = Parser.getTok().getString();
2222   if (Tok != Prefix) {
2223     return MatchOperand_NoMatch;
2224   }
2225 
2226   Parser.Lex();
2227   if (getLexer().isNot(AsmToken::Colon)) {
2228     return MatchOperand_ParseFail;
2229   }
2230 
2231   Parser.Lex();
2232   if (getLexer().isNot(AsmToken::Identifier)) {
2233     return MatchOperand_ParseFail;
2234   }
2235 
2236   Value = Parser.getTok().getString();
2237   return MatchOperand_Success;
2238 }
2239 
2240 //===----------------------------------------------------------------------===//
2241 // ds
2242 //===----------------------------------------------------------------------===//
2243 
2244 void AMDGPUAsmParser::cvtDSOffset01(MCInst &Inst,
2245                                     const OperandVector &Operands) {
2246   OptionalImmIndexMap OptionalIdx;
2247 
2248   for (unsigned i = 1, e = Operands.size(); i != e; ++i) {
2249     AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[i]);
2250 
2251     // Add the register arguments
2252     if (Op.isReg()) {
2253       Op.addRegOperands(Inst, 1);
2254       continue;
2255     }
2256 
2257     // Handle optional arguments
2258     OptionalIdx[Op.getImmTy()] = i;
2259   }
2260 
2261   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyOffset0);
2262   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyOffset1);
2263   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyGDS);
2264 
2265   Inst.addOperand(MCOperand::createReg(AMDGPU::M0)); // m0
2266 }
2267 
2268 void AMDGPUAsmParser::cvtDS(MCInst &Inst, const OperandVector &Operands) {
2269   std::map<enum AMDGPUOperand::ImmTy, unsigned> OptionalIdx;
2270   bool GDSOnly = false;
2271 
2272   for (unsigned i = 1, e = Operands.size(); i != e; ++i) {
2273     AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[i]);
2274 
2275     // Add the register arguments
2276     if (Op.isReg()) {
2277       Op.addRegOperands(Inst, 1);
2278       continue;
2279     }
2280 
2281     if (Op.isToken() && Op.getToken() == "gds") {
2282       GDSOnly = true;
2283       continue;
2284     }
2285 
2286     // Handle optional arguments
2287     OptionalIdx[Op.getImmTy()] = i;
2288   }
2289 
2290   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyOffset);
2291   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyGDS);
2292 
2293   if (!GDSOnly) {
2294     addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyGDS);
2295   }
2296   Inst.addOperand(MCOperand::createReg(AMDGPU::M0)); // m0
2297 }
2298 
2299 void AMDGPUAsmParser::cvtExp(MCInst &Inst, const OperandVector &Operands) {
2300   OptionalImmIndexMap OptionalIdx;
2301 
2302   unsigned EnMask = 0;
2303   int SrcIdx = 0;
2304 
2305   for (unsigned i = 1, e = Operands.size(); i != e; ++i) {
2306     AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[i]);
2307 
2308     // Add the register arguments
2309     if (Op.isReg()) {
2310       EnMask |= (1 << SrcIdx);
2311       Op.addRegOperands(Inst, 1);
2312       ++SrcIdx;
2313       continue;
2314     }
2315 
2316     if (Op.isOff()) {
2317       ++SrcIdx;
2318       Inst.addOperand(MCOperand::createReg(AMDGPU::NoRegister));
2319       continue;
2320     }
2321 
2322     if (Op.isImm() && Op.getImmTy() == AMDGPUOperand::ImmTyExpTgt) {
2323       Op.addImmOperands(Inst, 1);
2324       continue;
2325     }
2326 
2327     if (Op.isToken() && Op.getToken() == "done")
2328       continue;
2329 
2330     // Handle optional arguments
2331     OptionalIdx[Op.getImmTy()] = i;
2332   }
2333 
2334   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyExpVM);
2335   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyExpCompr);
2336 
2337   Inst.addOperand(MCOperand::createImm(EnMask));
2338 }
2339 
2340 //===----------------------------------------------------------------------===//
2341 // s_waitcnt
2342 //===----------------------------------------------------------------------===//
2343 
2344 bool AMDGPUAsmParser::parseCnt(int64_t &IntVal) {
2345   StringRef CntName = Parser.getTok().getString();
2346   int64_t CntVal;
2347 
2348   Parser.Lex();
2349   if (getLexer().isNot(AsmToken::LParen))
2350     return true;
2351 
2352   Parser.Lex();
2353   if (getLexer().isNot(AsmToken::Integer))
2354     return true;
2355 
2356   if (getParser().parseAbsoluteExpression(CntVal))
2357     return true;
2358 
2359   if (getLexer().isNot(AsmToken::RParen))
2360     return true;
2361 
2362   Parser.Lex();
2363   if (getLexer().is(AsmToken::Amp) || getLexer().is(AsmToken::Comma))
2364     Parser.Lex();
2365 
2366   IsaVersion IV = getIsaVersion(getSTI().getFeatureBits());
2367   if (CntName == "vmcnt")
2368     IntVal = encodeVmcnt(IV, IntVal, CntVal);
2369   else if (CntName == "expcnt")
2370     IntVal = encodeExpcnt(IV, IntVal, CntVal);
2371   else if (CntName == "lgkmcnt")
2372     IntVal = encodeLgkmcnt(IV, IntVal, CntVal);
2373   else
2374     return true;
2375 
2376   return false;
2377 }
2378 
2379 OperandMatchResultTy
2380 AMDGPUAsmParser::parseSWaitCntOps(OperandVector &Operands) {
2381   IsaVersion IV = getIsaVersion(getSTI().getFeatureBits());
2382   int64_t Waitcnt = getWaitcntBitMask(IV);
2383   SMLoc S = Parser.getTok().getLoc();
2384 
2385   switch(getLexer().getKind()) {
2386     default: return MatchOperand_ParseFail;
2387     case AsmToken::Integer:
2388       // The operand can be an integer value.
2389       if (getParser().parseAbsoluteExpression(Waitcnt))
2390         return MatchOperand_ParseFail;
2391       break;
2392 
2393     case AsmToken::Identifier:
2394       do {
2395         if (parseCnt(Waitcnt))
2396           return MatchOperand_ParseFail;
2397       } while(getLexer().isNot(AsmToken::EndOfStatement));
2398       break;
2399   }
2400   Operands.push_back(AMDGPUOperand::CreateImm(this, Waitcnt, S));
2401   return MatchOperand_Success;
2402 }
2403 
2404 bool AMDGPUAsmParser::parseHwregConstruct(OperandInfoTy &HwReg, int64_t &Offset, int64_t &Width) {
2405   using namespace llvm::AMDGPU::Hwreg;
2406 
2407   if (Parser.getTok().getString() != "hwreg")
2408     return true;
2409   Parser.Lex();
2410 
2411   if (getLexer().isNot(AsmToken::LParen))
2412     return true;
2413   Parser.Lex();
2414 
2415   if (getLexer().is(AsmToken::Identifier)) {
2416     HwReg.IsSymbolic = true;
2417     HwReg.Id = ID_UNKNOWN_;
2418     const StringRef tok = Parser.getTok().getString();
2419     for (int i = ID_SYMBOLIC_FIRST_; i < ID_SYMBOLIC_LAST_; ++i) {
2420       if (tok == IdSymbolic[i]) {
2421         HwReg.Id = i;
2422         break;
2423       }
2424     }
2425     Parser.Lex();
2426   } else {
2427     HwReg.IsSymbolic = false;
2428     if (getLexer().isNot(AsmToken::Integer))
2429       return true;
2430     if (getParser().parseAbsoluteExpression(HwReg.Id))
2431       return true;
2432   }
2433 
2434   if (getLexer().is(AsmToken::RParen)) {
2435     Parser.Lex();
2436     return false;
2437   }
2438 
2439   // optional params
2440   if (getLexer().isNot(AsmToken::Comma))
2441     return true;
2442   Parser.Lex();
2443 
2444   if (getLexer().isNot(AsmToken::Integer))
2445     return true;
2446   if (getParser().parseAbsoluteExpression(Offset))
2447     return true;
2448 
2449   if (getLexer().isNot(AsmToken::Comma))
2450     return true;
2451   Parser.Lex();
2452 
2453   if (getLexer().isNot(AsmToken::Integer))
2454     return true;
2455   if (getParser().parseAbsoluteExpression(Width))
2456     return true;
2457 
2458   if (getLexer().isNot(AsmToken::RParen))
2459     return true;
2460   Parser.Lex();
2461 
2462   return false;
2463 }
2464 
2465 OperandMatchResultTy
2466 AMDGPUAsmParser::parseHwreg(OperandVector &Operands) {
2467   using namespace llvm::AMDGPU::Hwreg;
2468 
2469   int64_t Imm16Val = 0;
2470   SMLoc S = Parser.getTok().getLoc();
2471 
2472   switch(getLexer().getKind()) {
2473     default: return MatchOperand_NoMatch;
2474     case AsmToken::Integer:
2475       // The operand can be an integer value.
2476       if (getParser().parseAbsoluteExpression(Imm16Val))
2477         return MatchOperand_NoMatch;
2478       if (Imm16Val < 0 || !isUInt<16>(Imm16Val)) {
2479         Error(S, "invalid immediate: only 16-bit values are legal");
2480         // Do not return error code, but create an imm operand anyway and proceed
2481         // to the next operand, if any. That avoids unneccessary error messages.
2482       }
2483       break;
2484 
2485     case AsmToken::Identifier: {
2486         OperandInfoTy HwReg(ID_UNKNOWN_);
2487         int64_t Offset = OFFSET_DEFAULT_;
2488         int64_t Width = WIDTH_M1_DEFAULT_ + 1;
2489         if (parseHwregConstruct(HwReg, Offset, Width))
2490           return MatchOperand_ParseFail;
2491         if (HwReg.Id < 0 || !isUInt<ID_WIDTH_>(HwReg.Id)) {
2492           if (HwReg.IsSymbolic)
2493             Error(S, "invalid symbolic name of hardware register");
2494           else
2495             Error(S, "invalid code of hardware register: only 6-bit values are legal");
2496         }
2497         if (Offset < 0 || !isUInt<OFFSET_WIDTH_>(Offset))
2498           Error(S, "invalid bit offset: only 5-bit values are legal");
2499         if ((Width-1) < 0 || !isUInt<WIDTH_M1_WIDTH_>(Width-1))
2500           Error(S, "invalid bitfield width: only values from 1 to 32 are legal");
2501         Imm16Val = (HwReg.Id << ID_SHIFT_) | (Offset << OFFSET_SHIFT_) | ((Width-1) << WIDTH_M1_SHIFT_);
2502       }
2503       break;
2504   }
2505   Operands.push_back(AMDGPUOperand::CreateImm(this, Imm16Val, S, AMDGPUOperand::ImmTyHwreg));
2506   return MatchOperand_Success;
2507 }
2508 
2509 bool AMDGPUOperand::isSWaitCnt() const {
2510   return isImm();
2511 }
2512 
2513 bool AMDGPUOperand::isHwreg() const {
2514   return isImmTy(ImmTyHwreg);
2515 }
2516 
2517 bool AMDGPUAsmParser::parseSendMsgConstruct(OperandInfoTy &Msg, OperandInfoTy &Operation, int64_t &StreamId) {
2518   using namespace llvm::AMDGPU::SendMsg;
2519 
2520   if (Parser.getTok().getString() != "sendmsg")
2521     return true;
2522   Parser.Lex();
2523 
2524   if (getLexer().isNot(AsmToken::LParen))
2525     return true;
2526   Parser.Lex();
2527 
2528   if (getLexer().is(AsmToken::Identifier)) {
2529     Msg.IsSymbolic = true;
2530     Msg.Id = ID_UNKNOWN_;
2531     const std::string tok = Parser.getTok().getString();
2532     for (int i = ID_GAPS_FIRST_; i < ID_GAPS_LAST_; ++i) {
2533       switch(i) {
2534         default: continue; // Omit gaps.
2535         case ID_INTERRUPT: case ID_GS: case ID_GS_DONE:  case ID_SYSMSG: break;
2536       }
2537       if (tok == IdSymbolic[i]) {
2538         Msg.Id = i;
2539         break;
2540       }
2541     }
2542     Parser.Lex();
2543   } else {
2544     Msg.IsSymbolic = false;
2545     if (getLexer().isNot(AsmToken::Integer))
2546       return true;
2547     if (getParser().parseAbsoluteExpression(Msg.Id))
2548       return true;
2549     if (getLexer().is(AsmToken::Integer))
2550       if (getParser().parseAbsoluteExpression(Msg.Id))
2551         Msg.Id = ID_UNKNOWN_;
2552   }
2553   if (Msg.Id == ID_UNKNOWN_) // Don't know how to parse the rest.
2554     return false;
2555 
2556   if (!(Msg.Id == ID_GS || Msg.Id == ID_GS_DONE || Msg.Id == ID_SYSMSG)) {
2557     if (getLexer().isNot(AsmToken::RParen))
2558       return true;
2559     Parser.Lex();
2560     return false;
2561   }
2562 
2563   if (getLexer().isNot(AsmToken::Comma))
2564     return true;
2565   Parser.Lex();
2566 
2567   assert(Msg.Id == ID_GS || Msg.Id == ID_GS_DONE || Msg.Id == ID_SYSMSG);
2568   Operation.Id = ID_UNKNOWN_;
2569   if (getLexer().is(AsmToken::Identifier)) {
2570     Operation.IsSymbolic = true;
2571     const char* const *S = (Msg.Id == ID_SYSMSG) ? OpSysSymbolic : OpGsSymbolic;
2572     const int F = (Msg.Id == ID_SYSMSG) ? OP_SYS_FIRST_ : OP_GS_FIRST_;
2573     const int L = (Msg.Id == ID_SYSMSG) ? OP_SYS_LAST_ : OP_GS_LAST_;
2574     const StringRef Tok = Parser.getTok().getString();
2575     for (int i = F; i < L; ++i) {
2576       if (Tok == S[i]) {
2577         Operation.Id = i;
2578         break;
2579       }
2580     }
2581     Parser.Lex();
2582   } else {
2583     Operation.IsSymbolic = false;
2584     if (getLexer().isNot(AsmToken::Integer))
2585       return true;
2586     if (getParser().parseAbsoluteExpression(Operation.Id))
2587       return true;
2588   }
2589 
2590   if ((Msg.Id == ID_GS || Msg.Id == ID_GS_DONE) && Operation.Id != OP_GS_NOP) {
2591     // Stream id is optional.
2592     if (getLexer().is(AsmToken::RParen)) {
2593       Parser.Lex();
2594       return false;
2595     }
2596 
2597     if (getLexer().isNot(AsmToken::Comma))
2598       return true;
2599     Parser.Lex();
2600 
2601     if (getLexer().isNot(AsmToken::Integer))
2602       return true;
2603     if (getParser().parseAbsoluteExpression(StreamId))
2604       return true;
2605   }
2606 
2607   if (getLexer().isNot(AsmToken::RParen))
2608     return true;
2609   Parser.Lex();
2610   return false;
2611 }
2612 
2613 OperandMatchResultTy AMDGPUAsmParser::parseInterpSlot(OperandVector &Operands) {
2614   if (getLexer().getKind() != AsmToken::Identifier)
2615     return MatchOperand_NoMatch;
2616 
2617   StringRef Str = Parser.getTok().getString();
2618   int Slot = StringSwitch<int>(Str)
2619     .Case("p10", 0)
2620     .Case("p20", 1)
2621     .Case("p0", 2)
2622     .Default(-1);
2623 
2624   SMLoc S = Parser.getTok().getLoc();
2625   if (Slot == -1)
2626     return MatchOperand_ParseFail;
2627 
2628   Parser.Lex();
2629   Operands.push_back(AMDGPUOperand::CreateImm(this, Slot, S,
2630                                               AMDGPUOperand::ImmTyInterpSlot));
2631   return MatchOperand_Success;
2632 }
2633 
2634 OperandMatchResultTy AMDGPUAsmParser::parseInterpAttr(OperandVector &Operands) {
2635   if (getLexer().getKind() != AsmToken::Identifier)
2636     return MatchOperand_NoMatch;
2637 
2638   StringRef Str = Parser.getTok().getString();
2639   if (!Str.startswith("attr"))
2640     return MatchOperand_NoMatch;
2641 
2642   StringRef Chan = Str.take_back(2);
2643   int AttrChan = StringSwitch<int>(Chan)
2644     .Case(".x", 0)
2645     .Case(".y", 1)
2646     .Case(".z", 2)
2647     .Case(".w", 3)
2648     .Default(-1);
2649   if (AttrChan == -1)
2650     return MatchOperand_ParseFail;
2651 
2652   Str = Str.drop_back(2).drop_front(4);
2653 
2654   uint8_t Attr;
2655   if (Str.getAsInteger(10, Attr))
2656     return MatchOperand_ParseFail;
2657 
2658   SMLoc S = Parser.getTok().getLoc();
2659   Parser.Lex();
2660   if (Attr > 63) {
2661     Error(S, "out of bounds attr");
2662     return MatchOperand_Success;
2663   }
2664 
2665   SMLoc SChan = SMLoc::getFromPointer(Chan.data());
2666 
2667   Operands.push_back(AMDGPUOperand::CreateImm(this, Attr, S,
2668                                               AMDGPUOperand::ImmTyInterpAttr));
2669   Operands.push_back(AMDGPUOperand::CreateImm(this, AttrChan, SChan,
2670                                               AMDGPUOperand::ImmTyAttrChan));
2671   return MatchOperand_Success;
2672 }
2673 
2674 void AMDGPUAsmParser::errorExpTgt() {
2675   Error(Parser.getTok().getLoc(), "invalid exp target");
2676 }
2677 
2678 OperandMatchResultTy AMDGPUAsmParser::parseExpTgtImpl(StringRef Str,
2679                                                       uint8_t &Val) {
2680   if (Str == "null") {
2681     Val = 9;
2682     return MatchOperand_Success;
2683   }
2684 
2685   if (Str.startswith("mrt")) {
2686     Str = Str.drop_front(3);
2687     if (Str == "z") { // == mrtz
2688       Val = 8;
2689       return MatchOperand_Success;
2690     }
2691 
2692     if (Str.getAsInteger(10, Val))
2693       return MatchOperand_ParseFail;
2694 
2695     if (Val > 7)
2696       errorExpTgt();
2697 
2698     return MatchOperand_Success;
2699   }
2700 
2701   if (Str.startswith("pos")) {
2702     Str = Str.drop_front(3);
2703     if (Str.getAsInteger(10, Val))
2704       return MatchOperand_ParseFail;
2705 
2706     if (Val > 3)
2707       errorExpTgt();
2708 
2709     Val += 12;
2710     return MatchOperand_Success;
2711   }
2712 
2713   if (Str.startswith("param")) {
2714     Str = Str.drop_front(5);
2715     if (Str.getAsInteger(10, Val))
2716       return MatchOperand_ParseFail;
2717 
2718     if (Val >= 32)
2719       errorExpTgt();
2720 
2721     Val += 32;
2722     return MatchOperand_Success;
2723   }
2724 
2725   if (Str.startswith("invalid_target_")) {
2726     Str = Str.drop_front(15);
2727     if (Str.getAsInteger(10, Val))
2728       return MatchOperand_ParseFail;
2729 
2730     errorExpTgt();
2731     return MatchOperand_Success;
2732   }
2733 
2734   return MatchOperand_NoMatch;
2735 }
2736 
2737 OperandMatchResultTy AMDGPUAsmParser::parseExpTgt(OperandVector &Operands) {
2738   uint8_t Val;
2739   StringRef Str = Parser.getTok().getString();
2740 
2741   auto Res = parseExpTgtImpl(Str, Val);
2742   if (Res != MatchOperand_Success)
2743     return Res;
2744 
2745   SMLoc S = Parser.getTok().getLoc();
2746   Parser.Lex();
2747 
2748   Operands.push_back(AMDGPUOperand::CreateImm(this, Val, S,
2749                                               AMDGPUOperand::ImmTyExpTgt));
2750   return MatchOperand_Success;
2751 }
2752 
2753 OperandMatchResultTy
2754 AMDGPUAsmParser::parseSendMsgOp(OperandVector &Operands) {
2755   using namespace llvm::AMDGPU::SendMsg;
2756 
2757   int64_t Imm16Val = 0;
2758   SMLoc S = Parser.getTok().getLoc();
2759 
2760   switch(getLexer().getKind()) {
2761   default:
2762     return MatchOperand_NoMatch;
2763   case AsmToken::Integer:
2764     // The operand can be an integer value.
2765     if (getParser().parseAbsoluteExpression(Imm16Val))
2766       return MatchOperand_NoMatch;
2767     if (Imm16Val < 0 || !isUInt<16>(Imm16Val)) {
2768       Error(S, "invalid immediate: only 16-bit values are legal");
2769       // Do not return error code, but create an imm operand anyway and proceed
2770       // to the next operand, if any. That avoids unneccessary error messages.
2771     }
2772     break;
2773   case AsmToken::Identifier: {
2774       OperandInfoTy Msg(ID_UNKNOWN_);
2775       OperandInfoTy Operation(OP_UNKNOWN_);
2776       int64_t StreamId = STREAM_ID_DEFAULT_;
2777       if (parseSendMsgConstruct(Msg, Operation, StreamId))
2778         return MatchOperand_ParseFail;
2779       do {
2780         // Validate and encode message ID.
2781         if (! ((ID_INTERRUPT <= Msg.Id && Msg.Id <= ID_GS_DONE)
2782                 || Msg.Id == ID_SYSMSG)) {
2783           if (Msg.IsSymbolic)
2784             Error(S, "invalid/unsupported symbolic name of message");
2785           else
2786             Error(S, "invalid/unsupported code of message");
2787           break;
2788         }
2789         Imm16Val = (Msg.Id << ID_SHIFT_);
2790         // Validate and encode operation ID.
2791         if (Msg.Id == ID_GS || Msg.Id == ID_GS_DONE) {
2792           if (! (OP_GS_FIRST_ <= Operation.Id && Operation.Id < OP_GS_LAST_)) {
2793             if (Operation.IsSymbolic)
2794               Error(S, "invalid symbolic name of GS_OP");
2795             else
2796               Error(S, "invalid code of GS_OP: only 2-bit values are legal");
2797             break;
2798           }
2799           if (Operation.Id == OP_GS_NOP
2800               && Msg.Id != ID_GS_DONE) {
2801             Error(S, "invalid GS_OP: NOP is for GS_DONE only");
2802             break;
2803           }
2804           Imm16Val |= (Operation.Id << OP_SHIFT_);
2805         }
2806         if (Msg.Id == ID_SYSMSG) {
2807           if (! (OP_SYS_FIRST_ <= Operation.Id && Operation.Id < OP_SYS_LAST_)) {
2808             if (Operation.IsSymbolic)
2809               Error(S, "invalid/unsupported symbolic name of SYSMSG_OP");
2810             else
2811               Error(S, "invalid/unsupported code of SYSMSG_OP");
2812             break;
2813           }
2814           Imm16Val |= (Operation.Id << OP_SHIFT_);
2815         }
2816         // Validate and encode stream ID.
2817         if ((Msg.Id == ID_GS || Msg.Id == ID_GS_DONE) && Operation.Id != OP_GS_NOP) {
2818           if (! (STREAM_ID_FIRST_ <= StreamId && StreamId < STREAM_ID_LAST_)) {
2819             Error(S, "invalid stream id: only 2-bit values are legal");
2820             break;
2821           }
2822           Imm16Val |= (StreamId << STREAM_ID_SHIFT_);
2823         }
2824       } while (false);
2825     }
2826     break;
2827   }
2828   Operands.push_back(AMDGPUOperand::CreateImm(this, Imm16Val, S, AMDGPUOperand::ImmTySendMsg));
2829   return MatchOperand_Success;
2830 }
2831 
2832 bool AMDGPUOperand::isSendMsg() const {
2833   return isImmTy(ImmTySendMsg);
2834 }
2835 
2836 //===----------------------------------------------------------------------===//
2837 // sopp branch targets
2838 //===----------------------------------------------------------------------===//
2839 
2840 OperandMatchResultTy
2841 AMDGPUAsmParser::parseSOppBrTarget(OperandVector &Operands) {
2842   SMLoc S = Parser.getTok().getLoc();
2843 
2844   switch (getLexer().getKind()) {
2845     default: return MatchOperand_ParseFail;
2846     case AsmToken::Integer: {
2847       int64_t Imm;
2848       if (getParser().parseAbsoluteExpression(Imm))
2849         return MatchOperand_ParseFail;
2850       Operands.push_back(AMDGPUOperand::CreateImm(this, Imm, S));
2851       return MatchOperand_Success;
2852     }
2853 
2854     case AsmToken::Identifier:
2855       Operands.push_back(AMDGPUOperand::CreateExpr(this,
2856           MCSymbolRefExpr::create(getContext().getOrCreateSymbol(
2857                                   Parser.getTok().getString()), getContext()), S));
2858       Parser.Lex();
2859       return MatchOperand_Success;
2860   }
2861 }
2862 
2863 //===----------------------------------------------------------------------===//
2864 // mubuf
2865 //===----------------------------------------------------------------------===//
2866 
2867 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultGLC() const {
2868   return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTyGLC);
2869 }
2870 
2871 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultSLC() const {
2872   return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTySLC);
2873 }
2874 
2875 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultTFE() const {
2876   return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTyTFE);
2877 }
2878 
2879 void AMDGPUAsmParser::cvtMubufImpl(MCInst &Inst,
2880                                const OperandVector &Operands,
2881                                bool IsAtomic, bool IsAtomicReturn) {
2882   OptionalImmIndexMap OptionalIdx;
2883   assert(IsAtomicReturn ? IsAtomic : true);
2884 
2885   for (unsigned i = 1, e = Operands.size(); i != e; ++i) {
2886     AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[i]);
2887 
2888     // Add the register arguments
2889     if (Op.isReg()) {
2890       Op.addRegOperands(Inst, 1);
2891       continue;
2892     }
2893 
2894     // Handle the case where soffset is an immediate
2895     if (Op.isImm() && Op.getImmTy() == AMDGPUOperand::ImmTyNone) {
2896       Op.addImmOperands(Inst, 1);
2897       continue;
2898     }
2899 
2900     // Handle tokens like 'offen' which are sometimes hard-coded into the
2901     // asm string.  There are no MCInst operands for these.
2902     if (Op.isToken()) {
2903       continue;
2904     }
2905     assert(Op.isImm());
2906 
2907     // Handle optional arguments
2908     OptionalIdx[Op.getImmTy()] = i;
2909   }
2910 
2911   // Copy $vdata_in operand and insert as $vdata for MUBUF_Atomic RTN insns.
2912   if (IsAtomicReturn) {
2913     MCInst::iterator I = Inst.begin(); // $vdata_in is always at the beginning.
2914     Inst.insert(I, *I);
2915   }
2916 
2917   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyOffset);
2918   if (!IsAtomic) { // glc is hard-coded.
2919     addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyGLC);
2920   }
2921   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySLC);
2922   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyTFE);
2923 }
2924 
2925 //===----------------------------------------------------------------------===//
2926 // mimg
2927 //===----------------------------------------------------------------------===//
2928 
2929 void AMDGPUAsmParser::cvtMIMG(MCInst &Inst, const OperandVector &Operands) {
2930   unsigned I = 1;
2931   const MCInstrDesc &Desc = MII.get(Inst.getOpcode());
2932   for (unsigned J = 0; J < Desc.getNumDefs(); ++J) {
2933     ((AMDGPUOperand &)*Operands[I++]).addRegOperands(Inst, 1);
2934   }
2935 
2936   OptionalImmIndexMap OptionalIdx;
2937 
2938   for (unsigned E = Operands.size(); I != E; ++I) {
2939     AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[I]);
2940 
2941     // Add the register arguments
2942     if (Op.isRegOrImm()) {
2943       Op.addRegOrImmOperands(Inst, 1);
2944       continue;
2945     } else if (Op.isImmModifier()) {
2946       OptionalIdx[Op.getImmTy()] = I;
2947     } else {
2948       llvm_unreachable("unexpected operand type");
2949     }
2950   }
2951 
2952   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyDMask);
2953   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyUNorm);
2954   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyGLC);
2955   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyDA);
2956   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyR128);
2957   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyTFE);
2958   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyLWE);
2959   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySLC);
2960 }
2961 
2962 void AMDGPUAsmParser::cvtMIMGAtomic(MCInst &Inst, const OperandVector &Operands) {
2963   unsigned I = 1;
2964   const MCInstrDesc &Desc = MII.get(Inst.getOpcode());
2965   for (unsigned J = 0; J < Desc.getNumDefs(); ++J) {
2966     ((AMDGPUOperand &)*Operands[I++]).addRegOperands(Inst, 1);
2967   }
2968 
2969   // Add src, same as dst
2970   ((AMDGPUOperand &)*Operands[I]).addRegOperands(Inst, 1);
2971 
2972   OptionalImmIndexMap OptionalIdx;
2973 
2974   for (unsigned E = Operands.size(); I != E; ++I) {
2975     AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[I]);
2976 
2977     // Add the register arguments
2978     if (Op.isRegOrImm()) {
2979       Op.addRegOrImmOperands(Inst, 1);
2980       continue;
2981     } else if (Op.isImmModifier()) {
2982       OptionalIdx[Op.getImmTy()] = I;
2983     } else {
2984       llvm_unreachable("unexpected operand type");
2985     }
2986   }
2987 
2988   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyDMask);
2989   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyUNorm);
2990   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyGLC);
2991   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyDA);
2992   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyR128);
2993   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyTFE);
2994   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyLWE);
2995   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySLC);
2996 }
2997 
2998 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultDMask() const {
2999   return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTyDMask);
3000 }
3001 
3002 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultUNorm() const {
3003   return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTyUNorm);
3004 }
3005 
3006 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultDA() const {
3007   return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTyDA);
3008 }
3009 
3010 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultR128() const {
3011   return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTyR128);
3012 }
3013 
3014 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultLWE() const {
3015   return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTyLWE);
3016 }
3017 
3018 //===----------------------------------------------------------------------===//
3019 // smrd
3020 //===----------------------------------------------------------------------===//
3021 
3022 bool AMDGPUOperand::isSMRDOffset8() const {
3023   return isImm() && isUInt<8>(getImm());
3024 }
3025 
3026 bool AMDGPUOperand::isSMRDOffset20() const {
3027   return isImm() && isUInt<20>(getImm());
3028 }
3029 
3030 bool AMDGPUOperand::isSMRDLiteralOffset() const {
3031   // 32-bit literals are only supported on CI and we only want to use them
3032   // when the offset is > 8-bits.
3033   return isImm() && !isUInt<8>(getImm()) && isUInt<32>(getImm());
3034 }
3035 
3036 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultSMRDOffset8() const {
3037   return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTyOffset);
3038 }
3039 
3040 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultSMRDOffset20() const {
3041   return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTyOffset);
3042 }
3043 
3044 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultSMRDLiteralOffset() const {
3045   return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTyOffset);
3046 }
3047 
3048 //===----------------------------------------------------------------------===//
3049 // vop3
3050 //===----------------------------------------------------------------------===//
3051 
3052 static bool ConvertOmodMul(int64_t &Mul) {
3053   if (Mul != 1 && Mul != 2 && Mul != 4)
3054     return false;
3055 
3056   Mul >>= 1;
3057   return true;
3058 }
3059 
3060 static bool ConvertOmodDiv(int64_t &Div) {
3061   if (Div == 1) {
3062     Div = 0;
3063     return true;
3064   }
3065 
3066   if (Div == 2) {
3067     Div = 3;
3068     return true;
3069   }
3070 
3071   return false;
3072 }
3073 
3074 static bool ConvertBoundCtrl(int64_t &BoundCtrl) {
3075   if (BoundCtrl == 0) {
3076     BoundCtrl = 1;
3077     return true;
3078   }
3079 
3080   if (BoundCtrl == -1) {
3081     BoundCtrl = 0;
3082     return true;
3083   }
3084 
3085   return false;
3086 }
3087 
3088 // Note: the order in this table matches the order of operands in AsmString.
3089 static const OptionalOperand AMDGPUOptionalOperandTable[] = {
3090   {"offen",   AMDGPUOperand::ImmTyOffen, true, nullptr},
3091   {"idxen",   AMDGPUOperand::ImmTyIdxen, true, nullptr},
3092   {"addr64",  AMDGPUOperand::ImmTyAddr64, true, nullptr},
3093   {"offset0", AMDGPUOperand::ImmTyOffset0, false, nullptr},
3094   {"offset1", AMDGPUOperand::ImmTyOffset1, false, nullptr},
3095   {"gds",     AMDGPUOperand::ImmTyGDS, true, nullptr},
3096   {"offset",  AMDGPUOperand::ImmTyOffset, false, nullptr},
3097   {"glc",     AMDGPUOperand::ImmTyGLC, true, nullptr},
3098   {"slc",     AMDGPUOperand::ImmTySLC, true, nullptr},
3099   {"tfe",     AMDGPUOperand::ImmTyTFE, true, nullptr},
3100   {"clamp",   AMDGPUOperand::ImmTyClampSI, true, nullptr},
3101   {"omod",    AMDGPUOperand::ImmTyOModSI, false, ConvertOmodMul},
3102   {"unorm",   AMDGPUOperand::ImmTyUNorm, true, nullptr},
3103   {"da",      AMDGPUOperand::ImmTyDA,    true, nullptr},
3104   {"r128",    AMDGPUOperand::ImmTyR128,  true, nullptr},
3105   {"lwe",     AMDGPUOperand::ImmTyLWE,   true, nullptr},
3106   {"dmask",   AMDGPUOperand::ImmTyDMask, false, nullptr},
3107   {"row_mask",   AMDGPUOperand::ImmTyDppRowMask, false, nullptr},
3108   {"bank_mask",  AMDGPUOperand::ImmTyDppBankMask, false, nullptr},
3109   {"bound_ctrl", AMDGPUOperand::ImmTyDppBoundCtrl, false, ConvertBoundCtrl},
3110   {"dst_sel",    AMDGPUOperand::ImmTySdwaDstSel, false, nullptr},
3111   {"src0_sel",   AMDGPUOperand::ImmTySdwaSrc0Sel, false, nullptr},
3112   {"src1_sel",   AMDGPUOperand::ImmTySdwaSrc1Sel, false, nullptr},
3113   {"dst_unused", AMDGPUOperand::ImmTySdwaDstUnused, false, nullptr},
3114   {"vm", AMDGPUOperand::ImmTyExpVM, true, nullptr},
3115 };
3116 
3117 OperandMatchResultTy AMDGPUAsmParser::parseOptionalOperand(OperandVector &Operands) {
3118   OperandMatchResultTy res;
3119   for (const OptionalOperand &Op : AMDGPUOptionalOperandTable) {
3120     // try to parse any optional operand here
3121     if (Op.IsBit) {
3122       res = parseNamedBit(Op.Name, Operands, Op.Type);
3123     } else if (Op.Type == AMDGPUOperand::ImmTyOModSI) {
3124       res = parseOModOperand(Operands);
3125     } else if (Op.Type == AMDGPUOperand::ImmTySdwaDstSel ||
3126                Op.Type == AMDGPUOperand::ImmTySdwaSrc0Sel ||
3127                Op.Type == AMDGPUOperand::ImmTySdwaSrc1Sel) {
3128       res = parseSDWASel(Operands, Op.Name, Op.Type);
3129     } else if (Op.Type == AMDGPUOperand::ImmTySdwaDstUnused) {
3130       res = parseSDWADstUnused(Operands);
3131     } else {
3132       res = parseIntWithPrefix(Op.Name, Operands, Op.Type, Op.ConvertResult);
3133     }
3134     if (res != MatchOperand_NoMatch) {
3135       return res;
3136     }
3137   }
3138   return MatchOperand_NoMatch;
3139 }
3140 
3141 OperandMatchResultTy AMDGPUAsmParser::parseOModOperand(OperandVector &Operands) {
3142   StringRef Name = Parser.getTok().getString();
3143   if (Name == "mul") {
3144     return parseIntWithPrefix("mul", Operands,
3145                               AMDGPUOperand::ImmTyOModSI, ConvertOmodMul);
3146   }
3147 
3148   if (Name == "div") {
3149     return parseIntWithPrefix("div", Operands,
3150                               AMDGPUOperand::ImmTyOModSI, ConvertOmodDiv);
3151   }
3152 
3153   return MatchOperand_NoMatch;
3154 }
3155 
3156 void AMDGPUAsmParser::cvtId(MCInst &Inst, const OperandVector &Operands) {
3157   unsigned I = 1;
3158   const MCInstrDesc &Desc = MII.get(Inst.getOpcode());
3159   for (unsigned J = 0; J < Desc.getNumDefs(); ++J) {
3160     ((AMDGPUOperand &)*Operands[I++]).addRegOperands(Inst, 1);
3161   }
3162   for (unsigned E = Operands.size(); I != E; ++I)
3163     ((AMDGPUOperand &)*Operands[I]).addRegOrImmOperands(Inst, 1);
3164 }
3165 
3166 void AMDGPUAsmParser::cvtVOP3_2_mod(MCInst &Inst, const OperandVector &Operands) {
3167   uint64_t TSFlags = MII.get(Inst.getOpcode()).TSFlags;
3168   if (TSFlags & SIInstrFlags::VOP3) {
3169     cvtVOP3(Inst, Operands);
3170   } else {
3171     cvtId(Inst, Operands);
3172   }
3173 }
3174 
3175 static bool isRegOrImmWithInputMods(const MCInstrDesc &Desc, unsigned OpNum) {
3176       // 1. This operand is input modifiers
3177   return Desc.OpInfo[OpNum].OperandType == AMDGPU::OPERAND_INPUT_MODS
3178       // 2. This is not last operand
3179       && Desc.NumOperands > (OpNum + 1)
3180       // 3. Next operand is register class
3181       && Desc.OpInfo[OpNum + 1].RegClass != -1
3182       // 4. Next register is not tied to any other operand
3183       && Desc.getOperandConstraint(OpNum + 1, MCOI::OperandConstraint::TIED_TO) == -1;
3184 }
3185 
3186 void AMDGPUAsmParser::cvtVOP3(MCInst &Inst, const OperandVector &Operands) {
3187   OptionalImmIndexMap OptionalIdx;
3188   unsigned I = 1;
3189   const MCInstrDesc &Desc = MII.get(Inst.getOpcode());
3190   for (unsigned J = 0; J < Desc.getNumDefs(); ++J) {
3191     ((AMDGPUOperand &)*Operands[I++]).addRegOperands(Inst, 1);
3192   }
3193 
3194   for (unsigned E = Operands.size(); I != E; ++I) {
3195     AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[I]);
3196     if (isRegOrImmWithInputMods(Desc, Inst.getNumOperands())) {
3197       Op.addRegOrImmWithFPInputModsOperands(Inst, 2);
3198     } else if (Op.isImm()) {
3199       OptionalIdx[Op.getImmTy()] = I;
3200     } else {
3201       llvm_unreachable("unhandled operand type");
3202     }
3203   }
3204 
3205   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyClampSI);
3206   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyOModSI);
3207 
3208   // special case v_mac_{f16, f32}:
3209   // it has src2 register operand that is tied to dst operand
3210   // we don't allow modifiers for this operand in assembler so src2_modifiers
3211   // should be 0
3212   if (Inst.getOpcode() == AMDGPU::V_MAC_F32_e64_si ||
3213       Inst.getOpcode() == AMDGPU::V_MAC_F32_e64_vi ||
3214       Inst.getOpcode() == AMDGPU::V_MAC_F16_e64_vi) {
3215     auto it = Inst.begin();
3216     std::advance(
3217       it,
3218       AMDGPU::getNamedOperandIdx(Inst.getOpcode() == AMDGPU::V_MAC_F16_e64_vi ?
3219                                      AMDGPU::V_MAC_F16_e64 :
3220                                      AMDGPU::V_MAC_F32_e64,
3221                                  AMDGPU::OpName::src2_modifiers));
3222     it = Inst.insert(it, MCOperand::createImm(0)); // no modifiers for src2
3223     ++it;
3224     Inst.insert(it, Inst.getOperand(0)); // src2 = dst
3225   }
3226 }
3227 
3228 //===----------------------------------------------------------------------===//
3229 // dpp
3230 //===----------------------------------------------------------------------===//
3231 
3232 bool AMDGPUOperand::isDPPCtrl() const {
3233   bool result = isImm() && getImmTy() == ImmTyDppCtrl && isUInt<9>(getImm());
3234   if (result) {
3235     int64_t Imm = getImm();
3236     return ((Imm >= 0x000) && (Imm <= 0x0ff)) ||
3237            ((Imm >= 0x101) && (Imm <= 0x10f)) ||
3238            ((Imm >= 0x111) && (Imm <= 0x11f)) ||
3239            ((Imm >= 0x121) && (Imm <= 0x12f)) ||
3240            (Imm == 0x130) ||
3241            (Imm == 0x134) ||
3242            (Imm == 0x138) ||
3243            (Imm == 0x13c) ||
3244            (Imm == 0x140) ||
3245            (Imm == 0x141) ||
3246            (Imm == 0x142) ||
3247            (Imm == 0x143);
3248   }
3249   return false;
3250 }
3251 
3252 bool AMDGPUOperand::isGPRIdxMode() const {
3253   return isImm() && isUInt<4>(getImm());
3254 }
3255 
3256 OperandMatchResultTy
3257 AMDGPUAsmParser::parseDPPCtrl(OperandVector &Operands) {
3258   SMLoc S = Parser.getTok().getLoc();
3259   StringRef Prefix;
3260   int64_t Int;
3261 
3262   if (getLexer().getKind() == AsmToken::Identifier) {
3263     Prefix = Parser.getTok().getString();
3264   } else {
3265     return MatchOperand_NoMatch;
3266   }
3267 
3268   if (Prefix == "row_mirror") {
3269     Int = 0x140;
3270     Parser.Lex();
3271   } else if (Prefix == "row_half_mirror") {
3272     Int = 0x141;
3273     Parser.Lex();
3274   } else {
3275     // Check to prevent parseDPPCtrlOps from eating invalid tokens
3276     if (Prefix != "quad_perm"
3277         && Prefix != "row_shl"
3278         && Prefix != "row_shr"
3279         && Prefix != "row_ror"
3280         && Prefix != "wave_shl"
3281         && Prefix != "wave_rol"
3282         && Prefix != "wave_shr"
3283         && Prefix != "wave_ror"
3284         && Prefix != "row_bcast") {
3285       return MatchOperand_NoMatch;
3286     }
3287 
3288     Parser.Lex();
3289     if (getLexer().isNot(AsmToken::Colon))
3290       return MatchOperand_ParseFail;
3291 
3292     if (Prefix == "quad_perm") {
3293       // quad_perm:[%d,%d,%d,%d]
3294       Parser.Lex();
3295       if (getLexer().isNot(AsmToken::LBrac))
3296         return MatchOperand_ParseFail;
3297       Parser.Lex();
3298 
3299       if (getParser().parseAbsoluteExpression(Int) || !(0 <= Int && Int <=3))
3300         return MatchOperand_ParseFail;
3301 
3302       for (int i = 0; i < 3; ++i) {
3303         if (getLexer().isNot(AsmToken::Comma))
3304           return MatchOperand_ParseFail;
3305         Parser.Lex();
3306 
3307         int64_t Temp;
3308         if (getParser().parseAbsoluteExpression(Temp) || !(0 <= Temp && Temp <=3))
3309           return MatchOperand_ParseFail;
3310         const int shift = i*2 + 2;
3311         Int += (Temp << shift);
3312       }
3313 
3314       if (getLexer().isNot(AsmToken::RBrac))
3315         return MatchOperand_ParseFail;
3316       Parser.Lex();
3317 
3318     } else {
3319       // sel:%d
3320       Parser.Lex();
3321       if (getParser().parseAbsoluteExpression(Int))
3322         return MatchOperand_ParseFail;
3323 
3324       if (Prefix == "row_shl" && 1 <= Int && Int <= 15) {
3325         Int |= 0x100;
3326       } else if (Prefix == "row_shr" && 1 <= Int && Int <= 15) {
3327         Int |= 0x110;
3328       } else if (Prefix == "row_ror" && 1 <= Int && Int <= 15) {
3329         Int |= 0x120;
3330       } else if (Prefix == "wave_shl" && 1 == Int) {
3331         Int = 0x130;
3332       } else if (Prefix == "wave_rol" && 1 == Int) {
3333         Int = 0x134;
3334       } else if (Prefix == "wave_shr" && 1 == Int) {
3335         Int = 0x138;
3336       } else if (Prefix == "wave_ror" && 1 == Int) {
3337         Int = 0x13C;
3338       } else if (Prefix == "row_bcast") {
3339         if (Int == 15) {
3340           Int = 0x142;
3341         } else if (Int == 31) {
3342           Int = 0x143;
3343         } else {
3344           return MatchOperand_ParseFail;
3345         }
3346       } else {
3347         return MatchOperand_ParseFail;
3348       }
3349     }
3350   }
3351 
3352   Operands.push_back(AMDGPUOperand::CreateImm(this, Int, S, AMDGPUOperand::ImmTyDppCtrl));
3353   return MatchOperand_Success;
3354 }
3355 
3356 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultRowMask() const {
3357   return AMDGPUOperand::CreateImm(this, 0xf, SMLoc(), AMDGPUOperand::ImmTyDppRowMask);
3358 }
3359 
3360 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultBankMask() const {
3361   return AMDGPUOperand::CreateImm(this, 0xf, SMLoc(), AMDGPUOperand::ImmTyDppBankMask);
3362 }
3363 
3364 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultBoundCtrl() const {
3365   return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTyDppBoundCtrl);
3366 }
3367 
3368 void AMDGPUAsmParser::cvtDPP(MCInst &Inst, const OperandVector &Operands) {
3369   OptionalImmIndexMap OptionalIdx;
3370 
3371   unsigned I = 1;
3372   const MCInstrDesc &Desc = MII.get(Inst.getOpcode());
3373   for (unsigned J = 0; J < Desc.getNumDefs(); ++J) {
3374     ((AMDGPUOperand &)*Operands[I++]).addRegOperands(Inst, 1);
3375   }
3376 
3377   for (unsigned E = Operands.size(); I != E; ++I) {
3378     AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[I]);
3379     // Add the register arguments
3380     if (Op.isReg() && Op.Reg.RegNo == AMDGPU::VCC) {
3381       // VOP2b (v_add_u32, v_sub_u32 ...) sdwa use "vcc" token.
3382       // Skip it.
3383       continue;
3384     } if (isRegOrImmWithInputMods(Desc, Inst.getNumOperands())) {
3385       Op.addRegOrImmWithFPInputModsOperands(Inst, 2);
3386     } else if (Op.isDPPCtrl()) {
3387       Op.addImmOperands(Inst, 1);
3388     } else if (Op.isImm()) {
3389       // Handle optional arguments
3390       OptionalIdx[Op.getImmTy()] = I;
3391     } else {
3392       llvm_unreachable("Invalid operand type");
3393     }
3394   }
3395 
3396   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyDppRowMask, 0xf);
3397   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyDppBankMask, 0xf);
3398   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyDppBoundCtrl);
3399 
3400   // special case v_mac_{f16, f32}:
3401   // it has src2 register operand that is tied to dst operand
3402   if (Inst.getOpcode() == AMDGPU::V_MAC_F32_dpp ||
3403       Inst.getOpcode() == AMDGPU::V_MAC_F16_dpp) {
3404     auto it = Inst.begin();
3405     std::advance(
3406         it, AMDGPU::getNamedOperandIdx(Inst.getOpcode(), AMDGPU::OpName::src2));
3407     Inst.insert(it, Inst.getOperand(0)); // src2 = dst
3408   }
3409 }
3410 
3411 //===----------------------------------------------------------------------===//
3412 // sdwa
3413 //===----------------------------------------------------------------------===//
3414 
3415 OperandMatchResultTy
3416 AMDGPUAsmParser::parseSDWASel(OperandVector &Operands, StringRef Prefix,
3417                               AMDGPUOperand::ImmTy Type) {
3418   using namespace llvm::AMDGPU::SDWA;
3419 
3420   SMLoc S = Parser.getTok().getLoc();
3421   StringRef Value;
3422   OperandMatchResultTy res;
3423 
3424   res = parseStringWithPrefix(Prefix, Value);
3425   if (res != MatchOperand_Success) {
3426     return res;
3427   }
3428 
3429   int64_t Int;
3430   Int = StringSwitch<int64_t>(Value)
3431         .Case("BYTE_0", SdwaSel::BYTE_0)
3432         .Case("BYTE_1", SdwaSel::BYTE_1)
3433         .Case("BYTE_2", SdwaSel::BYTE_2)
3434         .Case("BYTE_3", SdwaSel::BYTE_3)
3435         .Case("WORD_0", SdwaSel::WORD_0)
3436         .Case("WORD_1", SdwaSel::WORD_1)
3437         .Case("DWORD", SdwaSel::DWORD)
3438         .Default(0xffffffff);
3439   Parser.Lex(); // eat last token
3440 
3441   if (Int == 0xffffffff) {
3442     return MatchOperand_ParseFail;
3443   }
3444 
3445   Operands.push_back(AMDGPUOperand::CreateImm(this, Int, S, Type));
3446   return MatchOperand_Success;
3447 }
3448 
3449 OperandMatchResultTy
3450 AMDGPUAsmParser::parseSDWADstUnused(OperandVector &Operands) {
3451   using namespace llvm::AMDGPU::SDWA;
3452 
3453   SMLoc S = Parser.getTok().getLoc();
3454   StringRef Value;
3455   OperandMatchResultTy res;
3456 
3457   res = parseStringWithPrefix("dst_unused", Value);
3458   if (res != MatchOperand_Success) {
3459     return res;
3460   }
3461 
3462   int64_t Int;
3463   Int = StringSwitch<int64_t>(Value)
3464         .Case("UNUSED_PAD", DstUnused::UNUSED_PAD)
3465         .Case("UNUSED_SEXT", DstUnused::UNUSED_SEXT)
3466         .Case("UNUSED_PRESERVE", DstUnused::UNUSED_PRESERVE)
3467         .Default(0xffffffff);
3468   Parser.Lex(); // eat last token
3469 
3470   if (Int == 0xffffffff) {
3471     return MatchOperand_ParseFail;
3472   }
3473 
3474   Operands.push_back(AMDGPUOperand::CreateImm(this, Int, S, AMDGPUOperand::ImmTySdwaDstUnused));
3475   return MatchOperand_Success;
3476 }
3477 
3478 void AMDGPUAsmParser::cvtSdwaVOP1(MCInst &Inst, const OperandVector &Operands) {
3479   cvtSDWA(Inst, Operands, SIInstrFlags::VOP1);
3480 }
3481 
3482 void AMDGPUAsmParser::cvtSdwaVOP2(MCInst &Inst, const OperandVector &Operands) {
3483   cvtSDWA(Inst, Operands, SIInstrFlags::VOP2);
3484 }
3485 
3486 void AMDGPUAsmParser::cvtSdwaVOPC(MCInst &Inst, const OperandVector &Operands) {
3487   cvtSDWA(Inst, Operands, SIInstrFlags::VOPC);
3488 }
3489 
3490 void AMDGPUAsmParser::cvtSDWA(MCInst &Inst, const OperandVector &Operands,
3491                               uint64_t BasicInstType) {
3492   OptionalImmIndexMap OptionalIdx;
3493 
3494   unsigned I = 1;
3495   const MCInstrDesc &Desc = MII.get(Inst.getOpcode());
3496   for (unsigned J = 0; J < Desc.getNumDefs(); ++J) {
3497     ((AMDGPUOperand &)*Operands[I++]).addRegOperands(Inst, 1);
3498   }
3499 
3500   for (unsigned E = Operands.size(); I != E; ++I) {
3501     AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[I]);
3502     // Add the register arguments
3503     if ((BasicInstType == SIInstrFlags::VOPC ||
3504          BasicInstType == SIInstrFlags::VOP2)&&
3505         Op.isReg() &&
3506         Op.Reg.RegNo == AMDGPU::VCC) {
3507       // VOPC and VOP2b (v_add_u32, v_sub_u32 ...) sdwa use "vcc" token as dst.
3508       // Skip it.
3509       continue;
3510     } else if (isRegOrImmWithInputMods(Desc, Inst.getNumOperands())) {
3511       Op.addRegOrImmWithInputModsOperands(Inst, 2);
3512     } else if (Op.isImm()) {
3513       // Handle optional arguments
3514       OptionalIdx[Op.getImmTy()] = I;
3515     } else {
3516       llvm_unreachable("Invalid operand type");
3517     }
3518   }
3519 
3520   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyClampSI, 0);
3521 
3522   if (Inst.getOpcode() != AMDGPU::V_NOP_sdwa_vi) {
3523     // V_NOP_sdwa_vi has no optional sdwa arguments
3524     switch (BasicInstType) {
3525     case SIInstrFlags::VOP1:
3526       addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySdwaDstSel, 6);
3527       addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySdwaDstUnused, 2);
3528       addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySdwaSrc0Sel, 6);
3529       break;
3530 
3531     case SIInstrFlags::VOP2:
3532       addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySdwaDstSel, 6);
3533       addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySdwaDstUnused, 2);
3534       addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySdwaSrc0Sel, 6);
3535       addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySdwaSrc1Sel, 6);
3536       break;
3537 
3538     case SIInstrFlags::VOPC:
3539       addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySdwaSrc0Sel, 6);
3540       addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySdwaSrc1Sel, 6);
3541       break;
3542 
3543     default:
3544       llvm_unreachable("Invalid instruction type. Only VOP1, VOP2 and VOPC allowed");
3545     }
3546   }
3547 
3548   // special case v_mac_{f16, f32}:
3549   // it has src2 register operand that is tied to dst operand
3550   if (Inst.getOpcode() == AMDGPU::V_MAC_F32_sdwa_vi ||
3551       Inst.getOpcode() == AMDGPU::V_MAC_F16_sdwa_vi)  {
3552     auto it = Inst.begin();
3553     std::advance(
3554         it, AMDGPU::getNamedOperandIdx(Inst.getOpcode(), AMDGPU::OpName::src2));
3555     Inst.insert(it, Inst.getOperand(0)); // src2 = dst
3556   }
3557 
3558 }
3559 
3560 /// Force static initialization.
3561 extern "C" void LLVMInitializeAMDGPUAsmParser() {
3562   RegisterMCAsmParser<AMDGPUAsmParser> A(getTheAMDGPUTarget());
3563   RegisterMCAsmParser<AMDGPUAsmParser> B(getTheGCNTarget());
3564 }
3565 
3566 #define GET_REGISTER_MATCHER
3567 #define GET_MATCHER_IMPLEMENTATION
3568 #include "AMDGPUGenAsmMatcher.inc"
3569 
3570 // This fuction should be defined after auto-generated include so that we have
3571 // MatchClassKind enum defined
3572 unsigned AMDGPUAsmParser::validateTargetOperandClass(MCParsedAsmOperand &Op,
3573                                                      unsigned Kind) {
3574   // Tokens like "glc" would be parsed as immediate operands in ParseOperand().
3575   // But MatchInstructionImpl() expects to meet token and fails to validate
3576   // operand. This method checks if we are given immediate operand but expect to
3577   // get corresponding token.
3578   AMDGPUOperand &Operand = (AMDGPUOperand&)Op;
3579   switch (Kind) {
3580   case MCK_addr64:
3581     return Operand.isAddr64() ? Match_Success : Match_InvalidOperand;
3582   case MCK_gds:
3583     return Operand.isGDS() ? Match_Success : Match_InvalidOperand;
3584   case MCK_glc:
3585     return Operand.isGLC() ? Match_Success : Match_InvalidOperand;
3586   case MCK_idxen:
3587     return Operand.isIdxen() ? Match_Success : Match_InvalidOperand;
3588   case MCK_offen:
3589     return Operand.isOffen() ? Match_Success : Match_InvalidOperand;
3590   case MCK_SSrcB32:
3591     // When operands have expression values, they will return true for isToken,
3592     // because it is not possible to distinguish between a token and an
3593     // expression at parse time. MatchInstructionImpl() will always try to
3594     // match an operand as a token, when isToken returns true, and when the
3595     // name of the expression is not a valid token, the match will fail,
3596     // so we need to handle it here.
3597     return Operand.isSSrcB32() ? Match_Success : Match_InvalidOperand;
3598   case MCK_SSrcF32:
3599     return Operand.isSSrcF32() ? Match_Success : Match_InvalidOperand;
3600   case MCK_SoppBrTarget:
3601     return Operand.isSoppBrTarget() ? Match_Success : Match_InvalidOperand;
3602   case MCK_VReg32OrOff:
3603     return Operand.isVReg32OrOff() ? Match_Success : Match_InvalidOperand;
3604   case MCK_InterpSlot:
3605     return Operand.isInterpSlot() ? Match_Success : Match_InvalidOperand;
3606   case MCK_Attr:
3607     return Operand.isInterpAttr() ? Match_Success : Match_InvalidOperand;
3608   case MCK_AttrChan:
3609     return Operand.isAttrChan() ? Match_Success : Match_InvalidOperand;
3610   default:
3611     return Match_InvalidOperand;
3612   }
3613 }
3614