1 //===-- AMDGPUAsmParser.cpp - Parse SI asm to MCInst instructions ---------===//
2 //
3 //                     The LLVM Compiler Infrastructure
4 //
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
7 //
8 //===----------------------------------------------------------------------===//
9 
10 #include "AMDKernelCodeT.h"
11 #include "MCTargetDesc/AMDGPUMCTargetDesc.h"
12 #include "MCTargetDesc/AMDGPUTargetStreamer.h"
13 #include "SIDefines.h"
14 #include "Utils/AMDGPUBaseInfo.h"
15 #include "Utils/AMDKernelCodeTUtils.h"
16 #include "Utils/AMDGPUAsmUtils.h"
17 #include "llvm/ADT/APFloat.h"
18 #include "llvm/ADT/APInt.h"
19 #include "llvm/ADT/ArrayRef.h"
20 #include "llvm/ADT/SmallBitVector.h"
21 #include "llvm/ADT/SmallString.h"
22 #include "llvm/ADT/STLExtras.h"
23 #include "llvm/ADT/StringRef.h"
24 #include "llvm/ADT/StringSwitch.h"
25 #include "llvm/ADT/Twine.h"
26 #include "llvm/CodeGen/MachineValueType.h"
27 #include "llvm/MC/MCAsmInfo.h"
28 #include "llvm/MC/MCContext.h"
29 #include "llvm/MC/MCExpr.h"
30 #include "llvm/MC/MCInst.h"
31 #include "llvm/MC/MCInstrDesc.h"
32 #include "llvm/MC/MCInstrInfo.h"
33 #include "llvm/MC/MCParser/MCAsmLexer.h"
34 #include "llvm/MC/MCParser/MCAsmParser.h"
35 #include "llvm/MC/MCParser/MCAsmParserExtension.h"
36 #include "llvm/MC/MCParser/MCParsedAsmOperand.h"
37 #include "llvm/MC/MCParser/MCTargetAsmParser.h"
38 #include "llvm/MC/MCRegisterInfo.h"
39 #include "llvm/MC/MCStreamer.h"
40 #include "llvm/MC/MCSubtargetInfo.h"
41 #include "llvm/MC/MCSymbol.h"
42 #include "llvm/Support/Casting.h"
43 #include "llvm/Support/ELF.h"
44 #include "llvm/Support/ErrorHandling.h"
45 #include "llvm/Support/MathExtras.h"
46 #include "llvm/Support/raw_ostream.h"
47 #include "llvm/Support/SMLoc.h"
48 #include "llvm/Support/TargetRegistry.h"
49 #include <algorithm>
50 #include <cassert>
51 #include <cstdint>
52 #include <cstring>
53 #include <iterator>
54 #include <map>
55 #include <memory>
56 #include <string>
57 
58 using namespace llvm;
59 using namespace llvm::AMDGPU;
60 
61 namespace {
62 
63 class AMDGPUAsmParser;
64 
65 enum RegisterKind { IS_UNKNOWN, IS_VGPR, IS_SGPR, IS_TTMP, IS_SPECIAL };
66 
67 //===----------------------------------------------------------------------===//
68 // Operand
69 //===----------------------------------------------------------------------===//
70 
71 class AMDGPUOperand : public MCParsedAsmOperand {
72   enum KindTy {
73     Token,
74     Immediate,
75     Register,
76     Expression
77   } Kind;
78 
79   SMLoc StartLoc, EndLoc;
80   const AMDGPUAsmParser *AsmParser;
81 
82 public:
83   AMDGPUOperand(KindTy Kind_, const AMDGPUAsmParser *AsmParser_)
84     : MCParsedAsmOperand(), Kind(Kind_), AsmParser(AsmParser_) {}
85 
86   typedef std::unique_ptr<AMDGPUOperand> Ptr;
87 
88   struct Modifiers {
89     bool Abs = false;
90     bool Neg = false;
91     bool Sext = false;
92 
93     bool hasFPModifiers() const { return Abs || Neg; }
94     bool hasIntModifiers() const { return Sext; }
95     bool hasModifiers() const { return hasFPModifiers() || hasIntModifiers(); }
96 
97     int64_t getFPModifiersOperand() const {
98       int64_t Operand = 0;
99       Operand |= Abs ? SISrcMods::ABS : 0;
100       Operand |= Neg ? SISrcMods::NEG : 0;
101       return Operand;
102     }
103 
104     int64_t getIntModifiersOperand() const {
105       int64_t Operand = 0;
106       Operand |= Sext ? SISrcMods::SEXT : 0;
107       return Operand;
108     }
109 
110     int64_t getModifiersOperand() const {
111       assert(!(hasFPModifiers() && hasIntModifiers())
112            && "fp and int modifiers should not be used simultaneously");
113       if (hasFPModifiers()) {
114         return getFPModifiersOperand();
115       } else if (hasIntModifiers()) {
116         return getIntModifiersOperand();
117       } else {
118         return 0;
119       }
120     }
121 
122     friend raw_ostream &operator <<(raw_ostream &OS, AMDGPUOperand::Modifiers Mods);
123   };
124 
125   enum ImmTy {
126     ImmTyNone,
127     ImmTyGDS,
128     ImmTyOffen,
129     ImmTyIdxen,
130     ImmTyAddr64,
131     ImmTyOffset,
132     ImmTyOffset0,
133     ImmTyOffset1,
134     ImmTyGLC,
135     ImmTySLC,
136     ImmTyTFE,
137     ImmTyClampSI,
138     ImmTyOModSI,
139     ImmTyDppCtrl,
140     ImmTyDppRowMask,
141     ImmTyDppBankMask,
142     ImmTyDppBoundCtrl,
143     ImmTySdwaDstSel,
144     ImmTySdwaSrc0Sel,
145     ImmTySdwaSrc1Sel,
146     ImmTySdwaDstUnused,
147     ImmTyDMask,
148     ImmTyUNorm,
149     ImmTyDA,
150     ImmTyR128,
151     ImmTyLWE,
152     ImmTyExpTgt,
153     ImmTyExpCompr,
154     ImmTyExpVM,
155     ImmTyHwreg,
156     ImmTyOff,
157     ImmTySendMsg,
158     ImmTyInterpSlot,
159     ImmTyInterpAttr,
160     ImmTyAttrChan
161   };
162 
163   struct TokOp {
164     const char *Data;
165     unsigned Length;
166   };
167 
168   struct ImmOp {
169     int64_t Val;
170     ImmTy Type;
171     bool IsFPImm;
172     Modifiers Mods;
173   };
174 
175   struct RegOp {
176     unsigned RegNo;
177     bool IsForcedVOP3;
178     Modifiers Mods;
179   };
180 
181   union {
182     TokOp Tok;
183     ImmOp Imm;
184     RegOp Reg;
185     const MCExpr *Expr;
186   };
187 
188   bool isToken() const override {
189     if (Kind == Token)
190       return true;
191 
192     if (Kind != Expression || !Expr)
193       return false;
194 
195     // When parsing operands, we can't always tell if something was meant to be
196     // a token, like 'gds', or an expression that references a global variable.
197     // In this case, we assume the string is an expression, and if we need to
198     // interpret is a token, then we treat the symbol name as the token.
199     return isa<MCSymbolRefExpr>(Expr);
200   }
201 
202   bool isImm() const override {
203     return Kind == Immediate;
204   }
205 
206   bool isInlinableImm(MVT type) const;
207   bool isLiteralImm(MVT type) const;
208 
209   bool isRegKind() const {
210     return Kind == Register;
211   }
212 
213   bool isReg() const override {
214     return isRegKind() && !hasModifiers();
215   }
216 
217   bool isRegOrImmWithInputMods(MVT type) const {
218     return isRegKind() || isInlinableImm(type);
219   }
220 
221   bool isRegOrImmWithInt16InputMods() const {
222     return isRegOrImmWithInputMods(MVT::i16);
223   }
224 
225   bool isRegOrImmWithInt32InputMods() const {
226     return isRegOrImmWithInputMods(MVT::i32);
227   }
228 
229   bool isRegOrImmWithInt64InputMods() const {
230     return isRegOrImmWithInputMods(MVT::i64);
231   }
232 
233   bool isRegOrImmWithFP16InputMods() const {
234     return isRegOrImmWithInputMods(MVT::f16);
235   }
236 
237   bool isRegOrImmWithFP32InputMods() const {
238     return isRegOrImmWithInputMods(MVT::f32);
239   }
240 
241   bool isRegOrImmWithFP64InputMods() const {
242     return isRegOrImmWithInputMods(MVT::f64);
243   }
244 
245   bool isVReg() const {
246     return isRegClass(AMDGPU::VGPR_32RegClassID) ||
247            isRegClass(AMDGPU::VReg_64RegClassID) ||
248            isRegClass(AMDGPU::VReg_96RegClassID) ||
249            isRegClass(AMDGPU::VReg_128RegClassID) ||
250            isRegClass(AMDGPU::VReg_256RegClassID) ||
251            isRegClass(AMDGPU::VReg_512RegClassID);
252   }
253 
254   bool isVReg32OrOff() const {
255     return isOff() || isRegClass(AMDGPU::VGPR_32RegClassID);
256   }
257 
258   bool isImmTy(ImmTy ImmT) const {
259     return isImm() && Imm.Type == ImmT;
260   }
261 
262   bool isImmModifier() const {
263     return isImm() && Imm.Type != ImmTyNone;
264   }
265 
266   bool isClampSI() const { return isImmTy(ImmTyClampSI); }
267   bool isOModSI() const { return isImmTy(ImmTyOModSI); }
268   bool isDMask() const { return isImmTy(ImmTyDMask); }
269   bool isUNorm() const { return isImmTy(ImmTyUNorm); }
270   bool isDA() const { return isImmTy(ImmTyDA); }
271   bool isR128() const { return isImmTy(ImmTyUNorm); }
272   bool isLWE() const { return isImmTy(ImmTyLWE); }
273   bool isOff() const { return isImmTy(ImmTyOff); }
274   bool isExpTgt() const { return isImmTy(ImmTyExpTgt); }
275   bool isExpVM() const { return isImmTy(ImmTyExpVM); }
276   bool isExpCompr() const { return isImmTy(ImmTyExpCompr); }
277   bool isOffen() const { return isImmTy(ImmTyOffen); }
278   bool isIdxen() const { return isImmTy(ImmTyIdxen); }
279   bool isAddr64() const { return isImmTy(ImmTyAddr64); }
280   bool isOffset() const { return isImmTy(ImmTyOffset) && isUInt<16>(getImm()); }
281   bool isOffset0() const { return isImmTy(ImmTyOffset0) && isUInt<16>(getImm()); }
282   bool isOffset1() const { return isImmTy(ImmTyOffset1) && isUInt<8>(getImm()); }
283   bool isGDS() const { return isImmTy(ImmTyGDS); }
284   bool isGLC() const { return isImmTy(ImmTyGLC); }
285   bool isSLC() const { return isImmTy(ImmTySLC); }
286   bool isTFE() const { return isImmTy(ImmTyTFE); }
287   bool isBankMask() const { return isImmTy(ImmTyDppBankMask); }
288   bool isRowMask() const { return isImmTy(ImmTyDppRowMask); }
289   bool isBoundCtrl() const { return isImmTy(ImmTyDppBoundCtrl); }
290   bool isSDWADstSel() const { return isImmTy(ImmTySdwaDstSel); }
291   bool isSDWASrc0Sel() const { return isImmTy(ImmTySdwaSrc0Sel); }
292   bool isSDWASrc1Sel() const { return isImmTy(ImmTySdwaSrc1Sel); }
293   bool isSDWADstUnused() const { return isImmTy(ImmTySdwaDstUnused); }
294   bool isInterpSlot() const { return isImmTy(ImmTyInterpSlot); }
295   bool isInterpAttr() const { return isImmTy(ImmTyInterpAttr); }
296   bool isAttrChan() const { return isImmTy(ImmTyAttrChan); }
297 
298   bool isMod() const {
299     return isClampSI() || isOModSI();
300   }
301 
302   bool isRegOrImm() const {
303     return isReg() || isImm();
304   }
305 
306   bool isRegClass(unsigned RCID) const;
307 
308   bool isRegOrInlineNoMods(unsigned RCID, MVT type) const {
309     return (isRegClass(RCID) || isInlinableImm(type)) && !hasModifiers();
310   }
311 
312   bool isSCSrcB16() const {
313     return isRegOrInlineNoMods(AMDGPU::SReg_32RegClassID, MVT::i16);
314   }
315 
316   bool isSCSrcB32() const {
317     return isRegOrInlineNoMods(AMDGPU::SReg_32RegClassID, MVT::i32);
318   }
319 
320   bool isSCSrcB64() const {
321     return isRegOrInlineNoMods(AMDGPU::SReg_64RegClassID, MVT::i64);
322   }
323 
324   bool isSCSrcF16() const {
325     return isRegOrInlineNoMods(AMDGPU::SReg_32RegClassID, MVT::f16);
326   }
327 
328   bool isSCSrcF32() const {
329     return isRegOrInlineNoMods(AMDGPU::SReg_32RegClassID, MVT::f32);
330   }
331 
332   bool isSCSrcF64() const {
333     return isRegOrInlineNoMods(AMDGPU::SReg_64RegClassID, MVT::f64);
334   }
335 
336   bool isSSrcB32() const {
337     return isSCSrcB32() || isLiteralImm(MVT::i32) || isExpr();
338   }
339 
340   bool isSSrcB16() const {
341     return isSCSrcB16() || isLiteralImm(MVT::i16);
342   }
343 
344   bool isSSrcB64() const {
345     // TODO: Find out how SALU supports extension of 32-bit literals to 64 bits.
346     // See isVSrc64().
347     return isSCSrcB64() || isLiteralImm(MVT::i64);
348   }
349 
350   bool isSSrcF32() const {
351     return isSCSrcB32() || isLiteralImm(MVT::f32) || isExpr();
352   }
353 
354   bool isSSrcF64() const {
355     return isSCSrcB64() || isLiteralImm(MVT::f64);
356   }
357 
358   bool isSSrcF16() const {
359     return isSCSrcB16() || isLiteralImm(MVT::f16);
360   }
361 
362   bool isVCSrcB32() const {
363     return isRegOrInlineNoMods(AMDGPU::VS_32RegClassID, MVT::i32);
364   }
365 
366   bool isVCSrcB64() const {
367     return isRegOrInlineNoMods(AMDGPU::VS_64RegClassID, MVT::i64);
368   }
369 
370   bool isVCSrcB16() const {
371     return isRegOrInlineNoMods(AMDGPU::VS_32RegClassID, MVT::i16);
372   }
373 
374   bool isVCSrcF32() const {
375     return isRegOrInlineNoMods(AMDGPU::VS_32RegClassID, MVT::f32);
376   }
377 
378   bool isVCSrcF64() const {
379     return isRegOrInlineNoMods(AMDGPU::VS_64RegClassID, MVT::f64);
380   }
381 
382   bool isVCSrcF16() const {
383     return isRegOrInlineNoMods(AMDGPU::VS_32RegClassID, MVT::f16);
384   }
385 
386   bool isVSrcB32() const {
387     return isVCSrcF32() || isLiteralImm(MVT::i32);
388   }
389 
390   bool isVSrcB64() const {
391     return isVCSrcF64() || isLiteralImm(MVT::i64);
392   }
393 
394   bool isVSrcB16() const {
395     return isVCSrcF16() || isLiteralImm(MVT::i16);
396   }
397 
398   bool isVSrcF32() const {
399     return isVCSrcF32() || isLiteralImm(MVT::f32);
400   }
401 
402   bool isVSrcF64() const {
403     return isVCSrcF64() || isLiteralImm(MVT::f64);
404   }
405 
406   bool isVSrcF16() const {
407     return isVCSrcF16() || isLiteralImm(MVT::f16);
408   }
409 
410   bool isKImmFP32() const {
411     return isLiteralImm(MVT::f32);
412   }
413 
414   bool isKImmFP16() const {
415     return isLiteralImm(MVT::f16);
416   }
417 
418   bool isMem() const override {
419     return false;
420   }
421 
422   bool isExpr() const {
423     return Kind == Expression;
424   }
425 
426   bool isSoppBrTarget() const {
427     return isExpr() || isImm();
428   }
429 
430   bool isSWaitCnt() const;
431   bool isHwreg() const;
432   bool isSendMsg() const;
433   bool isSMRDOffset8() const;
434   bool isSMRDOffset20() const;
435   bool isSMRDLiteralOffset() const;
436   bool isDPPCtrl() const;
437   bool isGPRIdxMode() const;
438 
439   StringRef getExpressionAsToken() const {
440     assert(isExpr());
441     const MCSymbolRefExpr *S = cast<MCSymbolRefExpr>(Expr);
442     return S->getSymbol().getName();
443   }
444 
445   StringRef getToken() const {
446     assert(isToken());
447 
448     if (Kind == Expression)
449       return getExpressionAsToken();
450 
451     return StringRef(Tok.Data, Tok.Length);
452   }
453 
454   int64_t getImm() const {
455     assert(isImm());
456     return Imm.Val;
457   }
458 
459   ImmTy getImmTy() const {
460     assert(isImm());
461     return Imm.Type;
462   }
463 
464   unsigned getReg() const override {
465     return Reg.RegNo;
466   }
467 
468   SMLoc getStartLoc() const override {
469     return StartLoc;
470   }
471 
472   SMLoc getEndLoc() const override {
473     return EndLoc;
474   }
475 
476   Modifiers getModifiers() const {
477     assert(isRegKind() || isImmTy(ImmTyNone));
478     return isRegKind() ? Reg.Mods : Imm.Mods;
479   }
480 
481   void setModifiers(Modifiers Mods) {
482     assert(isRegKind() || isImmTy(ImmTyNone));
483     if (isRegKind())
484       Reg.Mods = Mods;
485     else
486       Imm.Mods = Mods;
487   }
488 
489   bool hasModifiers() const {
490     return getModifiers().hasModifiers();
491   }
492 
493   bool hasFPModifiers() const {
494     return getModifiers().hasFPModifiers();
495   }
496 
497   bool hasIntModifiers() const {
498     return getModifiers().hasIntModifiers();
499   }
500 
501   void addImmOperands(MCInst &Inst, unsigned N, bool ApplyModifiers = true) const;
502 
503   void addLiteralImmOperand(MCInst &Inst, int64_t Val) const;
504 
505   template <unsigned Bitwidth>
506   void addKImmFPOperands(MCInst &Inst, unsigned N) const;
507 
508   void addKImmFP16Operands(MCInst &Inst, unsigned N) const {
509     addKImmFPOperands<16>(Inst, N);
510   }
511 
512   void addKImmFP32Operands(MCInst &Inst, unsigned N) const {
513     addKImmFPOperands<32>(Inst, N);
514   }
515 
516   void addRegOperands(MCInst &Inst, unsigned N) const;
517 
518   void addRegOrImmOperands(MCInst &Inst, unsigned N) const {
519     if (isRegKind())
520       addRegOperands(Inst, N);
521     else if (isExpr())
522       Inst.addOperand(MCOperand::createExpr(Expr));
523     else
524       addImmOperands(Inst, N);
525   }
526 
527   void addRegOrImmWithInputModsOperands(MCInst &Inst, unsigned N) const {
528     Modifiers Mods = getModifiers();
529     Inst.addOperand(MCOperand::createImm(Mods.getModifiersOperand()));
530     if (isRegKind()) {
531       addRegOperands(Inst, N);
532     } else {
533       addImmOperands(Inst, N, false);
534     }
535   }
536 
537   void addRegOrImmWithFPInputModsOperands(MCInst &Inst, unsigned N) const {
538     assert(!hasIntModifiers());
539     addRegOrImmWithInputModsOperands(Inst, N);
540   }
541 
542   void addRegOrImmWithIntInputModsOperands(MCInst &Inst, unsigned N) const {
543     assert(!hasFPModifiers());
544     addRegOrImmWithInputModsOperands(Inst, N);
545   }
546 
547   void addRegWithInputModsOperands(MCInst &Inst, unsigned N) const {
548     Modifiers Mods = getModifiers();
549     Inst.addOperand(MCOperand::createImm(Mods.getModifiersOperand()));
550     assert(isRegKind());
551     addRegOperands(Inst, N);
552   }
553 
554   void addRegWithFPInputModsOperands(MCInst &Inst, unsigned N) const {
555     assert(!hasIntModifiers());
556     addRegWithInputModsOperands(Inst, N);
557   }
558 
559   void addRegWithIntInputModsOperands(MCInst &Inst, unsigned N) const {
560     assert(!hasFPModifiers());
561     addRegWithInputModsOperands(Inst, N);
562   }
563 
564   void addSoppBrTargetOperands(MCInst &Inst, unsigned N) const {
565     if (isImm())
566       addImmOperands(Inst, N);
567     else {
568       assert(isExpr());
569       Inst.addOperand(MCOperand::createExpr(Expr));
570     }
571   }
572 
573   static void printImmTy(raw_ostream& OS, ImmTy Type) {
574     switch (Type) {
575     case ImmTyNone: OS << "None"; break;
576     case ImmTyGDS: OS << "GDS"; break;
577     case ImmTyOffen: OS << "Offen"; break;
578     case ImmTyIdxen: OS << "Idxen"; break;
579     case ImmTyAddr64: OS << "Addr64"; break;
580     case ImmTyOffset: OS << "Offset"; break;
581     case ImmTyOffset0: OS << "Offset0"; break;
582     case ImmTyOffset1: OS << "Offset1"; break;
583     case ImmTyGLC: OS << "GLC"; break;
584     case ImmTySLC: OS << "SLC"; break;
585     case ImmTyTFE: OS << "TFE"; break;
586     case ImmTyClampSI: OS << "ClampSI"; break;
587     case ImmTyOModSI: OS << "OModSI"; break;
588     case ImmTyDppCtrl: OS << "DppCtrl"; break;
589     case ImmTyDppRowMask: OS << "DppRowMask"; break;
590     case ImmTyDppBankMask: OS << "DppBankMask"; break;
591     case ImmTyDppBoundCtrl: OS << "DppBoundCtrl"; break;
592     case ImmTySdwaDstSel: OS << "SdwaDstSel"; break;
593     case ImmTySdwaSrc0Sel: OS << "SdwaSrc0Sel"; break;
594     case ImmTySdwaSrc1Sel: OS << "SdwaSrc1Sel"; break;
595     case ImmTySdwaDstUnused: OS << "SdwaDstUnused"; break;
596     case ImmTyDMask: OS << "DMask"; break;
597     case ImmTyUNorm: OS << "UNorm"; break;
598     case ImmTyDA: OS << "DA"; break;
599     case ImmTyR128: OS << "R128"; break;
600     case ImmTyLWE: OS << "LWE"; break;
601     case ImmTyOff: OS << "Off"; break;
602     case ImmTyExpTgt: OS << "ExpTgt"; break;
603     case ImmTyExpCompr: OS << "ExpCompr"; break;
604     case ImmTyExpVM: OS << "ExpVM"; break;
605     case ImmTyHwreg: OS << "Hwreg"; break;
606     case ImmTySendMsg: OS << "SendMsg"; break;
607     case ImmTyInterpSlot: OS << "InterpSlot"; break;
608     case ImmTyInterpAttr: OS << "InterpAttr"; break;
609     case ImmTyAttrChan: OS << "AttrChan"; break;
610     }
611   }
612 
613   void print(raw_ostream &OS) const override {
614     switch (Kind) {
615     case Register:
616       OS << "<register " << getReg() << " mods: " << Reg.Mods << '>';
617       break;
618     case Immediate:
619       OS << '<' << getImm();
620       if (getImmTy() != ImmTyNone) {
621         OS << " type: "; printImmTy(OS, getImmTy());
622       }
623       OS << " mods: " << Imm.Mods << '>';
624       break;
625     case Token:
626       OS << '\'' << getToken() << '\'';
627       break;
628     case Expression:
629       OS << "<expr " << *Expr << '>';
630       break;
631     }
632   }
633 
634   static AMDGPUOperand::Ptr CreateImm(const AMDGPUAsmParser *AsmParser,
635                                       int64_t Val, SMLoc Loc,
636                                       ImmTy Type = ImmTyNone,
637                                       bool IsFPImm = false) {
638     auto Op = llvm::make_unique<AMDGPUOperand>(Immediate, AsmParser);
639     Op->Imm.Val = Val;
640     Op->Imm.IsFPImm = IsFPImm;
641     Op->Imm.Type = Type;
642     Op->Imm.Mods = Modifiers();
643     Op->StartLoc = Loc;
644     Op->EndLoc = Loc;
645     return Op;
646   }
647 
648   static AMDGPUOperand::Ptr CreateToken(const AMDGPUAsmParser *AsmParser,
649                                         StringRef Str, SMLoc Loc,
650                                         bool HasExplicitEncodingSize = true) {
651     auto Res = llvm::make_unique<AMDGPUOperand>(Token, AsmParser);
652     Res->Tok.Data = Str.data();
653     Res->Tok.Length = Str.size();
654     Res->StartLoc = Loc;
655     Res->EndLoc = Loc;
656     return Res;
657   }
658 
659   static AMDGPUOperand::Ptr CreateReg(const AMDGPUAsmParser *AsmParser,
660                                       unsigned RegNo, SMLoc S,
661                                       SMLoc E,
662                                       bool ForceVOP3) {
663     auto Op = llvm::make_unique<AMDGPUOperand>(Register, AsmParser);
664     Op->Reg.RegNo = RegNo;
665     Op->Reg.Mods = Modifiers();
666     Op->Reg.IsForcedVOP3 = ForceVOP3;
667     Op->StartLoc = S;
668     Op->EndLoc = E;
669     return Op;
670   }
671 
672   static AMDGPUOperand::Ptr CreateExpr(const AMDGPUAsmParser *AsmParser,
673                                        const class MCExpr *Expr, SMLoc S) {
674     auto Op = llvm::make_unique<AMDGPUOperand>(Expression, AsmParser);
675     Op->Expr = Expr;
676     Op->StartLoc = S;
677     Op->EndLoc = S;
678     return Op;
679   }
680 };
681 
682 raw_ostream &operator <<(raw_ostream &OS, AMDGPUOperand::Modifiers Mods) {
683   OS << "abs:" << Mods.Abs << " neg: " << Mods.Neg << " sext:" << Mods.Sext;
684   return OS;
685 }
686 
687 //===----------------------------------------------------------------------===//
688 // AsmParser
689 //===----------------------------------------------------------------------===//
690 
691 // Holds info related to the current kernel, e.g. count of SGPRs used.
692 // Kernel scope begins at .amdgpu_hsa_kernel directive, ends at next
693 // .amdgpu_hsa_kernel or at EOF.
694 class KernelScopeInfo {
695   int SgprIndexUnusedMin = -1;
696   int VgprIndexUnusedMin = -1;
697   MCContext *Ctx = nullptr;
698 
699   void usesSgprAt(int i) {
700     if (i >= SgprIndexUnusedMin) {
701       SgprIndexUnusedMin = ++i;
702       if (Ctx) {
703         MCSymbol * const Sym = Ctx->getOrCreateSymbol(Twine(".kernel.sgpr_count"));
704         Sym->setVariableValue(MCConstantExpr::create(SgprIndexUnusedMin, *Ctx));
705       }
706     }
707   }
708 
709   void usesVgprAt(int i) {
710     if (i >= VgprIndexUnusedMin) {
711       VgprIndexUnusedMin = ++i;
712       if (Ctx) {
713         MCSymbol * const Sym = Ctx->getOrCreateSymbol(Twine(".kernel.vgpr_count"));
714         Sym->setVariableValue(MCConstantExpr::create(VgprIndexUnusedMin, *Ctx));
715       }
716     }
717   }
718 
719 public:
720   KernelScopeInfo() = default;
721 
722   void initialize(MCContext &Context) {
723     Ctx = &Context;
724     usesSgprAt(SgprIndexUnusedMin = -1);
725     usesVgprAt(VgprIndexUnusedMin = -1);
726   }
727 
728   void usesRegister(RegisterKind RegKind, unsigned DwordRegIndex, unsigned RegWidth) {
729     switch (RegKind) {
730       case IS_SGPR: usesSgprAt(DwordRegIndex + RegWidth - 1); break;
731       case IS_VGPR: usesVgprAt(DwordRegIndex + RegWidth - 1); break;
732       default: break;
733     }
734   }
735 };
736 
737 class AMDGPUAsmParser : public MCTargetAsmParser {
738   const MCInstrInfo &MII;
739   MCAsmParser &Parser;
740 
741   unsigned ForcedEncodingSize = 0;
742   bool ForcedDPP = false;
743   bool ForcedSDWA = false;
744   KernelScopeInfo KernelScope;
745 
746   /// @name Auto-generated Match Functions
747   /// {
748 
749 #define GET_ASSEMBLER_HEADER
750 #include "AMDGPUGenAsmMatcher.inc"
751 
752   /// }
753 
754 private:
755   bool ParseAsAbsoluteExpression(uint32_t &Ret);
756   bool ParseDirectiveMajorMinor(uint32_t &Major, uint32_t &Minor);
757   bool ParseDirectiveHSACodeObjectVersion();
758   bool ParseDirectiveHSACodeObjectISA();
759   bool ParseDirectiveRuntimeMetadata();
760   bool ParseAMDKernelCodeTValue(StringRef ID, amd_kernel_code_t &Header);
761   bool ParseDirectiveAMDKernelCodeT();
762   bool ParseSectionDirectiveHSAText();
763   bool subtargetHasRegister(const MCRegisterInfo &MRI, unsigned RegNo) const;
764   bool ParseDirectiveAMDGPUHsaKernel();
765   bool ParseDirectiveAMDGPUHsaModuleGlobal();
766   bool ParseDirectiveAMDGPUHsaProgramGlobal();
767   bool ParseSectionDirectiveHSADataGlobalAgent();
768   bool ParseSectionDirectiveHSADataGlobalProgram();
769   bool ParseSectionDirectiveHSARodataReadonlyAgent();
770   bool AddNextRegisterToList(unsigned& Reg, unsigned& RegWidth,
771                              RegisterKind RegKind, unsigned Reg1,
772                              unsigned RegNum);
773   bool ParseAMDGPURegister(RegisterKind& RegKind, unsigned& Reg,
774                            unsigned& RegNum, unsigned& RegWidth,
775                            unsigned *DwordRegIndex);
776   void cvtMubufImpl(MCInst &Inst, const OperandVector &Operands,
777                     bool IsAtomic, bool IsAtomicReturn);
778   void cvtDSImpl(MCInst &Inst, const OperandVector &Operands,
779                  bool IsGdsHardcoded);
780 
781 public:
782   enum AMDGPUMatchResultTy {
783     Match_PreferE32 = FIRST_TARGET_MATCH_RESULT_TY
784   };
785 
786   AMDGPUAsmParser(const MCSubtargetInfo &STI, MCAsmParser &_Parser,
787                const MCInstrInfo &MII,
788                const MCTargetOptions &Options)
789       : MCTargetAsmParser(Options, STI), MII(MII), Parser(_Parser) {
790     MCAsmParserExtension::Initialize(Parser);
791 
792     if (getSTI().getFeatureBits().none()) {
793       // Set default features.
794       copySTI().ToggleFeature("SOUTHERN_ISLANDS");
795     }
796 
797     setAvailableFeatures(ComputeAvailableFeatures(getSTI().getFeatureBits()));
798 
799     {
800       // TODO: make those pre-defined variables read-only.
801       // Currently there is none suitable machinery in the core llvm-mc for this.
802       // MCSymbol::isRedefinable is intended for another purpose, and
803       // AsmParser::parseDirectiveSet() cannot be specialized for specific target.
804       AMDGPU::IsaVersion Isa = AMDGPU::getIsaVersion(getSTI().getFeatureBits());
805       MCContext &Ctx = getContext();
806       MCSymbol *Sym = Ctx.getOrCreateSymbol(Twine(".option.machine_version_major"));
807       Sym->setVariableValue(MCConstantExpr::create(Isa.Major, Ctx));
808       Sym = Ctx.getOrCreateSymbol(Twine(".option.machine_version_minor"));
809       Sym->setVariableValue(MCConstantExpr::create(Isa.Minor, Ctx));
810       Sym = Ctx.getOrCreateSymbol(Twine(".option.machine_version_stepping"));
811       Sym->setVariableValue(MCConstantExpr::create(Isa.Stepping, Ctx));
812     }
813     KernelScope.initialize(getContext());
814   }
815 
816   bool isSI() const {
817     return AMDGPU::isSI(getSTI());
818   }
819 
820   bool isCI() const {
821     return AMDGPU::isCI(getSTI());
822   }
823 
824   bool isVI() const {
825     return AMDGPU::isVI(getSTI());
826   }
827 
828   bool hasInv2PiInlineImm() const {
829     return getSTI().getFeatureBits()[AMDGPU::FeatureInv2PiInlineImm];
830   }
831 
832   bool hasSGPR102_SGPR103() const {
833     return !isVI();
834   }
835 
836   AMDGPUTargetStreamer &getTargetStreamer() {
837     MCTargetStreamer &TS = *getParser().getStreamer().getTargetStreamer();
838     return static_cast<AMDGPUTargetStreamer &>(TS);
839   }
840 
841   const MCRegisterInfo *getMRI() const {
842     // We need this const_cast because for some reason getContext() is not const
843     // in MCAsmParser.
844     return const_cast<AMDGPUAsmParser*>(this)->getContext().getRegisterInfo();
845   }
846 
847   const MCInstrInfo *getMII() const {
848     return &MII;
849   }
850 
851   void setForcedEncodingSize(unsigned Size) { ForcedEncodingSize = Size; }
852   void setForcedDPP(bool ForceDPP_) { ForcedDPP = ForceDPP_; }
853   void setForcedSDWA(bool ForceSDWA_) { ForcedSDWA = ForceSDWA_; }
854 
855   unsigned getForcedEncodingSize() const { return ForcedEncodingSize; }
856   bool isForcedVOP3() const { return ForcedEncodingSize == 64; }
857   bool isForcedDPP() const { return ForcedDPP; }
858   bool isForcedSDWA() const { return ForcedSDWA; }
859   ArrayRef<unsigned> getMatchedVariants() const;
860 
861   std::unique_ptr<AMDGPUOperand> parseRegister();
862   bool ParseRegister(unsigned &RegNo, SMLoc &StartLoc, SMLoc &EndLoc) override;
863   unsigned checkTargetMatchPredicate(MCInst &Inst) override;
864   unsigned validateTargetOperandClass(MCParsedAsmOperand &Op,
865                                       unsigned Kind) override;
866   bool MatchAndEmitInstruction(SMLoc IDLoc, unsigned &Opcode,
867                                OperandVector &Operands, MCStreamer &Out,
868                                uint64_t &ErrorInfo,
869                                bool MatchingInlineAsm) override;
870   bool ParseDirective(AsmToken DirectiveID) override;
871   OperandMatchResultTy parseOperand(OperandVector &Operands, StringRef Mnemonic);
872   StringRef parseMnemonicSuffix(StringRef Name);
873   bool ParseInstruction(ParseInstructionInfo &Info, StringRef Name,
874                         SMLoc NameLoc, OperandVector &Operands) override;
875   //bool ProcessInstruction(MCInst &Inst);
876 
877   OperandMatchResultTy parseIntWithPrefix(const char *Prefix, int64_t &Int);
878   OperandMatchResultTy
879   parseIntWithPrefix(const char *Prefix, OperandVector &Operands,
880                      AMDGPUOperand::ImmTy ImmTy = AMDGPUOperand::ImmTyNone,
881                      bool (*ConvertResult)(int64_t &) = nullptr);
882   OperandMatchResultTy
883   parseNamedBit(const char *Name, OperandVector &Operands,
884                 AMDGPUOperand::ImmTy ImmTy = AMDGPUOperand::ImmTyNone);
885   OperandMatchResultTy parseStringWithPrefix(StringRef Prefix,
886                                              StringRef &Value);
887 
888   OperandMatchResultTy parseImm(OperandVector &Operands);
889   OperandMatchResultTy parseReg(OperandVector &Operands);
890   OperandMatchResultTy parseRegOrImm(OperandVector &Operands);
891   OperandMatchResultTy parseRegOrImmWithFPInputMods(OperandVector &Operands, bool AllowImm = true);
892   OperandMatchResultTy parseRegOrImmWithIntInputMods(OperandVector &Operands, bool AllowImm = true);
893   OperandMatchResultTy parseRegWithFPInputMods(OperandVector &Operands);
894   OperandMatchResultTy parseRegWithIntInputMods(OperandVector &Operands);
895   OperandMatchResultTy parseVReg32OrOff(OperandVector &Operands);
896 
897   void cvtDSOffset01(MCInst &Inst, const OperandVector &Operands);
898   void cvtDS(MCInst &Inst, const OperandVector &Operands) { cvtDSImpl(Inst, Operands, false); }
899   void cvtDSGds(MCInst &Inst, const OperandVector &Operands) { cvtDSImpl(Inst, Operands, true); }
900   void cvtExp(MCInst &Inst, const OperandVector &Operands);
901 
902   bool parseCnt(int64_t &IntVal);
903   OperandMatchResultTy parseSWaitCntOps(OperandVector &Operands);
904   OperandMatchResultTy parseHwreg(OperandVector &Operands);
905 
906 private:
907   struct OperandInfoTy {
908     int64_t Id;
909     bool IsSymbolic;
910     OperandInfoTy(int64_t Id_) : Id(Id_), IsSymbolic(false) { }
911   };
912 
913   bool parseSendMsgConstruct(OperandInfoTy &Msg, OperandInfoTy &Operation, int64_t &StreamId);
914   bool parseHwregConstruct(OperandInfoTy &HwReg, int64_t &Offset, int64_t &Width);
915 
916   void errorExpTgt();
917   OperandMatchResultTy parseExpTgtImpl(StringRef Str, uint8_t &Val);
918 
919 public:
920   OperandMatchResultTy parseOptionalOperand(OperandVector &Operands);
921 
922   OperandMatchResultTy parseExpTgt(OperandVector &Operands);
923   OperandMatchResultTy parseSendMsgOp(OperandVector &Operands);
924   OperandMatchResultTy parseInterpSlot(OperandVector &Operands);
925   OperandMatchResultTy parseInterpAttr(OperandVector &Operands);
926   OperandMatchResultTy parseSOppBrTarget(OperandVector &Operands);
927 
928   void cvtMubuf(MCInst &Inst, const OperandVector &Operands) { cvtMubufImpl(Inst, Operands, false, false); }
929   void cvtMubufAtomic(MCInst &Inst, const OperandVector &Operands) { cvtMubufImpl(Inst, Operands, true, false); }
930   void cvtMubufAtomicReturn(MCInst &Inst, const OperandVector &Operands) { cvtMubufImpl(Inst, Operands, true, true); }
931   AMDGPUOperand::Ptr defaultGLC() const;
932   AMDGPUOperand::Ptr defaultSLC() const;
933   AMDGPUOperand::Ptr defaultTFE() const;
934 
935   AMDGPUOperand::Ptr defaultDMask() const;
936   AMDGPUOperand::Ptr defaultUNorm() const;
937   AMDGPUOperand::Ptr defaultDA() const;
938   AMDGPUOperand::Ptr defaultR128() const;
939   AMDGPUOperand::Ptr defaultLWE() const;
940   AMDGPUOperand::Ptr defaultSMRDOffset8() const;
941   AMDGPUOperand::Ptr defaultSMRDOffset20() const;
942   AMDGPUOperand::Ptr defaultSMRDLiteralOffset() const;
943 
944   OperandMatchResultTy parseOModOperand(OperandVector &Operands);
945 
946   void cvtId(MCInst &Inst, const OperandVector &Operands);
947   void cvtVOP3_2_mod(MCInst &Inst, const OperandVector &Operands);
948   void cvtVOP3(MCInst &Inst, const OperandVector &Operands);
949 
950   void cvtMIMG(MCInst &Inst, const OperandVector &Operands);
951   void cvtMIMGAtomic(MCInst &Inst, const OperandVector &Operands);
952 
953   OperandMatchResultTy parseDPPCtrl(OperandVector &Operands);
954   AMDGPUOperand::Ptr defaultRowMask() const;
955   AMDGPUOperand::Ptr defaultBankMask() const;
956   AMDGPUOperand::Ptr defaultBoundCtrl() const;
957   void cvtDPP(MCInst &Inst, const OperandVector &Operands);
958 
959   OperandMatchResultTy parseSDWASel(OperandVector &Operands, StringRef Prefix,
960                                     AMDGPUOperand::ImmTy Type);
961   OperandMatchResultTy parseSDWADstUnused(OperandVector &Operands);
962   void cvtSdwaVOP1(MCInst &Inst, const OperandVector &Operands);
963   void cvtSdwaVOP2(MCInst &Inst, const OperandVector &Operands);
964   void cvtSdwaVOPC(MCInst &Inst, const OperandVector &Operands);
965   void cvtSDWA(MCInst &Inst, const OperandVector &Operands,
966                uint64_t BasicInstType);
967 };
968 
969 struct OptionalOperand {
970   const char *Name;
971   AMDGPUOperand::ImmTy Type;
972   bool IsBit;
973   bool (*ConvertResult)(int64_t&);
974 };
975 
976 } // end anonymous namespace
977 
978 // May be called with integer type with equivalent bitwidth.
979 static const fltSemantics *getFltSemantics(unsigned Size) {
980   switch (Size) {
981   case 4:
982     return &APFloat::IEEEsingle();
983   case 8:
984     return &APFloat::IEEEdouble();
985   case 2:
986     return &APFloat::IEEEhalf();
987   default:
988     llvm_unreachable("unsupported fp type");
989   }
990 }
991 
992 static const fltSemantics *getFltSemantics(MVT VT) {
993   return getFltSemantics(VT.getSizeInBits() / 8);
994 }
995 
996 //===----------------------------------------------------------------------===//
997 // Operand
998 //===----------------------------------------------------------------------===//
999 
1000 static bool canLosslesslyConvertToFPType(APFloat &FPLiteral, MVT VT) {
1001   bool Lost;
1002 
1003   // Convert literal to single precision
1004   APFloat::opStatus Status = FPLiteral.convert(*getFltSemantics(VT),
1005                                                APFloat::rmNearestTiesToEven,
1006                                                &Lost);
1007   // We allow precision lost but not overflow or underflow
1008   if (Status != APFloat::opOK &&
1009       Lost &&
1010       ((Status & APFloat::opOverflow)  != 0 ||
1011        (Status & APFloat::opUnderflow) != 0)) {
1012     return false;
1013   }
1014 
1015   return true;
1016 }
1017 
1018 bool AMDGPUOperand::isInlinableImm(MVT type) const {
1019   if (!isImmTy(ImmTyNone)) {
1020     // Only plain immediates are inlinable (e.g. "clamp" attribute is not)
1021     return false;
1022   }
1023   // TODO: We should avoid using host float here. It would be better to
1024   // check the float bit values which is what a few other places do.
1025   // We've had bot failures before due to weird NaN support on mips hosts.
1026 
1027   APInt Literal(64, Imm.Val);
1028 
1029   if (Imm.IsFPImm) { // We got fp literal token
1030     if (type == MVT::f64 || type == MVT::i64) { // Expected 64-bit operand
1031       return AMDGPU::isInlinableLiteral64(Imm.Val,
1032                                           AsmParser->hasInv2PiInlineImm());
1033     }
1034 
1035     APFloat FPLiteral(APFloat::IEEEdouble(), APInt(64, Imm.Val));
1036     if (!canLosslesslyConvertToFPType(FPLiteral, type))
1037       return false;
1038 
1039     if (type.getScalarSizeInBits() == 16) {
1040       return AMDGPU::isInlinableLiteral16(
1041         static_cast<int32_t>(FPLiteral.bitcastToAPInt().getZExtValue()),
1042         AsmParser->hasInv2PiInlineImm());
1043     }
1044 
1045     // Check if single precision literal is inlinable
1046     return AMDGPU::isInlinableLiteral32(
1047       static_cast<int32_t>(FPLiteral.bitcastToAPInt().getZExtValue()),
1048       AsmParser->hasInv2PiInlineImm());
1049   }
1050 
1051   // We got int literal token.
1052   if (type == MVT::f64 || type == MVT::i64) { // Expected 64-bit operand
1053     return AMDGPU::isInlinableLiteral64(Imm.Val,
1054                                         AsmParser->hasInv2PiInlineImm());
1055   }
1056 
1057   if (type.getScalarSizeInBits() == 16) {
1058     return AMDGPU::isInlinableLiteral16(
1059       static_cast<int16_t>(Literal.getLoBits(16).getSExtValue()),
1060       AsmParser->hasInv2PiInlineImm());
1061   }
1062 
1063   return AMDGPU::isInlinableLiteral32(
1064     static_cast<int32_t>(Literal.getLoBits(32).getZExtValue()),
1065     AsmParser->hasInv2PiInlineImm());
1066 }
1067 
1068 bool AMDGPUOperand::isLiteralImm(MVT type) const {
1069   // Check that this imediate can be added as literal
1070   if (!isImmTy(ImmTyNone)) {
1071     return false;
1072   }
1073 
1074   if (!Imm.IsFPImm) {
1075     // We got int literal token.
1076 
1077     unsigned Size = type.getSizeInBits();
1078     if (Size == 64)
1079       Size = 32;
1080 
1081     // FIXME: 64-bit operands can zero extend, sign extend, or pad zeroes for FP
1082     // types.
1083     return isUIntN(Size, Imm.Val) || isIntN(Size, Imm.Val);
1084   }
1085 
1086   // We got fp literal token
1087   if (type == MVT::f64) { // Expected 64-bit fp operand
1088     // We would set low 64-bits of literal to zeroes but we accept this literals
1089     return true;
1090   }
1091 
1092   if (type == MVT::i64) { // Expected 64-bit int operand
1093     // We don't allow fp literals in 64-bit integer instructions. It is
1094     // unclear how we should encode them.
1095     return false;
1096   }
1097 
1098   APFloat FPLiteral(APFloat::IEEEdouble(), APInt(64, Imm.Val));
1099   return canLosslesslyConvertToFPType(FPLiteral, type);
1100 }
1101 
1102 bool AMDGPUOperand::isRegClass(unsigned RCID) const {
1103   return isRegKind() && AsmParser->getMRI()->getRegClass(RCID).contains(getReg());
1104 }
1105 
1106 void AMDGPUOperand::addImmOperands(MCInst &Inst, unsigned N, bool ApplyModifiers) const {
1107   int64_t Val = Imm.Val;
1108   if (isImmTy(ImmTyNone) && ApplyModifiers && Imm.Mods.hasFPModifiers() && Imm.Mods.Neg) {
1109     // Apply modifiers to immediate value. Only negate can get here
1110     if (Imm.IsFPImm) {
1111       APFloat F(BitsToDouble(Val));
1112       F.changeSign();
1113       Val = F.bitcastToAPInt().getZExtValue();
1114     } else {
1115       Val = -Val;
1116     }
1117   }
1118 
1119   if (AMDGPU::isSISrcOperand(AsmParser->getMII()->get(Inst.getOpcode()),
1120                              Inst.getNumOperands())) {
1121     addLiteralImmOperand(Inst, Val);
1122   } else {
1123     Inst.addOperand(MCOperand::createImm(Val));
1124   }
1125 }
1126 
1127 void AMDGPUOperand::addLiteralImmOperand(MCInst &Inst, int64_t Val) const {
1128   const auto& InstDesc = AsmParser->getMII()->get(Inst.getOpcode());
1129   auto OpNum = Inst.getNumOperands();
1130   // Check that this operand accepts literals
1131   assert(AMDGPU::isSISrcOperand(InstDesc, OpNum));
1132 
1133   auto OpSize = AMDGPU::getOperandSize(InstDesc, OpNum); // expected operand size
1134 
1135   if (Imm.IsFPImm) { // We got fp literal token
1136     APInt Literal(64, Val);
1137 
1138     switch (OpSize) {
1139     case 8:
1140       if (AMDGPU::isInlinableLiteral64(Literal.getZExtValue(),
1141                                        AsmParser->hasInv2PiInlineImm())) {
1142         Inst.addOperand(MCOperand::createImm(Literal.getZExtValue()));
1143         return;
1144       }
1145 
1146       // Non-inlineable
1147       if (AMDGPU::isSISrcFPOperand(InstDesc, OpNum)) { // Expected 64-bit fp operand
1148         // For fp operands we check if low 32 bits are zeros
1149         if (Literal.getLoBits(32) != 0) {
1150           const_cast<AMDGPUAsmParser *>(AsmParser)->Warning(Inst.getLoc(),
1151           "Can't encode literal as exact 64-bit floating-point operand. "
1152           "Low 32-bits will be set to zero");
1153         }
1154 
1155         Inst.addOperand(MCOperand::createImm(Literal.lshr(32).getZExtValue()));
1156         return;
1157       }
1158 
1159       // We don't allow fp literals in 64-bit integer instructions. It is
1160       // unclear how we should encode them. This case should be checked earlier
1161       // in predicate methods (isLiteralImm())
1162       llvm_unreachable("fp literal in 64-bit integer instruction.");
1163 
1164     case 4:
1165     case 2: {
1166       bool lost;
1167       APFloat FPLiteral(APFloat::IEEEdouble(), Literal);
1168       // Convert literal to single precision
1169       FPLiteral.convert(*getFltSemantics(OpSize),
1170                         APFloat::rmNearestTiesToEven, &lost);
1171       // We allow precision lost but not overflow or underflow. This should be
1172       // checked earlier in isLiteralImm()
1173       Inst.addOperand(MCOperand::createImm(FPLiteral.bitcastToAPInt().getZExtValue()));
1174       return;
1175     }
1176     default:
1177       llvm_unreachable("invalid operand size");
1178     }
1179 
1180     return;
1181   }
1182 
1183    // We got int literal token.
1184   // Only sign extend inline immediates.
1185   // FIXME: No errors on truncation
1186   switch (OpSize) {
1187   case 4:
1188     if (isInt<32>(Val) &&
1189         AMDGPU::isInlinableLiteral32(static_cast<int32_t>(Val),
1190                                      AsmParser->hasInv2PiInlineImm())) {
1191       Inst.addOperand(MCOperand::createImm(Val));
1192       return;
1193     }
1194 
1195     Inst.addOperand(MCOperand::createImm(Val & 0xffffffff));
1196     return;
1197 
1198   case 8:
1199     if (AMDGPU::isInlinableLiteral64(Val,
1200                                      AsmParser->hasInv2PiInlineImm())) {
1201       Inst.addOperand(MCOperand::createImm(Val));
1202       return;
1203     }
1204 
1205     Inst.addOperand(MCOperand::createImm(Lo_32(Val)));
1206     return;
1207 
1208   case 2:
1209     if (isInt<16>(Val) &&
1210         AMDGPU::isInlinableLiteral16(static_cast<int16_t>(Val),
1211                                      AsmParser->hasInv2PiInlineImm())) {
1212       Inst.addOperand(MCOperand::createImm(Val));
1213       return;
1214     }
1215 
1216     Inst.addOperand(MCOperand::createImm(Val & 0xffff));
1217     return;
1218 
1219   default:
1220     llvm_unreachable("invalid operand size");
1221   }
1222 }
1223 
1224 template <unsigned Bitwidth>
1225 void AMDGPUOperand::addKImmFPOperands(MCInst &Inst, unsigned N) const {
1226   APInt Literal(64, Imm.Val);
1227 
1228   if (!Imm.IsFPImm) {
1229     // We got int literal token.
1230     Inst.addOperand(MCOperand::createImm(Literal.getLoBits(Bitwidth).getZExtValue()));
1231     return;
1232   }
1233 
1234   bool Lost;
1235   APFloat FPLiteral(APFloat::IEEEdouble(), Literal);
1236   FPLiteral.convert(*getFltSemantics(Bitwidth / 8),
1237                     APFloat::rmNearestTiesToEven, &Lost);
1238   Inst.addOperand(MCOperand::createImm(FPLiteral.bitcastToAPInt().getZExtValue()));
1239 }
1240 
1241 void AMDGPUOperand::addRegOperands(MCInst &Inst, unsigned N) const {
1242   Inst.addOperand(MCOperand::createReg(AMDGPU::getMCReg(getReg(), AsmParser->getSTI())));
1243 }
1244 
1245 //===----------------------------------------------------------------------===//
1246 // AsmParser
1247 //===----------------------------------------------------------------------===//
1248 
1249 static int getRegClass(RegisterKind Is, unsigned RegWidth) {
1250   if (Is == IS_VGPR) {
1251     switch (RegWidth) {
1252       default: return -1;
1253       case 1: return AMDGPU::VGPR_32RegClassID;
1254       case 2: return AMDGPU::VReg_64RegClassID;
1255       case 3: return AMDGPU::VReg_96RegClassID;
1256       case 4: return AMDGPU::VReg_128RegClassID;
1257       case 8: return AMDGPU::VReg_256RegClassID;
1258       case 16: return AMDGPU::VReg_512RegClassID;
1259     }
1260   } else if (Is == IS_TTMP) {
1261     switch (RegWidth) {
1262       default: return -1;
1263       case 1: return AMDGPU::TTMP_32RegClassID;
1264       case 2: return AMDGPU::TTMP_64RegClassID;
1265       case 4: return AMDGPU::TTMP_128RegClassID;
1266     }
1267   } else if (Is == IS_SGPR) {
1268     switch (RegWidth) {
1269       default: return -1;
1270       case 1: return AMDGPU::SGPR_32RegClassID;
1271       case 2: return AMDGPU::SGPR_64RegClassID;
1272       case 4: return AMDGPU::SGPR_128RegClassID;
1273       case 8: return AMDGPU::SReg_256RegClassID;
1274       case 16: return AMDGPU::SReg_512RegClassID;
1275     }
1276   }
1277   return -1;
1278 }
1279 
1280 static unsigned getSpecialRegForName(StringRef RegName) {
1281   return StringSwitch<unsigned>(RegName)
1282     .Case("exec", AMDGPU::EXEC)
1283     .Case("vcc", AMDGPU::VCC)
1284     .Case("flat_scratch", AMDGPU::FLAT_SCR)
1285     .Case("m0", AMDGPU::M0)
1286     .Case("scc", AMDGPU::SCC)
1287     .Case("tba", AMDGPU::TBA)
1288     .Case("tma", AMDGPU::TMA)
1289     .Case("flat_scratch_lo", AMDGPU::FLAT_SCR_LO)
1290     .Case("flat_scratch_hi", AMDGPU::FLAT_SCR_HI)
1291     .Case("vcc_lo", AMDGPU::VCC_LO)
1292     .Case("vcc_hi", AMDGPU::VCC_HI)
1293     .Case("exec_lo", AMDGPU::EXEC_LO)
1294     .Case("exec_hi", AMDGPU::EXEC_HI)
1295     .Case("tma_lo", AMDGPU::TMA_LO)
1296     .Case("tma_hi", AMDGPU::TMA_HI)
1297     .Case("tba_lo", AMDGPU::TBA_LO)
1298     .Case("tba_hi", AMDGPU::TBA_HI)
1299     .Default(0);
1300 }
1301 
1302 bool AMDGPUAsmParser::ParseRegister(unsigned &RegNo, SMLoc &StartLoc,
1303                                     SMLoc &EndLoc) {
1304   auto R = parseRegister();
1305   if (!R) return true;
1306   assert(R->isReg());
1307   RegNo = R->getReg();
1308   StartLoc = R->getStartLoc();
1309   EndLoc = R->getEndLoc();
1310   return false;
1311 }
1312 
1313 bool AMDGPUAsmParser::AddNextRegisterToList(unsigned &Reg, unsigned &RegWidth,
1314                                             RegisterKind RegKind, unsigned Reg1,
1315                                             unsigned RegNum) {
1316   switch (RegKind) {
1317   case IS_SPECIAL:
1318     if (Reg == AMDGPU::EXEC_LO && Reg1 == AMDGPU::EXEC_HI) {
1319       Reg = AMDGPU::EXEC;
1320       RegWidth = 2;
1321       return true;
1322     }
1323     if (Reg == AMDGPU::FLAT_SCR_LO && Reg1 == AMDGPU::FLAT_SCR_HI) {
1324       Reg = AMDGPU::FLAT_SCR;
1325       RegWidth = 2;
1326       return true;
1327     }
1328     if (Reg == AMDGPU::VCC_LO && Reg1 == AMDGPU::VCC_HI) {
1329       Reg = AMDGPU::VCC;
1330       RegWidth = 2;
1331       return true;
1332     }
1333     if (Reg == AMDGPU::TBA_LO && Reg1 == AMDGPU::TBA_HI) {
1334       Reg = AMDGPU::TBA;
1335       RegWidth = 2;
1336       return true;
1337     }
1338     if (Reg == AMDGPU::TMA_LO && Reg1 == AMDGPU::TMA_HI) {
1339       Reg = AMDGPU::TMA;
1340       RegWidth = 2;
1341       return true;
1342     }
1343     return false;
1344   case IS_VGPR:
1345   case IS_SGPR:
1346   case IS_TTMP:
1347     if (Reg1 != Reg + RegWidth) {
1348       return false;
1349     }
1350     RegWidth++;
1351     return true;
1352   default:
1353     llvm_unreachable("unexpected register kind");
1354   }
1355 }
1356 
1357 bool AMDGPUAsmParser::ParseAMDGPURegister(RegisterKind &RegKind, unsigned &Reg,
1358                                           unsigned &RegNum, unsigned &RegWidth,
1359                                           unsigned *DwordRegIndex) {
1360   if (DwordRegIndex) { *DwordRegIndex = 0; }
1361   const MCRegisterInfo *TRI = getContext().getRegisterInfo();
1362   if (getLexer().is(AsmToken::Identifier)) {
1363     StringRef RegName = Parser.getTok().getString();
1364     if ((Reg = getSpecialRegForName(RegName))) {
1365       Parser.Lex();
1366       RegKind = IS_SPECIAL;
1367     } else {
1368       unsigned RegNumIndex = 0;
1369       if (RegName[0] == 'v') {
1370         RegNumIndex = 1;
1371         RegKind = IS_VGPR;
1372       } else if (RegName[0] == 's') {
1373         RegNumIndex = 1;
1374         RegKind = IS_SGPR;
1375       } else if (RegName.startswith("ttmp")) {
1376         RegNumIndex = strlen("ttmp");
1377         RegKind = IS_TTMP;
1378       } else {
1379         return false;
1380       }
1381       if (RegName.size() > RegNumIndex) {
1382         // Single 32-bit register: vXX.
1383         if (RegName.substr(RegNumIndex).getAsInteger(10, RegNum))
1384           return false;
1385         Parser.Lex();
1386         RegWidth = 1;
1387       } else {
1388         // Range of registers: v[XX:YY]. ":YY" is optional.
1389         Parser.Lex();
1390         int64_t RegLo, RegHi;
1391         if (getLexer().isNot(AsmToken::LBrac))
1392           return false;
1393         Parser.Lex();
1394 
1395         if (getParser().parseAbsoluteExpression(RegLo))
1396           return false;
1397 
1398         const bool isRBrace = getLexer().is(AsmToken::RBrac);
1399         if (!isRBrace && getLexer().isNot(AsmToken::Colon))
1400           return false;
1401         Parser.Lex();
1402 
1403         if (isRBrace) {
1404           RegHi = RegLo;
1405         } else {
1406           if (getParser().parseAbsoluteExpression(RegHi))
1407             return false;
1408 
1409           if (getLexer().isNot(AsmToken::RBrac))
1410             return false;
1411           Parser.Lex();
1412         }
1413         RegNum = (unsigned) RegLo;
1414         RegWidth = (RegHi - RegLo) + 1;
1415       }
1416     }
1417   } else if (getLexer().is(AsmToken::LBrac)) {
1418     // List of consecutive registers: [s0,s1,s2,s3]
1419     Parser.Lex();
1420     if (!ParseAMDGPURegister(RegKind, Reg, RegNum, RegWidth, nullptr))
1421       return false;
1422     if (RegWidth != 1)
1423       return false;
1424     RegisterKind RegKind1;
1425     unsigned Reg1, RegNum1, RegWidth1;
1426     do {
1427       if (getLexer().is(AsmToken::Comma)) {
1428         Parser.Lex();
1429       } else if (getLexer().is(AsmToken::RBrac)) {
1430         Parser.Lex();
1431         break;
1432       } else if (ParseAMDGPURegister(RegKind1, Reg1, RegNum1, RegWidth1, nullptr)) {
1433         if (RegWidth1 != 1) {
1434           return false;
1435         }
1436         if (RegKind1 != RegKind) {
1437           return false;
1438         }
1439         if (!AddNextRegisterToList(Reg, RegWidth, RegKind1, Reg1, RegNum1)) {
1440           return false;
1441         }
1442       } else {
1443         return false;
1444       }
1445     } while (true);
1446   } else {
1447     return false;
1448   }
1449   switch (RegKind) {
1450   case IS_SPECIAL:
1451     RegNum = 0;
1452     RegWidth = 1;
1453     break;
1454   case IS_VGPR:
1455   case IS_SGPR:
1456   case IS_TTMP:
1457   {
1458     unsigned Size = 1;
1459     if (RegKind == IS_SGPR || RegKind == IS_TTMP) {
1460       // SGPR and TTMP registers must be aligned. Max required alignment is 4 dwords.
1461       Size = std::min(RegWidth, 4u);
1462     }
1463     if (RegNum % Size != 0)
1464       return false;
1465     if (DwordRegIndex) { *DwordRegIndex = RegNum; }
1466     RegNum = RegNum / Size;
1467     int RCID = getRegClass(RegKind, RegWidth);
1468     if (RCID == -1)
1469       return false;
1470     const MCRegisterClass RC = TRI->getRegClass(RCID);
1471     if (RegNum >= RC.getNumRegs())
1472       return false;
1473     Reg = RC.getRegister(RegNum);
1474     break;
1475   }
1476 
1477   default:
1478     llvm_unreachable("unexpected register kind");
1479   }
1480 
1481   if (!subtargetHasRegister(*TRI, Reg))
1482     return false;
1483   return true;
1484 }
1485 
1486 std::unique_ptr<AMDGPUOperand> AMDGPUAsmParser::parseRegister() {
1487   const auto &Tok = Parser.getTok();
1488   SMLoc StartLoc = Tok.getLoc();
1489   SMLoc EndLoc = Tok.getEndLoc();
1490   RegisterKind RegKind;
1491   unsigned Reg, RegNum, RegWidth, DwordRegIndex;
1492 
1493   if (!ParseAMDGPURegister(RegKind, Reg, RegNum, RegWidth, &DwordRegIndex)) {
1494     return nullptr;
1495   }
1496   KernelScope.usesRegister(RegKind, DwordRegIndex, RegWidth);
1497   return AMDGPUOperand::CreateReg(this, Reg, StartLoc, EndLoc, false);
1498 }
1499 
1500 OperandMatchResultTy
1501 AMDGPUAsmParser::parseImm(OperandVector &Operands) {
1502   // TODO: add syntactic sugar for 1/(2*PI)
1503   bool Minus = false;
1504   if (getLexer().getKind() == AsmToken::Minus) {
1505     Minus = true;
1506     Parser.Lex();
1507   }
1508 
1509   SMLoc S = Parser.getTok().getLoc();
1510   switch(getLexer().getKind()) {
1511   case AsmToken::Integer: {
1512     int64_t IntVal;
1513     if (getParser().parseAbsoluteExpression(IntVal))
1514       return MatchOperand_ParseFail;
1515     if (Minus)
1516       IntVal *= -1;
1517     Operands.push_back(AMDGPUOperand::CreateImm(this, IntVal, S));
1518     return MatchOperand_Success;
1519   }
1520   case AsmToken::Real: {
1521     int64_t IntVal;
1522     if (getParser().parseAbsoluteExpression(IntVal))
1523       return MatchOperand_ParseFail;
1524 
1525     APFloat F(BitsToDouble(IntVal));
1526     if (Minus)
1527       F.changeSign();
1528     Operands.push_back(
1529         AMDGPUOperand::CreateImm(this, F.bitcastToAPInt().getZExtValue(), S,
1530                                  AMDGPUOperand::ImmTyNone, true));
1531     return MatchOperand_Success;
1532   }
1533   default:
1534     return Minus ? MatchOperand_ParseFail : MatchOperand_NoMatch;
1535   }
1536 }
1537 
1538 OperandMatchResultTy
1539 AMDGPUAsmParser::parseReg(OperandVector &Operands) {
1540   if (auto R = parseRegister()) {
1541     assert(R->isReg());
1542     R->Reg.IsForcedVOP3 = isForcedVOP3();
1543     Operands.push_back(std::move(R));
1544     return MatchOperand_Success;
1545   }
1546   return MatchOperand_NoMatch;
1547 }
1548 
1549 OperandMatchResultTy
1550 AMDGPUAsmParser::parseRegOrImm(OperandVector &Operands) {
1551   auto res = parseImm(Operands);
1552   if (res != MatchOperand_NoMatch) {
1553     return res;
1554   }
1555 
1556   return parseReg(Operands);
1557 }
1558 
1559 OperandMatchResultTy
1560 AMDGPUAsmParser::parseRegOrImmWithFPInputMods(OperandVector &Operands,
1561                                               bool AllowImm) {
1562   // XXX: During parsing we can't determine if minus sign means
1563   // negate-modifier or negative immediate value.
1564   // By default we suppose it is modifier.
1565   bool Negate = false, Abs = false, Abs2 = false;
1566 
1567   if (getLexer().getKind()== AsmToken::Minus) {
1568     Parser.Lex();
1569     Negate = true;
1570   }
1571 
1572   if (getLexer().getKind() == AsmToken::Identifier &&
1573       Parser.getTok().getString() == "abs") {
1574     Parser.Lex();
1575     Abs2 = true;
1576     if (getLexer().isNot(AsmToken::LParen)) {
1577       Error(Parser.getTok().getLoc(), "expected left paren after abs");
1578       return MatchOperand_ParseFail;
1579     }
1580     Parser.Lex();
1581   }
1582 
1583   if (getLexer().getKind() == AsmToken::Pipe) {
1584     if (Abs2) {
1585       Error(Parser.getTok().getLoc(), "expected register or immediate");
1586       return MatchOperand_ParseFail;
1587     }
1588     Parser.Lex();
1589     Abs = true;
1590   }
1591 
1592   OperandMatchResultTy Res;
1593   if (AllowImm) {
1594     Res = parseRegOrImm(Operands);
1595   } else {
1596     Res = parseReg(Operands);
1597   }
1598   if (Res != MatchOperand_Success) {
1599     return Res;
1600   }
1601 
1602   AMDGPUOperand::Modifiers Mods;
1603   if (Negate) {
1604     Mods.Neg = true;
1605   }
1606   if (Abs) {
1607     if (getLexer().getKind() != AsmToken::Pipe) {
1608       Error(Parser.getTok().getLoc(), "expected vertical bar");
1609       return MatchOperand_ParseFail;
1610     }
1611     Parser.Lex();
1612     Mods.Abs = true;
1613   }
1614   if (Abs2) {
1615     if (getLexer().isNot(AsmToken::RParen)) {
1616       Error(Parser.getTok().getLoc(), "expected closing parentheses");
1617       return MatchOperand_ParseFail;
1618     }
1619     Parser.Lex();
1620     Mods.Abs = true;
1621   }
1622 
1623   if (Mods.hasFPModifiers()) {
1624     AMDGPUOperand &Op = static_cast<AMDGPUOperand &>(*Operands.back());
1625     Op.setModifiers(Mods);
1626   }
1627   return MatchOperand_Success;
1628 }
1629 
1630 OperandMatchResultTy
1631 AMDGPUAsmParser::parseRegOrImmWithIntInputMods(OperandVector &Operands,
1632                                                bool AllowImm) {
1633   bool Sext = false;
1634 
1635   if (getLexer().getKind() == AsmToken::Identifier &&
1636       Parser.getTok().getString() == "sext") {
1637     Parser.Lex();
1638     Sext = true;
1639     if (getLexer().isNot(AsmToken::LParen)) {
1640       Error(Parser.getTok().getLoc(), "expected left paren after sext");
1641       return MatchOperand_ParseFail;
1642     }
1643     Parser.Lex();
1644   }
1645 
1646   OperandMatchResultTy Res;
1647   if (AllowImm) {
1648     Res = parseRegOrImm(Operands);
1649   } else {
1650     Res = parseReg(Operands);
1651   }
1652   if (Res != MatchOperand_Success) {
1653     return Res;
1654   }
1655 
1656   AMDGPUOperand::Modifiers Mods;
1657   if (Sext) {
1658     if (getLexer().isNot(AsmToken::RParen)) {
1659       Error(Parser.getTok().getLoc(), "expected closing parentheses");
1660       return MatchOperand_ParseFail;
1661     }
1662     Parser.Lex();
1663     Mods.Sext = true;
1664   }
1665 
1666   if (Mods.hasIntModifiers()) {
1667     AMDGPUOperand &Op = static_cast<AMDGPUOperand &>(*Operands.back());
1668     Op.setModifiers(Mods);
1669   }
1670 
1671   return MatchOperand_Success;
1672 }
1673 
1674 OperandMatchResultTy
1675 AMDGPUAsmParser::parseRegWithFPInputMods(OperandVector &Operands) {
1676   return parseRegOrImmWithFPInputMods(Operands, false);
1677 }
1678 
1679 OperandMatchResultTy
1680 AMDGPUAsmParser::parseRegWithIntInputMods(OperandVector &Operands) {
1681   return parseRegOrImmWithIntInputMods(Operands, false);
1682 }
1683 
1684 OperandMatchResultTy AMDGPUAsmParser::parseVReg32OrOff(OperandVector &Operands) {
1685   std::unique_ptr<AMDGPUOperand> Reg = parseRegister();
1686   if (Reg) {
1687     Operands.push_back(std::move(Reg));
1688     return MatchOperand_Success;
1689   }
1690 
1691   const AsmToken &Tok = Parser.getTok();
1692   if (Tok.getString() == "off") {
1693     Operands.push_back(AMDGPUOperand::CreateImm(this, 0, Tok.getLoc(),
1694                                                 AMDGPUOperand::ImmTyOff, false));
1695     Parser.Lex();
1696     return MatchOperand_Success;
1697   }
1698 
1699   return MatchOperand_NoMatch;
1700 }
1701 
1702 unsigned AMDGPUAsmParser::checkTargetMatchPredicate(MCInst &Inst) {
1703   uint64_t TSFlags = MII.get(Inst.getOpcode()).TSFlags;
1704 
1705   if ((getForcedEncodingSize() == 32 && (TSFlags & SIInstrFlags::VOP3)) ||
1706       (getForcedEncodingSize() == 64 && !(TSFlags & SIInstrFlags::VOP3)) ||
1707       (isForcedDPP() && !(TSFlags & SIInstrFlags::DPP)) ||
1708       (isForcedSDWA() && !(TSFlags & SIInstrFlags::SDWA)) )
1709     return Match_InvalidOperand;
1710 
1711   if ((TSFlags & SIInstrFlags::VOP3) &&
1712       (TSFlags & SIInstrFlags::VOPAsmPrefer32Bit) &&
1713       getForcedEncodingSize() != 64)
1714     return Match_PreferE32;
1715 
1716   if (Inst.getOpcode() == AMDGPU::V_MAC_F32_sdwa_vi ||
1717       Inst.getOpcode() == AMDGPU::V_MAC_F16_sdwa_vi) {
1718     // v_mac_f32/16 allow only dst_sel == DWORD;
1719     auto OpNum =
1720         AMDGPU::getNamedOperandIdx(Inst.getOpcode(), AMDGPU::OpName::dst_sel);
1721     const auto &Op = Inst.getOperand(OpNum);
1722     if (!Op.isImm() || Op.getImm() != AMDGPU::SDWA::SdwaSel::DWORD) {
1723       return Match_InvalidOperand;
1724     }
1725   }
1726 
1727   return Match_Success;
1728 }
1729 
1730 // What asm variants we should check
1731 ArrayRef<unsigned> AMDGPUAsmParser::getMatchedVariants() const {
1732   if (getForcedEncodingSize() == 32) {
1733     static const unsigned Variants[] = {AMDGPUAsmVariants::DEFAULT};
1734     return makeArrayRef(Variants);
1735   }
1736 
1737   if (isForcedVOP3()) {
1738     static const unsigned Variants[] = {AMDGPUAsmVariants::VOP3};
1739     return makeArrayRef(Variants);
1740   }
1741 
1742   if (isForcedSDWA()) {
1743     static const unsigned Variants[] = {AMDGPUAsmVariants::SDWA};
1744     return makeArrayRef(Variants);
1745   }
1746 
1747   if (isForcedDPP()) {
1748     static const unsigned Variants[] = {AMDGPUAsmVariants::DPP};
1749     return makeArrayRef(Variants);
1750   }
1751 
1752   static const unsigned Variants[] = {
1753     AMDGPUAsmVariants::DEFAULT, AMDGPUAsmVariants::VOP3,
1754     AMDGPUAsmVariants::SDWA, AMDGPUAsmVariants::DPP
1755   };
1756 
1757   return makeArrayRef(Variants);
1758 }
1759 
1760 bool AMDGPUAsmParser::MatchAndEmitInstruction(SMLoc IDLoc, unsigned &Opcode,
1761                                               OperandVector &Operands,
1762                                               MCStreamer &Out,
1763                                               uint64_t &ErrorInfo,
1764                                               bool MatchingInlineAsm) {
1765   MCInst Inst;
1766   unsigned Result = Match_Success;
1767   for (auto Variant : getMatchedVariants()) {
1768     uint64_t EI;
1769     auto R = MatchInstructionImpl(Operands, Inst, EI, MatchingInlineAsm,
1770                                   Variant);
1771     // We order match statuses from least to most specific. We use most specific
1772     // status as resulting
1773     // Match_MnemonicFail < Match_InvalidOperand < Match_MissingFeature < Match_PreferE32
1774     if ((R == Match_Success) ||
1775         (R == Match_PreferE32) ||
1776         (R == Match_MissingFeature && Result != Match_PreferE32) ||
1777         (R == Match_InvalidOperand && Result != Match_MissingFeature
1778                                    && Result != Match_PreferE32) ||
1779         (R == Match_MnemonicFail   && Result != Match_InvalidOperand
1780                                    && Result != Match_MissingFeature
1781                                    && Result != Match_PreferE32)) {
1782       Result = R;
1783       ErrorInfo = EI;
1784     }
1785     if (R == Match_Success)
1786       break;
1787   }
1788 
1789   switch (Result) {
1790   default: break;
1791   case Match_Success:
1792     Inst.setLoc(IDLoc);
1793     Out.EmitInstruction(Inst, getSTI());
1794     return false;
1795 
1796   case Match_MissingFeature:
1797     return Error(IDLoc, "instruction not supported on this GPU");
1798 
1799   case Match_MnemonicFail:
1800     return Error(IDLoc, "unrecognized instruction mnemonic");
1801 
1802   case Match_InvalidOperand: {
1803     SMLoc ErrorLoc = IDLoc;
1804     if (ErrorInfo != ~0ULL) {
1805       if (ErrorInfo >= Operands.size()) {
1806         return Error(IDLoc, "too few operands for instruction");
1807       }
1808       ErrorLoc = ((AMDGPUOperand &)*Operands[ErrorInfo]).getStartLoc();
1809       if (ErrorLoc == SMLoc())
1810         ErrorLoc = IDLoc;
1811     }
1812     return Error(ErrorLoc, "invalid operand for instruction");
1813   }
1814 
1815   case Match_PreferE32:
1816     return Error(IDLoc, "internal error: instruction without _e64 suffix "
1817                         "should be encoded as e32");
1818   }
1819   llvm_unreachable("Implement any new match types added!");
1820 }
1821 
1822 bool AMDGPUAsmParser::ParseAsAbsoluteExpression(uint32_t &Ret) {
1823   int64_t Tmp = -1;
1824   if (getLexer().isNot(AsmToken::Integer) && getLexer().isNot(AsmToken::Identifier)) {
1825     return true;
1826   }
1827   if (getParser().parseAbsoluteExpression(Tmp)) {
1828     return true;
1829   }
1830   Ret = static_cast<uint32_t>(Tmp);
1831   return false;
1832 }
1833 
1834 bool AMDGPUAsmParser::ParseDirectiveMajorMinor(uint32_t &Major,
1835                                                uint32_t &Minor) {
1836   if (ParseAsAbsoluteExpression(Major))
1837     return TokError("invalid major version");
1838 
1839   if (getLexer().isNot(AsmToken::Comma))
1840     return TokError("minor version number required, comma expected");
1841   Lex();
1842 
1843   if (ParseAsAbsoluteExpression(Minor))
1844     return TokError("invalid minor version");
1845 
1846   return false;
1847 }
1848 
1849 bool AMDGPUAsmParser::ParseDirectiveHSACodeObjectVersion() {
1850   uint32_t Major;
1851   uint32_t Minor;
1852 
1853   if (ParseDirectiveMajorMinor(Major, Minor))
1854     return true;
1855 
1856   getTargetStreamer().EmitDirectiveHSACodeObjectVersion(Major, Minor);
1857   return false;
1858 }
1859 
1860 bool AMDGPUAsmParser::ParseDirectiveHSACodeObjectISA() {
1861   uint32_t Major;
1862   uint32_t Minor;
1863   uint32_t Stepping;
1864   StringRef VendorName;
1865   StringRef ArchName;
1866 
1867   // If this directive has no arguments, then use the ISA version for the
1868   // targeted GPU.
1869   if (getLexer().is(AsmToken::EndOfStatement)) {
1870     AMDGPU::IsaVersion Isa = AMDGPU::getIsaVersion(getSTI().getFeatureBits());
1871     getTargetStreamer().EmitDirectiveHSACodeObjectISA(Isa.Major, Isa.Minor,
1872                                                       Isa.Stepping,
1873                                                       "AMD", "AMDGPU");
1874     return false;
1875   }
1876 
1877   if (ParseDirectiveMajorMinor(Major, Minor))
1878     return true;
1879 
1880   if (getLexer().isNot(AsmToken::Comma))
1881     return TokError("stepping version number required, comma expected");
1882   Lex();
1883 
1884   if (ParseAsAbsoluteExpression(Stepping))
1885     return TokError("invalid stepping version");
1886 
1887   if (getLexer().isNot(AsmToken::Comma))
1888     return TokError("vendor name required, comma expected");
1889   Lex();
1890 
1891   if (getLexer().isNot(AsmToken::String))
1892     return TokError("invalid vendor name");
1893 
1894   VendorName = getLexer().getTok().getStringContents();
1895   Lex();
1896 
1897   if (getLexer().isNot(AsmToken::Comma))
1898     return TokError("arch name required, comma expected");
1899   Lex();
1900 
1901   if (getLexer().isNot(AsmToken::String))
1902     return TokError("invalid arch name");
1903 
1904   ArchName = getLexer().getTok().getStringContents();
1905   Lex();
1906 
1907   getTargetStreamer().EmitDirectiveHSACodeObjectISA(Major, Minor, Stepping,
1908                                                     VendorName, ArchName);
1909   return false;
1910 }
1911 
1912 bool AMDGPUAsmParser::ParseDirectiveRuntimeMetadata() {
1913   std::string Metadata;
1914   raw_string_ostream MS(Metadata);
1915 
1916   getLexer().setSkipSpace(false);
1917 
1918   bool FoundEnd = false;
1919   while (!getLexer().is(AsmToken::Eof)) {
1920     while (getLexer().is(AsmToken::Space)) {
1921       MS << ' ';
1922       Lex();
1923     }
1924 
1925     if (getLexer().is(AsmToken::Identifier)) {
1926       StringRef ID = getLexer().getTok().getIdentifier();
1927       if (ID == ".end_amdgpu_runtime_metadata") {
1928         Lex();
1929         FoundEnd = true;
1930         break;
1931       }
1932     }
1933 
1934     MS << Parser.parseStringToEndOfStatement()
1935        << getContext().getAsmInfo()->getSeparatorString();
1936 
1937     Parser.eatToEndOfStatement();
1938   }
1939 
1940   getLexer().setSkipSpace(true);
1941 
1942   if (getLexer().is(AsmToken::Eof) && !FoundEnd)
1943     return TokError("expected directive .end_amdgpu_runtime_metadata not found");
1944 
1945   MS.flush();
1946 
1947   getTargetStreamer().EmitRuntimeMetadata(Metadata);
1948 
1949   return false;
1950 }
1951 
1952 bool AMDGPUAsmParser::ParseAMDKernelCodeTValue(StringRef ID,
1953                                                amd_kernel_code_t &Header) {
1954   SmallString<40> ErrStr;
1955   raw_svector_ostream Err(ErrStr);
1956   if (!parseAmdKernelCodeField(ID, getParser(), Header, Err)) {
1957     return TokError(Err.str());
1958   }
1959   Lex();
1960   return false;
1961 }
1962 
1963 bool AMDGPUAsmParser::ParseDirectiveAMDKernelCodeT() {
1964   amd_kernel_code_t Header;
1965   AMDGPU::initDefaultAMDKernelCodeT(Header, getSTI().getFeatureBits());
1966 
1967   while (true) {
1968     // Lex EndOfStatement.  This is in a while loop, because lexing a comment
1969     // will set the current token to EndOfStatement.
1970     while(getLexer().is(AsmToken::EndOfStatement))
1971       Lex();
1972 
1973     if (getLexer().isNot(AsmToken::Identifier))
1974       return TokError("expected value identifier or .end_amd_kernel_code_t");
1975 
1976     StringRef ID = getLexer().getTok().getIdentifier();
1977     Lex();
1978 
1979     if (ID == ".end_amd_kernel_code_t")
1980       break;
1981 
1982     if (ParseAMDKernelCodeTValue(ID, Header))
1983       return true;
1984   }
1985 
1986   getTargetStreamer().EmitAMDKernelCodeT(Header);
1987 
1988   return false;
1989 }
1990 
1991 bool AMDGPUAsmParser::ParseSectionDirectiveHSAText() {
1992   getParser().getStreamer().SwitchSection(
1993       AMDGPU::getHSATextSection(getContext()));
1994   return false;
1995 }
1996 
1997 bool AMDGPUAsmParser::ParseDirectiveAMDGPUHsaKernel() {
1998   if (getLexer().isNot(AsmToken::Identifier))
1999     return TokError("expected symbol name");
2000 
2001   StringRef KernelName = Parser.getTok().getString();
2002 
2003   getTargetStreamer().EmitAMDGPUSymbolType(KernelName,
2004                                            ELF::STT_AMDGPU_HSA_KERNEL);
2005   Lex();
2006   KernelScope.initialize(getContext());
2007   return false;
2008 }
2009 
2010 bool AMDGPUAsmParser::ParseDirectiveAMDGPUHsaModuleGlobal() {
2011   if (getLexer().isNot(AsmToken::Identifier))
2012     return TokError("expected symbol name");
2013 
2014   StringRef GlobalName = Parser.getTok().getIdentifier();
2015 
2016   getTargetStreamer().EmitAMDGPUHsaModuleScopeGlobal(GlobalName);
2017   Lex();
2018   return false;
2019 }
2020 
2021 bool AMDGPUAsmParser::ParseDirectiveAMDGPUHsaProgramGlobal() {
2022   if (getLexer().isNot(AsmToken::Identifier))
2023     return TokError("expected symbol name");
2024 
2025   StringRef GlobalName = Parser.getTok().getIdentifier();
2026 
2027   getTargetStreamer().EmitAMDGPUHsaProgramScopeGlobal(GlobalName);
2028   Lex();
2029   return false;
2030 }
2031 
2032 bool AMDGPUAsmParser::ParseSectionDirectiveHSADataGlobalAgent() {
2033   getParser().getStreamer().SwitchSection(
2034       AMDGPU::getHSADataGlobalAgentSection(getContext()));
2035   return false;
2036 }
2037 
2038 bool AMDGPUAsmParser::ParseSectionDirectiveHSADataGlobalProgram() {
2039   getParser().getStreamer().SwitchSection(
2040       AMDGPU::getHSADataGlobalProgramSection(getContext()));
2041   return false;
2042 }
2043 
2044 bool AMDGPUAsmParser::ParseSectionDirectiveHSARodataReadonlyAgent() {
2045   getParser().getStreamer().SwitchSection(
2046       AMDGPU::getHSARodataReadonlyAgentSection(getContext()));
2047   return false;
2048 }
2049 
2050 bool AMDGPUAsmParser::ParseDirective(AsmToken DirectiveID) {
2051   StringRef IDVal = DirectiveID.getString();
2052 
2053   if (IDVal == ".hsa_code_object_version")
2054     return ParseDirectiveHSACodeObjectVersion();
2055 
2056   if (IDVal == ".hsa_code_object_isa")
2057     return ParseDirectiveHSACodeObjectISA();
2058 
2059   if (IDVal == ".amdgpu_runtime_metadata")
2060     return ParseDirectiveRuntimeMetadata();
2061 
2062   if (IDVal == ".amd_kernel_code_t")
2063     return ParseDirectiveAMDKernelCodeT();
2064 
2065   if (IDVal == ".hsatext")
2066     return ParseSectionDirectiveHSAText();
2067 
2068   if (IDVal == ".amdgpu_hsa_kernel")
2069     return ParseDirectiveAMDGPUHsaKernel();
2070 
2071   if (IDVal == ".amdgpu_hsa_module_global")
2072     return ParseDirectiveAMDGPUHsaModuleGlobal();
2073 
2074   if (IDVal == ".amdgpu_hsa_program_global")
2075     return ParseDirectiveAMDGPUHsaProgramGlobal();
2076 
2077   if (IDVal == ".hsadata_global_agent")
2078     return ParseSectionDirectiveHSADataGlobalAgent();
2079 
2080   if (IDVal == ".hsadata_global_program")
2081     return ParseSectionDirectiveHSADataGlobalProgram();
2082 
2083   if (IDVal == ".hsarodata_readonly_agent")
2084     return ParseSectionDirectiveHSARodataReadonlyAgent();
2085 
2086   return true;
2087 }
2088 
2089 bool AMDGPUAsmParser::subtargetHasRegister(const MCRegisterInfo &MRI,
2090                                            unsigned RegNo) const {
2091   if (isCI())
2092     return true;
2093 
2094   if (isSI()) {
2095     // No flat_scr
2096     switch (RegNo) {
2097     case AMDGPU::FLAT_SCR:
2098     case AMDGPU::FLAT_SCR_LO:
2099     case AMDGPU::FLAT_SCR_HI:
2100       return false;
2101     default:
2102       return true;
2103     }
2104   }
2105 
2106   // VI only has 102 SGPRs, so make sure we aren't trying to use the 2 more that
2107   // SI/CI have.
2108   for (MCRegAliasIterator R(AMDGPU::SGPR102_SGPR103, &MRI, true);
2109        R.isValid(); ++R) {
2110     if (*R == RegNo)
2111       return false;
2112   }
2113 
2114   return true;
2115 }
2116 
2117 OperandMatchResultTy
2118 AMDGPUAsmParser::parseOperand(OperandVector &Operands, StringRef Mnemonic) {
2119   // Try to parse with a custom parser
2120   OperandMatchResultTy ResTy = MatchOperandParserImpl(Operands, Mnemonic);
2121 
2122   // If we successfully parsed the operand or if there as an error parsing,
2123   // we are done.
2124   //
2125   // If we are parsing after we reach EndOfStatement then this means we
2126   // are appending default values to the Operands list.  This is only done
2127   // by custom parser, so we shouldn't continue on to the generic parsing.
2128   if (ResTy == MatchOperand_Success || ResTy == MatchOperand_ParseFail ||
2129       getLexer().is(AsmToken::EndOfStatement))
2130     return ResTy;
2131 
2132   ResTy = parseRegOrImm(Operands);
2133 
2134   if (ResTy == MatchOperand_Success)
2135     return ResTy;
2136 
2137   if (getLexer().getKind() == AsmToken::Identifier) {
2138     // If this identifier is a symbol, we want to create an expression for it.
2139     // It is a little difficult to distinguish between a symbol name, and
2140     // an instruction flag like 'gds'.  In order to do this, we parse
2141     // all tokens as expressions and then treate the symbol name as the token
2142     // string when we want to interpret the operand as a token.
2143     const auto &Tok = Parser.getTok();
2144     SMLoc S = Tok.getLoc();
2145     const MCExpr *Expr = nullptr;
2146     if (!Parser.parseExpression(Expr)) {
2147       Operands.push_back(AMDGPUOperand::CreateExpr(this, Expr, S));
2148       return MatchOperand_Success;
2149     }
2150 
2151     Operands.push_back(AMDGPUOperand::CreateToken(this, Tok.getString(), Tok.getLoc()));
2152     Parser.Lex();
2153     return MatchOperand_Success;
2154   }
2155   return MatchOperand_NoMatch;
2156 }
2157 
2158 StringRef AMDGPUAsmParser::parseMnemonicSuffix(StringRef Name) {
2159   // Clear any forced encodings from the previous instruction.
2160   setForcedEncodingSize(0);
2161   setForcedDPP(false);
2162   setForcedSDWA(false);
2163 
2164   if (Name.endswith("_e64")) {
2165     setForcedEncodingSize(64);
2166     return Name.substr(0, Name.size() - 4);
2167   } else if (Name.endswith("_e32")) {
2168     setForcedEncodingSize(32);
2169     return Name.substr(0, Name.size() - 4);
2170   } else if (Name.endswith("_dpp")) {
2171     setForcedDPP(true);
2172     return Name.substr(0, Name.size() - 4);
2173   } else if (Name.endswith("_sdwa")) {
2174     setForcedSDWA(true);
2175     return Name.substr(0, Name.size() - 5);
2176   }
2177   return Name;
2178 }
2179 
2180 bool AMDGPUAsmParser::ParseInstruction(ParseInstructionInfo &Info,
2181                                        StringRef Name,
2182                                        SMLoc NameLoc, OperandVector &Operands) {
2183   // Add the instruction mnemonic
2184   Name = parseMnemonicSuffix(Name);
2185   Operands.push_back(AMDGPUOperand::CreateToken(this, Name, NameLoc));
2186 
2187   while (!getLexer().is(AsmToken::EndOfStatement)) {
2188     OperandMatchResultTy Res = parseOperand(Operands, Name);
2189 
2190     // Eat the comma or space if there is one.
2191     if (getLexer().is(AsmToken::Comma))
2192       Parser.Lex();
2193 
2194     switch (Res) {
2195       case MatchOperand_Success: break;
2196       case MatchOperand_ParseFail:
2197         Error(getLexer().getLoc(), "failed parsing operand.");
2198         while (!getLexer().is(AsmToken::EndOfStatement)) {
2199           Parser.Lex();
2200         }
2201         return true;
2202       case MatchOperand_NoMatch:
2203         Error(getLexer().getLoc(), "not a valid operand.");
2204         while (!getLexer().is(AsmToken::EndOfStatement)) {
2205           Parser.Lex();
2206         }
2207         return true;
2208     }
2209   }
2210 
2211   return false;
2212 }
2213 
2214 //===----------------------------------------------------------------------===//
2215 // Utility functions
2216 //===----------------------------------------------------------------------===//
2217 
2218 OperandMatchResultTy
2219 AMDGPUAsmParser::parseIntWithPrefix(const char *Prefix, int64_t &Int) {
2220   switch(getLexer().getKind()) {
2221     default: return MatchOperand_NoMatch;
2222     case AsmToken::Identifier: {
2223       StringRef Name = Parser.getTok().getString();
2224       if (!Name.equals(Prefix)) {
2225         return MatchOperand_NoMatch;
2226       }
2227 
2228       Parser.Lex();
2229       if (getLexer().isNot(AsmToken::Colon))
2230         return MatchOperand_ParseFail;
2231 
2232       Parser.Lex();
2233       if (getLexer().isNot(AsmToken::Integer))
2234         return MatchOperand_ParseFail;
2235 
2236       if (getParser().parseAbsoluteExpression(Int))
2237         return MatchOperand_ParseFail;
2238       break;
2239     }
2240   }
2241   return MatchOperand_Success;
2242 }
2243 
2244 OperandMatchResultTy
2245 AMDGPUAsmParser::parseIntWithPrefix(const char *Prefix, OperandVector &Operands,
2246                                     AMDGPUOperand::ImmTy ImmTy,
2247                                     bool (*ConvertResult)(int64_t&)) {
2248   SMLoc S = Parser.getTok().getLoc();
2249   int64_t Value = 0;
2250 
2251   OperandMatchResultTy Res = parseIntWithPrefix(Prefix, Value);
2252   if (Res != MatchOperand_Success)
2253     return Res;
2254 
2255   if (ConvertResult && !ConvertResult(Value)) {
2256     return MatchOperand_ParseFail;
2257   }
2258 
2259   Operands.push_back(AMDGPUOperand::CreateImm(this, Value, S, ImmTy));
2260   return MatchOperand_Success;
2261 }
2262 
2263 OperandMatchResultTy
2264 AMDGPUAsmParser::parseNamedBit(const char *Name, OperandVector &Operands,
2265                                AMDGPUOperand::ImmTy ImmTy) {
2266   int64_t Bit = 0;
2267   SMLoc S = Parser.getTok().getLoc();
2268 
2269   // We are at the end of the statement, and this is a default argument, so
2270   // use a default value.
2271   if (getLexer().isNot(AsmToken::EndOfStatement)) {
2272     switch(getLexer().getKind()) {
2273       case AsmToken::Identifier: {
2274         StringRef Tok = Parser.getTok().getString();
2275         if (Tok == Name) {
2276           Bit = 1;
2277           Parser.Lex();
2278         } else if (Tok.startswith("no") && Tok.endswith(Name)) {
2279           Bit = 0;
2280           Parser.Lex();
2281         } else {
2282           return MatchOperand_NoMatch;
2283         }
2284         break;
2285       }
2286       default:
2287         return MatchOperand_NoMatch;
2288     }
2289   }
2290 
2291   Operands.push_back(AMDGPUOperand::CreateImm(this, Bit, S, ImmTy));
2292   return MatchOperand_Success;
2293 }
2294 
2295 typedef std::map<enum AMDGPUOperand::ImmTy, unsigned> OptionalImmIndexMap;
2296 
2297 void addOptionalImmOperand(MCInst& Inst, const OperandVector& Operands,
2298                            OptionalImmIndexMap& OptionalIdx,
2299                            AMDGPUOperand::ImmTy ImmT, int64_t Default = 0) {
2300   auto i = OptionalIdx.find(ImmT);
2301   if (i != OptionalIdx.end()) {
2302     unsigned Idx = i->second;
2303     ((AMDGPUOperand &)*Operands[Idx]).addImmOperands(Inst, 1);
2304   } else {
2305     Inst.addOperand(MCOperand::createImm(Default));
2306   }
2307 }
2308 
2309 OperandMatchResultTy
2310 AMDGPUAsmParser::parseStringWithPrefix(StringRef Prefix, StringRef &Value) {
2311   if (getLexer().isNot(AsmToken::Identifier)) {
2312     return MatchOperand_NoMatch;
2313   }
2314   StringRef Tok = Parser.getTok().getString();
2315   if (Tok != Prefix) {
2316     return MatchOperand_NoMatch;
2317   }
2318 
2319   Parser.Lex();
2320   if (getLexer().isNot(AsmToken::Colon)) {
2321     return MatchOperand_ParseFail;
2322   }
2323 
2324   Parser.Lex();
2325   if (getLexer().isNot(AsmToken::Identifier)) {
2326     return MatchOperand_ParseFail;
2327   }
2328 
2329   Value = Parser.getTok().getString();
2330   return MatchOperand_Success;
2331 }
2332 
2333 //===----------------------------------------------------------------------===//
2334 // ds
2335 //===----------------------------------------------------------------------===//
2336 
2337 void AMDGPUAsmParser::cvtDSOffset01(MCInst &Inst,
2338                                     const OperandVector &Operands) {
2339   OptionalImmIndexMap OptionalIdx;
2340 
2341   for (unsigned i = 1, e = Operands.size(); i != e; ++i) {
2342     AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[i]);
2343 
2344     // Add the register arguments
2345     if (Op.isReg()) {
2346       Op.addRegOperands(Inst, 1);
2347       continue;
2348     }
2349 
2350     // Handle optional arguments
2351     OptionalIdx[Op.getImmTy()] = i;
2352   }
2353 
2354   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyOffset0);
2355   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyOffset1);
2356   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyGDS);
2357 
2358   Inst.addOperand(MCOperand::createReg(AMDGPU::M0)); // m0
2359 }
2360 
2361 void AMDGPUAsmParser::cvtDSImpl(MCInst &Inst, const OperandVector &Operands,
2362                                 bool IsGdsHardcoded) {
2363   OptionalImmIndexMap OptionalIdx;
2364 
2365   for (unsigned i = 1, e = Operands.size(); i != e; ++i) {
2366     AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[i]);
2367 
2368     // Add the register arguments
2369     if (Op.isReg()) {
2370       Op.addRegOperands(Inst, 1);
2371       continue;
2372     }
2373 
2374     if (Op.isToken() && Op.getToken() == "gds") {
2375       IsGdsHardcoded = true;
2376       continue;
2377     }
2378 
2379     // Handle optional arguments
2380     OptionalIdx[Op.getImmTy()] = i;
2381   }
2382 
2383   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyOffset);
2384   if (!IsGdsHardcoded) {
2385     addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyGDS);
2386   }
2387   Inst.addOperand(MCOperand::createReg(AMDGPU::M0)); // m0
2388 }
2389 
2390 void AMDGPUAsmParser::cvtExp(MCInst &Inst, const OperandVector &Operands) {
2391   OptionalImmIndexMap OptionalIdx;
2392 
2393   unsigned EnMask = 0;
2394   int SrcIdx = 0;
2395 
2396   for (unsigned i = 1, e = Operands.size(); i != e; ++i) {
2397     AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[i]);
2398 
2399     // Add the register arguments
2400     if (Op.isReg()) {
2401       EnMask |= (1 << SrcIdx);
2402       Op.addRegOperands(Inst, 1);
2403       ++SrcIdx;
2404       continue;
2405     }
2406 
2407     if (Op.isOff()) {
2408       ++SrcIdx;
2409       Inst.addOperand(MCOperand::createReg(AMDGPU::NoRegister));
2410       continue;
2411     }
2412 
2413     if (Op.isImm() && Op.getImmTy() == AMDGPUOperand::ImmTyExpTgt) {
2414       Op.addImmOperands(Inst, 1);
2415       continue;
2416     }
2417 
2418     if (Op.isToken() && Op.getToken() == "done")
2419       continue;
2420 
2421     // Handle optional arguments
2422     OptionalIdx[Op.getImmTy()] = i;
2423   }
2424 
2425   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyExpVM);
2426   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyExpCompr);
2427 
2428   Inst.addOperand(MCOperand::createImm(EnMask));
2429 }
2430 
2431 //===----------------------------------------------------------------------===//
2432 // s_waitcnt
2433 //===----------------------------------------------------------------------===//
2434 
2435 bool AMDGPUAsmParser::parseCnt(int64_t &IntVal) {
2436   StringRef CntName = Parser.getTok().getString();
2437   int64_t CntVal;
2438 
2439   Parser.Lex();
2440   if (getLexer().isNot(AsmToken::LParen))
2441     return true;
2442 
2443   Parser.Lex();
2444   if (getLexer().isNot(AsmToken::Integer))
2445     return true;
2446 
2447   if (getParser().parseAbsoluteExpression(CntVal))
2448     return true;
2449 
2450   if (getLexer().isNot(AsmToken::RParen))
2451     return true;
2452 
2453   Parser.Lex();
2454   if (getLexer().is(AsmToken::Amp) || getLexer().is(AsmToken::Comma))
2455     Parser.Lex();
2456 
2457   IsaVersion IV = getIsaVersion(getSTI().getFeatureBits());
2458   if (CntName == "vmcnt")
2459     IntVal = encodeVmcnt(IV, IntVal, CntVal);
2460   else if (CntName == "expcnt")
2461     IntVal = encodeExpcnt(IV, IntVal, CntVal);
2462   else if (CntName == "lgkmcnt")
2463     IntVal = encodeLgkmcnt(IV, IntVal, CntVal);
2464   else
2465     return true;
2466 
2467   return false;
2468 }
2469 
2470 OperandMatchResultTy
2471 AMDGPUAsmParser::parseSWaitCntOps(OperandVector &Operands) {
2472   IsaVersion IV = getIsaVersion(getSTI().getFeatureBits());
2473   int64_t Waitcnt = getWaitcntBitMask(IV);
2474   SMLoc S = Parser.getTok().getLoc();
2475 
2476   switch(getLexer().getKind()) {
2477     default: return MatchOperand_ParseFail;
2478     case AsmToken::Integer:
2479       // The operand can be an integer value.
2480       if (getParser().parseAbsoluteExpression(Waitcnt))
2481         return MatchOperand_ParseFail;
2482       break;
2483 
2484     case AsmToken::Identifier:
2485       do {
2486         if (parseCnt(Waitcnt))
2487           return MatchOperand_ParseFail;
2488       } while(getLexer().isNot(AsmToken::EndOfStatement));
2489       break;
2490   }
2491   Operands.push_back(AMDGPUOperand::CreateImm(this, Waitcnt, S));
2492   return MatchOperand_Success;
2493 }
2494 
2495 bool AMDGPUAsmParser::parseHwregConstruct(OperandInfoTy &HwReg, int64_t &Offset,
2496                                           int64_t &Width) {
2497   using namespace llvm::AMDGPU::Hwreg;
2498 
2499   if (Parser.getTok().getString() != "hwreg")
2500     return true;
2501   Parser.Lex();
2502 
2503   if (getLexer().isNot(AsmToken::LParen))
2504     return true;
2505   Parser.Lex();
2506 
2507   if (getLexer().is(AsmToken::Identifier)) {
2508     HwReg.IsSymbolic = true;
2509     HwReg.Id = ID_UNKNOWN_;
2510     const StringRef tok = Parser.getTok().getString();
2511     for (int i = ID_SYMBOLIC_FIRST_; i < ID_SYMBOLIC_LAST_; ++i) {
2512       if (tok == IdSymbolic[i]) {
2513         HwReg.Id = i;
2514         break;
2515       }
2516     }
2517     Parser.Lex();
2518   } else {
2519     HwReg.IsSymbolic = false;
2520     if (getLexer().isNot(AsmToken::Integer))
2521       return true;
2522     if (getParser().parseAbsoluteExpression(HwReg.Id))
2523       return true;
2524   }
2525 
2526   if (getLexer().is(AsmToken::RParen)) {
2527     Parser.Lex();
2528     return false;
2529   }
2530 
2531   // optional params
2532   if (getLexer().isNot(AsmToken::Comma))
2533     return true;
2534   Parser.Lex();
2535 
2536   if (getLexer().isNot(AsmToken::Integer))
2537     return true;
2538   if (getParser().parseAbsoluteExpression(Offset))
2539     return true;
2540 
2541   if (getLexer().isNot(AsmToken::Comma))
2542     return true;
2543   Parser.Lex();
2544 
2545   if (getLexer().isNot(AsmToken::Integer))
2546     return true;
2547   if (getParser().parseAbsoluteExpression(Width))
2548     return true;
2549 
2550   if (getLexer().isNot(AsmToken::RParen))
2551     return true;
2552   Parser.Lex();
2553 
2554   return false;
2555 }
2556 
2557 OperandMatchResultTy AMDGPUAsmParser::parseHwreg(OperandVector &Operands) {
2558   using namespace llvm::AMDGPU::Hwreg;
2559 
2560   int64_t Imm16Val = 0;
2561   SMLoc S = Parser.getTok().getLoc();
2562 
2563   switch(getLexer().getKind()) {
2564     default: return MatchOperand_NoMatch;
2565     case AsmToken::Integer:
2566       // The operand can be an integer value.
2567       if (getParser().parseAbsoluteExpression(Imm16Val))
2568         return MatchOperand_NoMatch;
2569       if (Imm16Val < 0 || !isUInt<16>(Imm16Val)) {
2570         Error(S, "invalid immediate: only 16-bit values are legal");
2571         // Do not return error code, but create an imm operand anyway and proceed
2572         // to the next operand, if any. That avoids unneccessary error messages.
2573       }
2574       break;
2575 
2576     case AsmToken::Identifier: {
2577         OperandInfoTy HwReg(ID_UNKNOWN_);
2578         int64_t Offset = OFFSET_DEFAULT_;
2579         int64_t Width = WIDTH_M1_DEFAULT_ + 1;
2580         if (parseHwregConstruct(HwReg, Offset, Width))
2581           return MatchOperand_ParseFail;
2582         if (HwReg.Id < 0 || !isUInt<ID_WIDTH_>(HwReg.Id)) {
2583           if (HwReg.IsSymbolic)
2584             Error(S, "invalid symbolic name of hardware register");
2585           else
2586             Error(S, "invalid code of hardware register: only 6-bit values are legal");
2587         }
2588         if (Offset < 0 || !isUInt<OFFSET_WIDTH_>(Offset))
2589           Error(S, "invalid bit offset: only 5-bit values are legal");
2590         if ((Width-1) < 0 || !isUInt<WIDTH_M1_WIDTH_>(Width-1))
2591           Error(S, "invalid bitfield width: only values from 1 to 32 are legal");
2592         Imm16Val = (HwReg.Id << ID_SHIFT_) | (Offset << OFFSET_SHIFT_) | ((Width-1) << WIDTH_M1_SHIFT_);
2593       }
2594       break;
2595   }
2596   Operands.push_back(AMDGPUOperand::CreateImm(this, Imm16Val, S, AMDGPUOperand::ImmTyHwreg));
2597   return MatchOperand_Success;
2598 }
2599 
2600 bool AMDGPUOperand::isSWaitCnt() const {
2601   return isImm();
2602 }
2603 
2604 bool AMDGPUOperand::isHwreg() const {
2605   return isImmTy(ImmTyHwreg);
2606 }
2607 
2608 bool AMDGPUAsmParser::parseSendMsgConstruct(OperandInfoTy &Msg, OperandInfoTy &Operation, int64_t &StreamId) {
2609   using namespace llvm::AMDGPU::SendMsg;
2610 
2611   if (Parser.getTok().getString() != "sendmsg")
2612     return true;
2613   Parser.Lex();
2614 
2615   if (getLexer().isNot(AsmToken::LParen))
2616     return true;
2617   Parser.Lex();
2618 
2619   if (getLexer().is(AsmToken::Identifier)) {
2620     Msg.IsSymbolic = true;
2621     Msg.Id = ID_UNKNOWN_;
2622     const std::string tok = Parser.getTok().getString();
2623     for (int i = ID_GAPS_FIRST_; i < ID_GAPS_LAST_; ++i) {
2624       switch(i) {
2625         default: continue; // Omit gaps.
2626         case ID_INTERRUPT: case ID_GS: case ID_GS_DONE:  case ID_SYSMSG: break;
2627       }
2628       if (tok == IdSymbolic[i]) {
2629         Msg.Id = i;
2630         break;
2631       }
2632     }
2633     Parser.Lex();
2634   } else {
2635     Msg.IsSymbolic = false;
2636     if (getLexer().isNot(AsmToken::Integer))
2637       return true;
2638     if (getParser().parseAbsoluteExpression(Msg.Id))
2639       return true;
2640     if (getLexer().is(AsmToken::Integer))
2641       if (getParser().parseAbsoluteExpression(Msg.Id))
2642         Msg.Id = ID_UNKNOWN_;
2643   }
2644   if (Msg.Id == ID_UNKNOWN_) // Don't know how to parse the rest.
2645     return false;
2646 
2647   if (!(Msg.Id == ID_GS || Msg.Id == ID_GS_DONE || Msg.Id == ID_SYSMSG)) {
2648     if (getLexer().isNot(AsmToken::RParen))
2649       return true;
2650     Parser.Lex();
2651     return false;
2652   }
2653 
2654   if (getLexer().isNot(AsmToken::Comma))
2655     return true;
2656   Parser.Lex();
2657 
2658   assert(Msg.Id == ID_GS || Msg.Id == ID_GS_DONE || Msg.Id == ID_SYSMSG);
2659   Operation.Id = ID_UNKNOWN_;
2660   if (getLexer().is(AsmToken::Identifier)) {
2661     Operation.IsSymbolic = true;
2662     const char* const *S = (Msg.Id == ID_SYSMSG) ? OpSysSymbolic : OpGsSymbolic;
2663     const int F = (Msg.Id == ID_SYSMSG) ? OP_SYS_FIRST_ : OP_GS_FIRST_;
2664     const int L = (Msg.Id == ID_SYSMSG) ? OP_SYS_LAST_ : OP_GS_LAST_;
2665     const StringRef Tok = Parser.getTok().getString();
2666     for (int i = F; i < L; ++i) {
2667       if (Tok == S[i]) {
2668         Operation.Id = i;
2669         break;
2670       }
2671     }
2672     Parser.Lex();
2673   } else {
2674     Operation.IsSymbolic = false;
2675     if (getLexer().isNot(AsmToken::Integer))
2676       return true;
2677     if (getParser().parseAbsoluteExpression(Operation.Id))
2678       return true;
2679   }
2680 
2681   if ((Msg.Id == ID_GS || Msg.Id == ID_GS_DONE) && Operation.Id != OP_GS_NOP) {
2682     // Stream id is optional.
2683     if (getLexer().is(AsmToken::RParen)) {
2684       Parser.Lex();
2685       return false;
2686     }
2687 
2688     if (getLexer().isNot(AsmToken::Comma))
2689       return true;
2690     Parser.Lex();
2691 
2692     if (getLexer().isNot(AsmToken::Integer))
2693       return true;
2694     if (getParser().parseAbsoluteExpression(StreamId))
2695       return true;
2696   }
2697 
2698   if (getLexer().isNot(AsmToken::RParen))
2699     return true;
2700   Parser.Lex();
2701   return false;
2702 }
2703 
2704 OperandMatchResultTy AMDGPUAsmParser::parseInterpSlot(OperandVector &Operands) {
2705   if (getLexer().getKind() != AsmToken::Identifier)
2706     return MatchOperand_NoMatch;
2707 
2708   StringRef Str = Parser.getTok().getString();
2709   int Slot = StringSwitch<int>(Str)
2710     .Case("p10", 0)
2711     .Case("p20", 1)
2712     .Case("p0", 2)
2713     .Default(-1);
2714 
2715   SMLoc S = Parser.getTok().getLoc();
2716   if (Slot == -1)
2717     return MatchOperand_ParseFail;
2718 
2719   Parser.Lex();
2720   Operands.push_back(AMDGPUOperand::CreateImm(this, Slot, S,
2721                                               AMDGPUOperand::ImmTyInterpSlot));
2722   return MatchOperand_Success;
2723 }
2724 
2725 OperandMatchResultTy AMDGPUAsmParser::parseInterpAttr(OperandVector &Operands) {
2726   if (getLexer().getKind() != AsmToken::Identifier)
2727     return MatchOperand_NoMatch;
2728 
2729   StringRef Str = Parser.getTok().getString();
2730   if (!Str.startswith("attr"))
2731     return MatchOperand_NoMatch;
2732 
2733   StringRef Chan = Str.take_back(2);
2734   int AttrChan = StringSwitch<int>(Chan)
2735     .Case(".x", 0)
2736     .Case(".y", 1)
2737     .Case(".z", 2)
2738     .Case(".w", 3)
2739     .Default(-1);
2740   if (AttrChan == -1)
2741     return MatchOperand_ParseFail;
2742 
2743   Str = Str.drop_back(2).drop_front(4);
2744 
2745   uint8_t Attr;
2746   if (Str.getAsInteger(10, Attr))
2747     return MatchOperand_ParseFail;
2748 
2749   SMLoc S = Parser.getTok().getLoc();
2750   Parser.Lex();
2751   if (Attr > 63) {
2752     Error(S, "out of bounds attr");
2753     return MatchOperand_Success;
2754   }
2755 
2756   SMLoc SChan = SMLoc::getFromPointer(Chan.data());
2757 
2758   Operands.push_back(AMDGPUOperand::CreateImm(this, Attr, S,
2759                                               AMDGPUOperand::ImmTyInterpAttr));
2760   Operands.push_back(AMDGPUOperand::CreateImm(this, AttrChan, SChan,
2761                                               AMDGPUOperand::ImmTyAttrChan));
2762   return MatchOperand_Success;
2763 }
2764 
2765 void AMDGPUAsmParser::errorExpTgt() {
2766   Error(Parser.getTok().getLoc(), "invalid exp target");
2767 }
2768 
2769 OperandMatchResultTy AMDGPUAsmParser::parseExpTgtImpl(StringRef Str,
2770                                                       uint8_t &Val) {
2771   if (Str == "null") {
2772     Val = 9;
2773     return MatchOperand_Success;
2774   }
2775 
2776   if (Str.startswith("mrt")) {
2777     Str = Str.drop_front(3);
2778     if (Str == "z") { // == mrtz
2779       Val = 8;
2780       return MatchOperand_Success;
2781     }
2782 
2783     if (Str.getAsInteger(10, Val))
2784       return MatchOperand_ParseFail;
2785 
2786     if (Val > 7)
2787       errorExpTgt();
2788 
2789     return MatchOperand_Success;
2790   }
2791 
2792   if (Str.startswith("pos")) {
2793     Str = Str.drop_front(3);
2794     if (Str.getAsInteger(10, Val))
2795       return MatchOperand_ParseFail;
2796 
2797     if (Val > 3)
2798       errorExpTgt();
2799 
2800     Val += 12;
2801     return MatchOperand_Success;
2802   }
2803 
2804   if (Str.startswith("param")) {
2805     Str = Str.drop_front(5);
2806     if (Str.getAsInteger(10, Val))
2807       return MatchOperand_ParseFail;
2808 
2809     if (Val >= 32)
2810       errorExpTgt();
2811 
2812     Val += 32;
2813     return MatchOperand_Success;
2814   }
2815 
2816   if (Str.startswith("invalid_target_")) {
2817     Str = Str.drop_front(15);
2818     if (Str.getAsInteger(10, Val))
2819       return MatchOperand_ParseFail;
2820 
2821     errorExpTgt();
2822     return MatchOperand_Success;
2823   }
2824 
2825   return MatchOperand_NoMatch;
2826 }
2827 
2828 OperandMatchResultTy AMDGPUAsmParser::parseExpTgt(OperandVector &Operands) {
2829   uint8_t Val;
2830   StringRef Str = Parser.getTok().getString();
2831 
2832   auto Res = parseExpTgtImpl(Str, Val);
2833   if (Res != MatchOperand_Success)
2834     return Res;
2835 
2836   SMLoc S = Parser.getTok().getLoc();
2837   Parser.Lex();
2838 
2839   Operands.push_back(AMDGPUOperand::CreateImm(this, Val, S,
2840                                               AMDGPUOperand::ImmTyExpTgt));
2841   return MatchOperand_Success;
2842 }
2843 
2844 OperandMatchResultTy
2845 AMDGPUAsmParser::parseSendMsgOp(OperandVector &Operands) {
2846   using namespace llvm::AMDGPU::SendMsg;
2847 
2848   int64_t Imm16Val = 0;
2849   SMLoc S = Parser.getTok().getLoc();
2850 
2851   switch(getLexer().getKind()) {
2852   default:
2853     return MatchOperand_NoMatch;
2854   case AsmToken::Integer:
2855     // The operand can be an integer value.
2856     if (getParser().parseAbsoluteExpression(Imm16Val))
2857       return MatchOperand_NoMatch;
2858     if (Imm16Val < 0 || !isUInt<16>(Imm16Val)) {
2859       Error(S, "invalid immediate: only 16-bit values are legal");
2860       // Do not return error code, but create an imm operand anyway and proceed
2861       // to the next operand, if any. That avoids unneccessary error messages.
2862     }
2863     break;
2864   case AsmToken::Identifier: {
2865       OperandInfoTy Msg(ID_UNKNOWN_);
2866       OperandInfoTy Operation(OP_UNKNOWN_);
2867       int64_t StreamId = STREAM_ID_DEFAULT_;
2868       if (parseSendMsgConstruct(Msg, Operation, StreamId))
2869         return MatchOperand_ParseFail;
2870       do {
2871         // Validate and encode message ID.
2872         if (! ((ID_INTERRUPT <= Msg.Id && Msg.Id <= ID_GS_DONE)
2873                 || Msg.Id == ID_SYSMSG)) {
2874           if (Msg.IsSymbolic)
2875             Error(S, "invalid/unsupported symbolic name of message");
2876           else
2877             Error(S, "invalid/unsupported code of message");
2878           break;
2879         }
2880         Imm16Val = (Msg.Id << ID_SHIFT_);
2881         // Validate and encode operation ID.
2882         if (Msg.Id == ID_GS || Msg.Id == ID_GS_DONE) {
2883           if (! (OP_GS_FIRST_ <= Operation.Id && Operation.Id < OP_GS_LAST_)) {
2884             if (Operation.IsSymbolic)
2885               Error(S, "invalid symbolic name of GS_OP");
2886             else
2887               Error(S, "invalid code of GS_OP: only 2-bit values are legal");
2888             break;
2889           }
2890           if (Operation.Id == OP_GS_NOP
2891               && Msg.Id != ID_GS_DONE) {
2892             Error(S, "invalid GS_OP: NOP is for GS_DONE only");
2893             break;
2894           }
2895           Imm16Val |= (Operation.Id << OP_SHIFT_);
2896         }
2897         if (Msg.Id == ID_SYSMSG) {
2898           if (! (OP_SYS_FIRST_ <= Operation.Id && Operation.Id < OP_SYS_LAST_)) {
2899             if (Operation.IsSymbolic)
2900               Error(S, "invalid/unsupported symbolic name of SYSMSG_OP");
2901             else
2902               Error(S, "invalid/unsupported code of SYSMSG_OP");
2903             break;
2904           }
2905           Imm16Val |= (Operation.Id << OP_SHIFT_);
2906         }
2907         // Validate and encode stream ID.
2908         if ((Msg.Id == ID_GS || Msg.Id == ID_GS_DONE) && Operation.Id != OP_GS_NOP) {
2909           if (! (STREAM_ID_FIRST_ <= StreamId && StreamId < STREAM_ID_LAST_)) {
2910             Error(S, "invalid stream id: only 2-bit values are legal");
2911             break;
2912           }
2913           Imm16Val |= (StreamId << STREAM_ID_SHIFT_);
2914         }
2915       } while (false);
2916     }
2917     break;
2918   }
2919   Operands.push_back(AMDGPUOperand::CreateImm(this, Imm16Val, S, AMDGPUOperand::ImmTySendMsg));
2920   return MatchOperand_Success;
2921 }
2922 
2923 bool AMDGPUOperand::isSendMsg() const {
2924   return isImmTy(ImmTySendMsg);
2925 }
2926 
2927 //===----------------------------------------------------------------------===//
2928 // sopp branch targets
2929 //===----------------------------------------------------------------------===//
2930 
2931 OperandMatchResultTy
2932 AMDGPUAsmParser::parseSOppBrTarget(OperandVector &Operands) {
2933   SMLoc S = Parser.getTok().getLoc();
2934 
2935   switch (getLexer().getKind()) {
2936     default: return MatchOperand_ParseFail;
2937     case AsmToken::Integer: {
2938       int64_t Imm;
2939       if (getParser().parseAbsoluteExpression(Imm))
2940         return MatchOperand_ParseFail;
2941       Operands.push_back(AMDGPUOperand::CreateImm(this, Imm, S));
2942       return MatchOperand_Success;
2943     }
2944 
2945     case AsmToken::Identifier:
2946       Operands.push_back(AMDGPUOperand::CreateExpr(this,
2947           MCSymbolRefExpr::create(getContext().getOrCreateSymbol(
2948                                   Parser.getTok().getString()), getContext()), S));
2949       Parser.Lex();
2950       return MatchOperand_Success;
2951   }
2952 }
2953 
2954 //===----------------------------------------------------------------------===//
2955 // mubuf
2956 //===----------------------------------------------------------------------===//
2957 
2958 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultGLC() const {
2959   return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTyGLC);
2960 }
2961 
2962 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultSLC() const {
2963   return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTySLC);
2964 }
2965 
2966 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultTFE() const {
2967   return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTyTFE);
2968 }
2969 
2970 void AMDGPUAsmParser::cvtMubufImpl(MCInst &Inst,
2971                                const OperandVector &Operands,
2972                                bool IsAtomic, bool IsAtomicReturn) {
2973   OptionalImmIndexMap OptionalIdx;
2974   assert(IsAtomicReturn ? IsAtomic : true);
2975 
2976   for (unsigned i = 1, e = Operands.size(); i != e; ++i) {
2977     AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[i]);
2978 
2979     // Add the register arguments
2980     if (Op.isReg()) {
2981       Op.addRegOperands(Inst, 1);
2982       continue;
2983     }
2984 
2985     // Handle the case where soffset is an immediate
2986     if (Op.isImm() && Op.getImmTy() == AMDGPUOperand::ImmTyNone) {
2987       Op.addImmOperands(Inst, 1);
2988       continue;
2989     }
2990 
2991     // Handle tokens like 'offen' which are sometimes hard-coded into the
2992     // asm string.  There are no MCInst operands for these.
2993     if (Op.isToken()) {
2994       continue;
2995     }
2996     assert(Op.isImm());
2997 
2998     // Handle optional arguments
2999     OptionalIdx[Op.getImmTy()] = i;
3000   }
3001 
3002   // Copy $vdata_in operand and insert as $vdata for MUBUF_Atomic RTN insns.
3003   if (IsAtomicReturn) {
3004     MCInst::iterator I = Inst.begin(); // $vdata_in is always at the beginning.
3005     Inst.insert(I, *I);
3006   }
3007 
3008   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyOffset);
3009   if (!IsAtomic) { // glc is hard-coded.
3010     addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyGLC);
3011   }
3012   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySLC);
3013   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyTFE);
3014 }
3015 
3016 //===----------------------------------------------------------------------===//
3017 // mimg
3018 //===----------------------------------------------------------------------===//
3019 
3020 void AMDGPUAsmParser::cvtMIMG(MCInst &Inst, const OperandVector &Operands) {
3021   unsigned I = 1;
3022   const MCInstrDesc &Desc = MII.get(Inst.getOpcode());
3023   for (unsigned J = 0; J < Desc.getNumDefs(); ++J) {
3024     ((AMDGPUOperand &)*Operands[I++]).addRegOperands(Inst, 1);
3025   }
3026 
3027   OptionalImmIndexMap OptionalIdx;
3028 
3029   for (unsigned E = Operands.size(); I != E; ++I) {
3030     AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[I]);
3031 
3032     // Add the register arguments
3033     if (Op.isRegOrImm()) {
3034       Op.addRegOrImmOperands(Inst, 1);
3035       continue;
3036     } else if (Op.isImmModifier()) {
3037       OptionalIdx[Op.getImmTy()] = I;
3038     } else {
3039       llvm_unreachable("unexpected operand type");
3040     }
3041   }
3042 
3043   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyDMask);
3044   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyUNorm);
3045   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyGLC);
3046   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyDA);
3047   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyR128);
3048   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyTFE);
3049   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyLWE);
3050   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySLC);
3051 }
3052 
3053 void AMDGPUAsmParser::cvtMIMGAtomic(MCInst &Inst, const OperandVector &Operands) {
3054   unsigned I = 1;
3055   const MCInstrDesc &Desc = MII.get(Inst.getOpcode());
3056   for (unsigned J = 0; J < Desc.getNumDefs(); ++J) {
3057     ((AMDGPUOperand &)*Operands[I++]).addRegOperands(Inst, 1);
3058   }
3059 
3060   // Add src, same as dst
3061   ((AMDGPUOperand &)*Operands[I]).addRegOperands(Inst, 1);
3062 
3063   OptionalImmIndexMap OptionalIdx;
3064 
3065   for (unsigned E = Operands.size(); I != E; ++I) {
3066     AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[I]);
3067 
3068     // Add the register arguments
3069     if (Op.isRegOrImm()) {
3070       Op.addRegOrImmOperands(Inst, 1);
3071       continue;
3072     } else if (Op.isImmModifier()) {
3073       OptionalIdx[Op.getImmTy()] = I;
3074     } else {
3075       llvm_unreachable("unexpected operand type");
3076     }
3077   }
3078 
3079   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyDMask);
3080   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyUNorm);
3081   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyGLC);
3082   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyDA);
3083   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyR128);
3084   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyTFE);
3085   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyLWE);
3086   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySLC);
3087 }
3088 
3089 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultDMask() const {
3090   return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTyDMask);
3091 }
3092 
3093 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultUNorm() const {
3094   return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTyUNorm);
3095 }
3096 
3097 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultDA() const {
3098   return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTyDA);
3099 }
3100 
3101 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultR128() const {
3102   return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTyR128);
3103 }
3104 
3105 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultLWE() const {
3106   return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTyLWE);
3107 }
3108 
3109 //===----------------------------------------------------------------------===//
3110 // smrd
3111 //===----------------------------------------------------------------------===//
3112 
3113 bool AMDGPUOperand::isSMRDOffset8() const {
3114   return isImm() && isUInt<8>(getImm());
3115 }
3116 
3117 bool AMDGPUOperand::isSMRDOffset20() const {
3118   return isImm() && isUInt<20>(getImm());
3119 }
3120 
3121 bool AMDGPUOperand::isSMRDLiteralOffset() const {
3122   // 32-bit literals are only supported on CI and we only want to use them
3123   // when the offset is > 8-bits.
3124   return isImm() && !isUInt<8>(getImm()) && isUInt<32>(getImm());
3125 }
3126 
3127 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultSMRDOffset8() const {
3128   return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTyOffset);
3129 }
3130 
3131 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultSMRDOffset20() const {
3132   return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTyOffset);
3133 }
3134 
3135 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultSMRDLiteralOffset() const {
3136   return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTyOffset);
3137 }
3138 
3139 //===----------------------------------------------------------------------===//
3140 // vop3
3141 //===----------------------------------------------------------------------===//
3142 
3143 static bool ConvertOmodMul(int64_t &Mul) {
3144   if (Mul != 1 && Mul != 2 && Mul != 4)
3145     return false;
3146 
3147   Mul >>= 1;
3148   return true;
3149 }
3150 
3151 static bool ConvertOmodDiv(int64_t &Div) {
3152   if (Div == 1) {
3153     Div = 0;
3154     return true;
3155   }
3156 
3157   if (Div == 2) {
3158     Div = 3;
3159     return true;
3160   }
3161 
3162   return false;
3163 }
3164 
3165 static bool ConvertBoundCtrl(int64_t &BoundCtrl) {
3166   if (BoundCtrl == 0) {
3167     BoundCtrl = 1;
3168     return true;
3169   }
3170 
3171   if (BoundCtrl == -1) {
3172     BoundCtrl = 0;
3173     return true;
3174   }
3175 
3176   return false;
3177 }
3178 
3179 // Note: the order in this table matches the order of operands in AsmString.
3180 static const OptionalOperand AMDGPUOptionalOperandTable[] = {
3181   {"offen",   AMDGPUOperand::ImmTyOffen, true, nullptr},
3182   {"idxen",   AMDGPUOperand::ImmTyIdxen, true, nullptr},
3183   {"addr64",  AMDGPUOperand::ImmTyAddr64, true, nullptr},
3184   {"offset0", AMDGPUOperand::ImmTyOffset0, false, nullptr},
3185   {"offset1", AMDGPUOperand::ImmTyOffset1, false, nullptr},
3186   {"gds",     AMDGPUOperand::ImmTyGDS, true, nullptr},
3187   {"offset",  AMDGPUOperand::ImmTyOffset, false, nullptr},
3188   {"glc",     AMDGPUOperand::ImmTyGLC, true, nullptr},
3189   {"slc",     AMDGPUOperand::ImmTySLC, true, nullptr},
3190   {"tfe",     AMDGPUOperand::ImmTyTFE, true, nullptr},
3191   {"clamp",   AMDGPUOperand::ImmTyClampSI, true, nullptr},
3192   {"omod",    AMDGPUOperand::ImmTyOModSI, false, ConvertOmodMul},
3193   {"unorm",   AMDGPUOperand::ImmTyUNorm, true, nullptr},
3194   {"da",      AMDGPUOperand::ImmTyDA,    true, nullptr},
3195   {"r128",    AMDGPUOperand::ImmTyR128,  true, nullptr},
3196   {"lwe",     AMDGPUOperand::ImmTyLWE,   true, nullptr},
3197   {"dmask",   AMDGPUOperand::ImmTyDMask, false, nullptr},
3198   {"row_mask",   AMDGPUOperand::ImmTyDppRowMask, false, nullptr},
3199   {"bank_mask",  AMDGPUOperand::ImmTyDppBankMask, false, nullptr},
3200   {"bound_ctrl", AMDGPUOperand::ImmTyDppBoundCtrl, false, ConvertBoundCtrl},
3201   {"dst_sel",    AMDGPUOperand::ImmTySdwaDstSel, false, nullptr},
3202   {"src0_sel",   AMDGPUOperand::ImmTySdwaSrc0Sel, false, nullptr},
3203   {"src1_sel",   AMDGPUOperand::ImmTySdwaSrc1Sel, false, nullptr},
3204   {"dst_unused", AMDGPUOperand::ImmTySdwaDstUnused, false, nullptr},
3205   {"vm", AMDGPUOperand::ImmTyExpVM, true, nullptr},
3206 };
3207 
3208 OperandMatchResultTy AMDGPUAsmParser::parseOptionalOperand(OperandVector &Operands) {
3209   OperandMatchResultTy res;
3210   for (const OptionalOperand &Op : AMDGPUOptionalOperandTable) {
3211     // try to parse any optional operand here
3212     if (Op.IsBit) {
3213       res = parseNamedBit(Op.Name, Operands, Op.Type);
3214     } else if (Op.Type == AMDGPUOperand::ImmTyOModSI) {
3215       res = parseOModOperand(Operands);
3216     } else if (Op.Type == AMDGPUOperand::ImmTySdwaDstSel ||
3217                Op.Type == AMDGPUOperand::ImmTySdwaSrc0Sel ||
3218                Op.Type == AMDGPUOperand::ImmTySdwaSrc1Sel) {
3219       res = parseSDWASel(Operands, Op.Name, Op.Type);
3220     } else if (Op.Type == AMDGPUOperand::ImmTySdwaDstUnused) {
3221       res = parseSDWADstUnused(Operands);
3222     } else {
3223       res = parseIntWithPrefix(Op.Name, Operands, Op.Type, Op.ConvertResult);
3224     }
3225     if (res != MatchOperand_NoMatch) {
3226       return res;
3227     }
3228   }
3229   return MatchOperand_NoMatch;
3230 }
3231 
3232 OperandMatchResultTy AMDGPUAsmParser::parseOModOperand(OperandVector &Operands) {
3233   StringRef Name = Parser.getTok().getString();
3234   if (Name == "mul") {
3235     return parseIntWithPrefix("mul", Operands,
3236                               AMDGPUOperand::ImmTyOModSI, ConvertOmodMul);
3237   }
3238 
3239   if (Name == "div") {
3240     return parseIntWithPrefix("div", Operands,
3241                               AMDGPUOperand::ImmTyOModSI, ConvertOmodDiv);
3242   }
3243 
3244   return MatchOperand_NoMatch;
3245 }
3246 
3247 void AMDGPUAsmParser::cvtId(MCInst &Inst, const OperandVector &Operands) {
3248   unsigned I = 1;
3249   const MCInstrDesc &Desc = MII.get(Inst.getOpcode());
3250   for (unsigned J = 0; J < Desc.getNumDefs(); ++J) {
3251     ((AMDGPUOperand &)*Operands[I++]).addRegOperands(Inst, 1);
3252   }
3253   for (unsigned E = Operands.size(); I != E; ++I)
3254     ((AMDGPUOperand &)*Operands[I]).addRegOrImmOperands(Inst, 1);
3255 }
3256 
3257 void AMDGPUAsmParser::cvtVOP3_2_mod(MCInst &Inst, const OperandVector &Operands) {
3258   uint64_t TSFlags = MII.get(Inst.getOpcode()).TSFlags;
3259   if (TSFlags & SIInstrFlags::VOP3) {
3260     cvtVOP3(Inst, Operands);
3261   } else {
3262     cvtId(Inst, Operands);
3263   }
3264 }
3265 
3266 static bool isRegOrImmWithInputMods(const MCInstrDesc &Desc, unsigned OpNum) {
3267       // 1. This operand is input modifiers
3268   return Desc.OpInfo[OpNum].OperandType == AMDGPU::OPERAND_INPUT_MODS
3269       // 2. This is not last operand
3270       && Desc.NumOperands > (OpNum + 1)
3271       // 3. Next operand is register class
3272       && Desc.OpInfo[OpNum + 1].RegClass != -1
3273       // 4. Next register is not tied to any other operand
3274       && Desc.getOperandConstraint(OpNum + 1, MCOI::OperandConstraint::TIED_TO) == -1;
3275 }
3276 
3277 void AMDGPUAsmParser::cvtVOP3(MCInst &Inst, const OperandVector &Operands) {
3278   OptionalImmIndexMap OptionalIdx;
3279   unsigned I = 1;
3280   const MCInstrDesc &Desc = MII.get(Inst.getOpcode());
3281   for (unsigned J = 0; J < Desc.getNumDefs(); ++J) {
3282     ((AMDGPUOperand &)*Operands[I++]).addRegOperands(Inst, 1);
3283   }
3284 
3285   for (unsigned E = Operands.size(); I != E; ++I) {
3286     AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[I]);
3287     if (isRegOrImmWithInputMods(Desc, Inst.getNumOperands())) {
3288       Op.addRegOrImmWithFPInputModsOperands(Inst, 2);
3289     } else if (Op.isImm()) {
3290       OptionalIdx[Op.getImmTy()] = I;
3291     } else {
3292       llvm_unreachable("unhandled operand type");
3293     }
3294   }
3295 
3296   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyClampSI);
3297   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyOModSI);
3298 
3299   // special case v_mac_{f16, f32}:
3300   // it has src2 register operand that is tied to dst operand
3301   // we don't allow modifiers for this operand in assembler so src2_modifiers
3302   // should be 0
3303   if (Inst.getOpcode() == AMDGPU::V_MAC_F32_e64_si ||
3304       Inst.getOpcode() == AMDGPU::V_MAC_F32_e64_vi ||
3305       Inst.getOpcode() == AMDGPU::V_MAC_F16_e64_vi) {
3306     auto it = Inst.begin();
3307     std::advance(
3308       it,
3309       AMDGPU::getNamedOperandIdx(Inst.getOpcode() == AMDGPU::V_MAC_F16_e64_vi ?
3310                                      AMDGPU::V_MAC_F16_e64 :
3311                                      AMDGPU::V_MAC_F32_e64,
3312                                  AMDGPU::OpName::src2_modifiers));
3313     it = Inst.insert(it, MCOperand::createImm(0)); // no modifiers for src2
3314     ++it;
3315     Inst.insert(it, Inst.getOperand(0)); // src2 = dst
3316   }
3317 }
3318 
3319 //===----------------------------------------------------------------------===//
3320 // dpp
3321 //===----------------------------------------------------------------------===//
3322 
3323 bool AMDGPUOperand::isDPPCtrl() const {
3324   bool result = isImm() && getImmTy() == ImmTyDppCtrl && isUInt<9>(getImm());
3325   if (result) {
3326     int64_t Imm = getImm();
3327     return ((Imm >= 0x000) && (Imm <= 0x0ff)) ||
3328            ((Imm >= 0x101) && (Imm <= 0x10f)) ||
3329            ((Imm >= 0x111) && (Imm <= 0x11f)) ||
3330            ((Imm >= 0x121) && (Imm <= 0x12f)) ||
3331            (Imm == 0x130) ||
3332            (Imm == 0x134) ||
3333            (Imm == 0x138) ||
3334            (Imm == 0x13c) ||
3335            (Imm == 0x140) ||
3336            (Imm == 0x141) ||
3337            (Imm == 0x142) ||
3338            (Imm == 0x143);
3339   }
3340   return false;
3341 }
3342 
3343 bool AMDGPUOperand::isGPRIdxMode() const {
3344   return isImm() && isUInt<4>(getImm());
3345 }
3346 
3347 OperandMatchResultTy
3348 AMDGPUAsmParser::parseDPPCtrl(OperandVector &Operands) {
3349   SMLoc S = Parser.getTok().getLoc();
3350   StringRef Prefix;
3351   int64_t Int;
3352 
3353   if (getLexer().getKind() == AsmToken::Identifier) {
3354     Prefix = Parser.getTok().getString();
3355   } else {
3356     return MatchOperand_NoMatch;
3357   }
3358 
3359   if (Prefix == "row_mirror") {
3360     Int = 0x140;
3361     Parser.Lex();
3362   } else if (Prefix == "row_half_mirror") {
3363     Int = 0x141;
3364     Parser.Lex();
3365   } else {
3366     // Check to prevent parseDPPCtrlOps from eating invalid tokens
3367     if (Prefix != "quad_perm"
3368         && Prefix != "row_shl"
3369         && Prefix != "row_shr"
3370         && Prefix != "row_ror"
3371         && Prefix != "wave_shl"
3372         && Prefix != "wave_rol"
3373         && Prefix != "wave_shr"
3374         && Prefix != "wave_ror"
3375         && Prefix != "row_bcast") {
3376       return MatchOperand_NoMatch;
3377     }
3378 
3379     Parser.Lex();
3380     if (getLexer().isNot(AsmToken::Colon))
3381       return MatchOperand_ParseFail;
3382 
3383     if (Prefix == "quad_perm") {
3384       // quad_perm:[%d,%d,%d,%d]
3385       Parser.Lex();
3386       if (getLexer().isNot(AsmToken::LBrac))
3387         return MatchOperand_ParseFail;
3388       Parser.Lex();
3389 
3390       if (getParser().parseAbsoluteExpression(Int) || !(0 <= Int && Int <=3))
3391         return MatchOperand_ParseFail;
3392 
3393       for (int i = 0; i < 3; ++i) {
3394         if (getLexer().isNot(AsmToken::Comma))
3395           return MatchOperand_ParseFail;
3396         Parser.Lex();
3397 
3398         int64_t Temp;
3399         if (getParser().parseAbsoluteExpression(Temp) || !(0 <= Temp && Temp <=3))
3400           return MatchOperand_ParseFail;
3401         const int shift = i*2 + 2;
3402         Int += (Temp << shift);
3403       }
3404 
3405       if (getLexer().isNot(AsmToken::RBrac))
3406         return MatchOperand_ParseFail;
3407       Parser.Lex();
3408 
3409     } else {
3410       // sel:%d
3411       Parser.Lex();
3412       if (getParser().parseAbsoluteExpression(Int))
3413         return MatchOperand_ParseFail;
3414 
3415       if (Prefix == "row_shl" && 1 <= Int && Int <= 15) {
3416         Int |= 0x100;
3417       } else if (Prefix == "row_shr" && 1 <= Int && Int <= 15) {
3418         Int |= 0x110;
3419       } else if (Prefix == "row_ror" && 1 <= Int && Int <= 15) {
3420         Int |= 0x120;
3421       } else if (Prefix == "wave_shl" && 1 == Int) {
3422         Int = 0x130;
3423       } else if (Prefix == "wave_rol" && 1 == Int) {
3424         Int = 0x134;
3425       } else if (Prefix == "wave_shr" && 1 == Int) {
3426         Int = 0x138;
3427       } else if (Prefix == "wave_ror" && 1 == Int) {
3428         Int = 0x13C;
3429       } else if (Prefix == "row_bcast") {
3430         if (Int == 15) {
3431           Int = 0x142;
3432         } else if (Int == 31) {
3433           Int = 0x143;
3434         } else {
3435           return MatchOperand_ParseFail;
3436         }
3437       } else {
3438         return MatchOperand_ParseFail;
3439       }
3440     }
3441   }
3442 
3443   Operands.push_back(AMDGPUOperand::CreateImm(this, Int, S, AMDGPUOperand::ImmTyDppCtrl));
3444   return MatchOperand_Success;
3445 }
3446 
3447 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultRowMask() const {
3448   return AMDGPUOperand::CreateImm(this, 0xf, SMLoc(), AMDGPUOperand::ImmTyDppRowMask);
3449 }
3450 
3451 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultBankMask() const {
3452   return AMDGPUOperand::CreateImm(this, 0xf, SMLoc(), AMDGPUOperand::ImmTyDppBankMask);
3453 }
3454 
3455 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultBoundCtrl() const {
3456   return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTyDppBoundCtrl);
3457 }
3458 
3459 void AMDGPUAsmParser::cvtDPP(MCInst &Inst, const OperandVector &Operands) {
3460   OptionalImmIndexMap OptionalIdx;
3461 
3462   unsigned I = 1;
3463   const MCInstrDesc &Desc = MII.get(Inst.getOpcode());
3464   for (unsigned J = 0; J < Desc.getNumDefs(); ++J) {
3465     ((AMDGPUOperand &)*Operands[I++]).addRegOperands(Inst, 1);
3466   }
3467 
3468   for (unsigned E = Operands.size(); I != E; ++I) {
3469     AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[I]);
3470     // Add the register arguments
3471     if (Op.isReg() && Op.Reg.RegNo == AMDGPU::VCC) {
3472       // VOP2b (v_add_u32, v_sub_u32 ...) dpp use "vcc" token.
3473       // Skip it.
3474       continue;
3475     } if (isRegOrImmWithInputMods(Desc, Inst.getNumOperands())) {
3476       Op.addRegWithFPInputModsOperands(Inst, 2);
3477     } else if (Op.isDPPCtrl()) {
3478       Op.addImmOperands(Inst, 1);
3479     } else if (Op.isImm()) {
3480       // Handle optional arguments
3481       OptionalIdx[Op.getImmTy()] = I;
3482     } else {
3483       llvm_unreachable("Invalid operand type");
3484     }
3485   }
3486 
3487   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyDppRowMask, 0xf);
3488   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyDppBankMask, 0xf);
3489   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyDppBoundCtrl);
3490 
3491   // special case v_mac_{f16, f32}:
3492   // it has src2 register operand that is tied to dst operand
3493   if (Inst.getOpcode() == AMDGPU::V_MAC_F32_dpp ||
3494       Inst.getOpcode() == AMDGPU::V_MAC_F16_dpp) {
3495     auto it = Inst.begin();
3496     std::advance(
3497         it, AMDGPU::getNamedOperandIdx(Inst.getOpcode(), AMDGPU::OpName::src2));
3498     Inst.insert(it, Inst.getOperand(0)); // src2 = dst
3499   }
3500 }
3501 
3502 //===----------------------------------------------------------------------===//
3503 // sdwa
3504 //===----------------------------------------------------------------------===//
3505 
3506 OperandMatchResultTy
3507 AMDGPUAsmParser::parseSDWASel(OperandVector &Operands, StringRef Prefix,
3508                               AMDGPUOperand::ImmTy Type) {
3509   using namespace llvm::AMDGPU::SDWA;
3510 
3511   SMLoc S = Parser.getTok().getLoc();
3512   StringRef Value;
3513   OperandMatchResultTy res;
3514 
3515   res = parseStringWithPrefix(Prefix, Value);
3516   if (res != MatchOperand_Success) {
3517     return res;
3518   }
3519 
3520   int64_t Int;
3521   Int = StringSwitch<int64_t>(Value)
3522         .Case("BYTE_0", SdwaSel::BYTE_0)
3523         .Case("BYTE_1", SdwaSel::BYTE_1)
3524         .Case("BYTE_2", SdwaSel::BYTE_2)
3525         .Case("BYTE_3", SdwaSel::BYTE_3)
3526         .Case("WORD_0", SdwaSel::WORD_0)
3527         .Case("WORD_1", SdwaSel::WORD_1)
3528         .Case("DWORD", SdwaSel::DWORD)
3529         .Default(0xffffffff);
3530   Parser.Lex(); // eat last token
3531 
3532   if (Int == 0xffffffff) {
3533     return MatchOperand_ParseFail;
3534   }
3535 
3536   Operands.push_back(AMDGPUOperand::CreateImm(this, Int, S, Type));
3537   return MatchOperand_Success;
3538 }
3539 
3540 OperandMatchResultTy
3541 AMDGPUAsmParser::parseSDWADstUnused(OperandVector &Operands) {
3542   using namespace llvm::AMDGPU::SDWA;
3543 
3544   SMLoc S = Parser.getTok().getLoc();
3545   StringRef Value;
3546   OperandMatchResultTy res;
3547 
3548   res = parseStringWithPrefix("dst_unused", Value);
3549   if (res != MatchOperand_Success) {
3550     return res;
3551   }
3552 
3553   int64_t Int;
3554   Int = StringSwitch<int64_t>(Value)
3555         .Case("UNUSED_PAD", DstUnused::UNUSED_PAD)
3556         .Case("UNUSED_SEXT", DstUnused::UNUSED_SEXT)
3557         .Case("UNUSED_PRESERVE", DstUnused::UNUSED_PRESERVE)
3558         .Default(0xffffffff);
3559   Parser.Lex(); // eat last token
3560 
3561   if (Int == 0xffffffff) {
3562     return MatchOperand_ParseFail;
3563   }
3564 
3565   Operands.push_back(AMDGPUOperand::CreateImm(this, Int, S, AMDGPUOperand::ImmTySdwaDstUnused));
3566   return MatchOperand_Success;
3567 }
3568 
3569 void AMDGPUAsmParser::cvtSdwaVOP1(MCInst &Inst, const OperandVector &Operands) {
3570   cvtSDWA(Inst, Operands, SIInstrFlags::VOP1);
3571 }
3572 
3573 void AMDGPUAsmParser::cvtSdwaVOP2(MCInst &Inst, const OperandVector &Operands) {
3574   cvtSDWA(Inst, Operands, SIInstrFlags::VOP2);
3575 }
3576 
3577 void AMDGPUAsmParser::cvtSdwaVOPC(MCInst &Inst, const OperandVector &Operands) {
3578   cvtSDWA(Inst, Operands, SIInstrFlags::VOPC);
3579 }
3580 
3581 void AMDGPUAsmParser::cvtSDWA(MCInst &Inst, const OperandVector &Operands,
3582                               uint64_t BasicInstType) {
3583   using namespace llvm::AMDGPU::SDWA;
3584   OptionalImmIndexMap OptionalIdx;
3585 
3586   unsigned I = 1;
3587   const MCInstrDesc &Desc = MII.get(Inst.getOpcode());
3588   for (unsigned J = 0; J < Desc.getNumDefs(); ++J) {
3589     ((AMDGPUOperand &)*Operands[I++]).addRegOperands(Inst, 1);
3590   }
3591 
3592   for (unsigned E = Operands.size(); I != E; ++I) {
3593     AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[I]);
3594     // Add the register arguments
3595     if ((BasicInstType == SIInstrFlags::VOPC ||
3596          BasicInstType == SIInstrFlags::VOP2)&&
3597         Op.isReg() &&
3598         Op.Reg.RegNo == AMDGPU::VCC) {
3599       // VOPC and VOP2b (v_add_u32, v_sub_u32 ...) sdwa use "vcc" token as dst.
3600       // Skip it.
3601       continue;
3602     } else if (isRegOrImmWithInputMods(Desc, Inst.getNumOperands())) {
3603       Op.addRegWithInputModsOperands(Inst, 2);
3604     } else if (Op.isImm()) {
3605       // Handle optional arguments
3606       OptionalIdx[Op.getImmTy()] = I;
3607     } else {
3608       llvm_unreachable("Invalid operand type");
3609     }
3610   }
3611 
3612   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyClampSI, 0);
3613 
3614   if (Inst.getOpcode() != AMDGPU::V_NOP_sdwa_vi) {
3615     // V_NOP_sdwa_vi has no optional sdwa arguments
3616     switch (BasicInstType) {
3617     case SIInstrFlags::VOP1:
3618       addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySdwaDstSel, SdwaSel::DWORD);
3619       addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySdwaDstUnused, DstUnused::UNUSED_PRESERVE);
3620       addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySdwaSrc0Sel, SdwaSel::DWORD);
3621       break;
3622 
3623     case SIInstrFlags::VOP2:
3624       addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySdwaDstSel, SdwaSel::DWORD);
3625       addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySdwaDstUnused, DstUnused::UNUSED_PRESERVE);
3626       addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySdwaSrc0Sel, SdwaSel::DWORD);
3627       addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySdwaSrc1Sel, SdwaSel::DWORD);
3628       break;
3629 
3630     case SIInstrFlags::VOPC:
3631       addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySdwaSrc0Sel, SdwaSel::DWORD);
3632       addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySdwaSrc1Sel, SdwaSel::DWORD);
3633       break;
3634 
3635     default:
3636       llvm_unreachable("Invalid instruction type. Only VOP1, VOP2 and VOPC allowed");
3637     }
3638   }
3639 
3640   // special case v_mac_{f16, f32}:
3641   // it has src2 register operand that is tied to dst operand
3642   if (Inst.getOpcode() == AMDGPU::V_MAC_F32_sdwa_vi ||
3643       Inst.getOpcode() == AMDGPU::V_MAC_F16_sdwa_vi)  {
3644     auto it = Inst.begin();
3645     std::advance(
3646         it, AMDGPU::getNamedOperandIdx(Inst.getOpcode(), AMDGPU::OpName::src2));
3647     Inst.insert(it, Inst.getOperand(0)); // src2 = dst
3648   }
3649 
3650 }
3651 
3652 /// Force static initialization.
3653 extern "C" void LLVMInitializeAMDGPUAsmParser() {
3654   RegisterMCAsmParser<AMDGPUAsmParser> A(getTheAMDGPUTarget());
3655   RegisterMCAsmParser<AMDGPUAsmParser> B(getTheGCNTarget());
3656 }
3657 
3658 #define GET_REGISTER_MATCHER
3659 #define GET_MATCHER_IMPLEMENTATION
3660 #include "AMDGPUGenAsmMatcher.inc"
3661 
3662 // This fuction should be defined after auto-generated include so that we have
3663 // MatchClassKind enum defined
3664 unsigned AMDGPUAsmParser::validateTargetOperandClass(MCParsedAsmOperand &Op,
3665                                                      unsigned Kind) {
3666   // Tokens like "glc" would be parsed as immediate operands in ParseOperand().
3667   // But MatchInstructionImpl() expects to meet token and fails to validate
3668   // operand. This method checks if we are given immediate operand but expect to
3669   // get corresponding token.
3670   AMDGPUOperand &Operand = (AMDGPUOperand&)Op;
3671   switch (Kind) {
3672   case MCK_addr64:
3673     return Operand.isAddr64() ? Match_Success : Match_InvalidOperand;
3674   case MCK_gds:
3675     return Operand.isGDS() ? Match_Success : Match_InvalidOperand;
3676   case MCK_glc:
3677     return Operand.isGLC() ? Match_Success : Match_InvalidOperand;
3678   case MCK_idxen:
3679     return Operand.isIdxen() ? Match_Success : Match_InvalidOperand;
3680   case MCK_offen:
3681     return Operand.isOffen() ? Match_Success : Match_InvalidOperand;
3682   case MCK_SSrcB32:
3683     // When operands have expression values, they will return true for isToken,
3684     // because it is not possible to distinguish between a token and an
3685     // expression at parse time. MatchInstructionImpl() will always try to
3686     // match an operand as a token, when isToken returns true, and when the
3687     // name of the expression is not a valid token, the match will fail,
3688     // so we need to handle it here.
3689     return Operand.isSSrcB32() ? Match_Success : Match_InvalidOperand;
3690   case MCK_SSrcF32:
3691     return Operand.isSSrcF32() ? Match_Success : Match_InvalidOperand;
3692   case MCK_SoppBrTarget:
3693     return Operand.isSoppBrTarget() ? Match_Success : Match_InvalidOperand;
3694   case MCK_VReg32OrOff:
3695     return Operand.isVReg32OrOff() ? Match_Success : Match_InvalidOperand;
3696   case MCK_InterpSlot:
3697     return Operand.isInterpSlot() ? Match_Success : Match_InvalidOperand;
3698   case MCK_Attr:
3699     return Operand.isInterpAttr() ? Match_Success : Match_InvalidOperand;
3700   case MCK_AttrChan:
3701     return Operand.isAttrChan() ? Match_Success : Match_InvalidOperand;
3702   default:
3703     return Match_InvalidOperand;
3704   }
3705 }
3706