1 //===-- AMDGPUAsmParser.cpp - Parse SI asm to MCInst instructions ---------===//
2 //
3 //                     The LLVM Compiler Infrastructure
4 //
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
7 //
8 //===----------------------------------------------------------------------===//
9 
10 #include "AMDKernelCodeT.h"
11 #include "MCTargetDesc/AMDGPUMCTargetDesc.h"
12 #include "MCTargetDesc/AMDGPUTargetStreamer.h"
13 #include "SIDefines.h"
14 #include "Utils/AMDGPUBaseInfo.h"
15 #include "Utils/AMDKernelCodeTUtils.h"
16 #include "Utils/AMDGPUAsmUtils.h"
17 #include "llvm/ADT/APFloat.h"
18 #include "llvm/ADT/APInt.h"
19 #include "llvm/ADT/ArrayRef.h"
20 #include "llvm/ADT/SmallBitVector.h"
21 #include "llvm/ADT/SmallString.h"
22 #include "llvm/ADT/STLExtras.h"
23 #include "llvm/ADT/StringRef.h"
24 #include "llvm/ADT/StringSwitch.h"
25 #include "llvm/ADT/Twine.h"
26 #include "llvm/CodeGen/MachineValueType.h"
27 #include "llvm/MC/MCAsmInfo.h"
28 #include "llvm/MC/MCContext.h"
29 #include "llvm/MC/MCExpr.h"
30 #include "llvm/MC/MCInst.h"
31 #include "llvm/MC/MCInstrDesc.h"
32 #include "llvm/MC/MCInstrInfo.h"
33 #include "llvm/MC/MCParser/MCAsmLexer.h"
34 #include "llvm/MC/MCParser/MCAsmParser.h"
35 #include "llvm/MC/MCParser/MCAsmParserExtension.h"
36 #include "llvm/MC/MCParser/MCParsedAsmOperand.h"
37 #include "llvm/MC/MCParser/MCTargetAsmParser.h"
38 #include "llvm/MC/MCRegisterInfo.h"
39 #include "llvm/MC/MCStreamer.h"
40 #include "llvm/MC/MCSubtargetInfo.h"
41 #include "llvm/MC/MCSymbol.h"
42 #include "llvm/Support/Casting.h"
43 #include "llvm/Support/ELF.h"
44 #include "llvm/Support/ErrorHandling.h"
45 #include "llvm/Support/MathExtras.h"
46 #include "llvm/Support/raw_ostream.h"
47 #include "llvm/Support/SMLoc.h"
48 #include "llvm/Support/TargetRegistry.h"
49 #include <algorithm>
50 #include <cassert>
51 #include <cstdint>
52 #include <cstring>
53 #include <iterator>
54 #include <map>
55 #include <memory>
56 #include <string>
57 
58 using namespace llvm;
59 using namespace llvm::AMDGPU;
60 
61 namespace {
62 
63 class AMDGPUAsmParser;
64 
65 enum RegisterKind { IS_UNKNOWN, IS_VGPR, IS_SGPR, IS_TTMP, IS_SPECIAL };
66 
67 //===----------------------------------------------------------------------===//
68 // Operand
69 //===----------------------------------------------------------------------===//
70 
71 class AMDGPUOperand : public MCParsedAsmOperand {
72   enum KindTy {
73     Token,
74     Immediate,
75     Register,
76     Expression
77   } Kind;
78 
79   SMLoc StartLoc, EndLoc;
80   const AMDGPUAsmParser *AsmParser;
81 
82 public:
83   AMDGPUOperand(KindTy Kind_, const AMDGPUAsmParser *AsmParser_)
84     : MCParsedAsmOperand(), Kind(Kind_), AsmParser(AsmParser_) {}
85 
86   typedef std::unique_ptr<AMDGPUOperand> Ptr;
87 
88   struct Modifiers {
89     bool Abs = false;
90     bool Neg = false;
91     bool Sext = false;
92 
93     bool hasFPModifiers() const { return Abs || Neg; }
94     bool hasIntModifiers() const { return Sext; }
95     bool hasModifiers() const { return hasFPModifiers() || hasIntModifiers(); }
96 
97     int64_t getFPModifiersOperand() const {
98       int64_t Operand = 0;
99       Operand |= Abs ? SISrcMods::ABS : 0;
100       Operand |= Neg ? SISrcMods::NEG : 0;
101       return Operand;
102     }
103 
104     int64_t getIntModifiersOperand() const {
105       int64_t Operand = 0;
106       Operand |= Sext ? SISrcMods::SEXT : 0;
107       return Operand;
108     }
109 
110     int64_t getModifiersOperand() const {
111       assert(!(hasFPModifiers() && hasIntModifiers())
112            && "fp and int modifiers should not be used simultaneously");
113       if (hasFPModifiers()) {
114         return getFPModifiersOperand();
115       } else if (hasIntModifiers()) {
116         return getIntModifiersOperand();
117       } else {
118         return 0;
119       }
120     }
121 
122     friend raw_ostream &operator <<(raw_ostream &OS, AMDGPUOperand::Modifiers Mods);
123   };
124 
125   enum ImmTy {
126     ImmTyNone,
127     ImmTyGDS,
128     ImmTyOffen,
129     ImmTyIdxen,
130     ImmTyAddr64,
131     ImmTyOffset,
132     ImmTyOffset0,
133     ImmTyOffset1,
134     ImmTyGLC,
135     ImmTySLC,
136     ImmTyTFE,
137     ImmTyClampSI,
138     ImmTyOModSI,
139     ImmTyDppCtrl,
140     ImmTyDppRowMask,
141     ImmTyDppBankMask,
142     ImmTyDppBoundCtrl,
143     ImmTySdwaDstSel,
144     ImmTySdwaSrc0Sel,
145     ImmTySdwaSrc1Sel,
146     ImmTySdwaDstUnused,
147     ImmTyDMask,
148     ImmTyUNorm,
149     ImmTyDA,
150     ImmTyR128,
151     ImmTyLWE,
152     ImmTyExpTgt,
153     ImmTyExpCompr,
154     ImmTyExpVM,
155     ImmTyHwreg,
156     ImmTyOff,
157     ImmTySendMsg,
158     ImmTyInterpSlot,
159     ImmTyInterpAttr,
160     ImmTyAttrChan
161   };
162 
163   struct TokOp {
164     const char *Data;
165     unsigned Length;
166   };
167 
168   struct ImmOp {
169     int64_t Val;
170     ImmTy Type;
171     bool IsFPImm;
172     Modifiers Mods;
173   };
174 
175   struct RegOp {
176     unsigned RegNo;
177     bool IsForcedVOP3;
178     Modifiers Mods;
179   };
180 
181   union {
182     TokOp Tok;
183     ImmOp Imm;
184     RegOp Reg;
185     const MCExpr *Expr;
186   };
187 
188   bool isToken() const override {
189     if (Kind == Token)
190       return true;
191 
192     if (Kind != Expression || !Expr)
193       return false;
194 
195     // When parsing operands, we can't always tell if something was meant to be
196     // a token, like 'gds', or an expression that references a global variable.
197     // In this case, we assume the string is an expression, and if we need to
198     // interpret is a token, then we treat the symbol name as the token.
199     return isa<MCSymbolRefExpr>(Expr);
200   }
201 
202   bool isImm() const override {
203     return Kind == Immediate;
204   }
205 
206   bool isInlinableImm(MVT type) const;
207   bool isLiteralImm(MVT type) const;
208 
209   bool isRegKind() const {
210     return Kind == Register;
211   }
212 
213   bool isReg() const override {
214     return isRegKind() && !hasModifiers();
215   }
216 
217   bool isRegOrImmWithInputMods(MVT type) const {
218     return isRegKind() || isInlinableImm(type);
219   }
220 
221   bool isRegOrImmWithInt16InputMods() const {
222     return isRegOrImmWithInputMods(MVT::i16);
223   }
224 
225   bool isRegOrImmWithInt32InputMods() const {
226     return isRegOrImmWithInputMods(MVT::i32);
227   }
228 
229   bool isRegOrImmWithInt64InputMods() const {
230     return isRegOrImmWithInputMods(MVT::i64);
231   }
232 
233   bool isRegOrImmWithFP16InputMods() const {
234     return isRegOrImmWithInputMods(MVT::f16);
235   }
236 
237   bool isRegOrImmWithFP32InputMods() const {
238     return isRegOrImmWithInputMods(MVT::f32);
239   }
240 
241   bool isRegOrImmWithFP64InputMods() const {
242     return isRegOrImmWithInputMods(MVT::f64);
243   }
244 
245   bool isVReg() const {
246     return isRegClass(AMDGPU::VGPR_32RegClassID) ||
247            isRegClass(AMDGPU::VReg_64RegClassID) ||
248            isRegClass(AMDGPU::VReg_96RegClassID) ||
249            isRegClass(AMDGPU::VReg_128RegClassID) ||
250            isRegClass(AMDGPU::VReg_256RegClassID) ||
251            isRegClass(AMDGPU::VReg_512RegClassID);
252   }
253 
254   bool isVReg32OrOff() const {
255     return isOff() || isRegClass(AMDGPU::VGPR_32RegClassID);
256   }
257 
258   bool isImmTy(ImmTy ImmT) const {
259     return isImm() && Imm.Type == ImmT;
260   }
261 
262   bool isImmModifier() const {
263     return isImm() && Imm.Type != ImmTyNone;
264   }
265 
266   bool isClampSI() const { return isImmTy(ImmTyClampSI); }
267   bool isOModSI() const { return isImmTy(ImmTyOModSI); }
268   bool isDMask() const { return isImmTy(ImmTyDMask); }
269   bool isUNorm() const { return isImmTy(ImmTyUNorm); }
270   bool isDA() const { return isImmTy(ImmTyDA); }
271   bool isR128() const { return isImmTy(ImmTyUNorm); }
272   bool isLWE() const { return isImmTy(ImmTyLWE); }
273   bool isOff() const { return isImmTy(ImmTyOff); }
274   bool isExpTgt() const { return isImmTy(ImmTyExpTgt); }
275   bool isExpVM() const { return isImmTy(ImmTyExpVM); }
276   bool isExpCompr() const { return isImmTy(ImmTyExpCompr); }
277   bool isOffen() const { return isImmTy(ImmTyOffen); }
278   bool isIdxen() const { return isImmTy(ImmTyIdxen); }
279   bool isAddr64() const { return isImmTy(ImmTyAddr64); }
280   bool isOffset() const { return isImmTy(ImmTyOffset) && isUInt<16>(getImm()); }
281   bool isOffset0() const { return isImmTy(ImmTyOffset0) && isUInt<16>(getImm()); }
282   bool isOffset1() const { return isImmTy(ImmTyOffset1) && isUInt<8>(getImm()); }
283   bool isGDS() const { return isImmTy(ImmTyGDS); }
284   bool isGLC() const { return isImmTy(ImmTyGLC); }
285   bool isSLC() const { return isImmTy(ImmTySLC); }
286   bool isTFE() const { return isImmTy(ImmTyTFE); }
287   bool isBankMask() const { return isImmTy(ImmTyDppBankMask); }
288   bool isRowMask() const { return isImmTy(ImmTyDppRowMask); }
289   bool isBoundCtrl() const { return isImmTy(ImmTyDppBoundCtrl); }
290   bool isSDWADstSel() const { return isImmTy(ImmTySdwaDstSel); }
291   bool isSDWASrc0Sel() const { return isImmTy(ImmTySdwaSrc0Sel); }
292   bool isSDWASrc1Sel() const { return isImmTy(ImmTySdwaSrc1Sel); }
293   bool isSDWADstUnused() const { return isImmTy(ImmTySdwaDstUnused); }
294   bool isInterpSlot() const { return isImmTy(ImmTyInterpSlot); }
295   bool isInterpAttr() const { return isImmTy(ImmTyInterpAttr); }
296   bool isAttrChan() const { return isImmTy(ImmTyAttrChan); }
297 
298   bool isMod() const {
299     return isClampSI() || isOModSI();
300   }
301 
302   bool isRegOrImm() const {
303     return isReg() || isImm();
304   }
305 
306   bool isRegClass(unsigned RCID) const;
307 
308   bool isRegOrInlineNoMods(unsigned RCID, MVT type) const {
309     return (isRegClass(RCID) || isInlinableImm(type)) && !hasModifiers();
310   }
311 
312   bool isSCSrcB16() const {
313     return isRegOrInlineNoMods(AMDGPU::SReg_32RegClassID, MVT::i16);
314   }
315 
316   bool isSCSrcB32() const {
317     return isRegOrInlineNoMods(AMDGPU::SReg_32RegClassID, MVT::i32);
318   }
319 
320   bool isSCSrcB64() const {
321     return isRegOrInlineNoMods(AMDGPU::SReg_64RegClassID, MVT::i64);
322   }
323 
324   bool isSCSrcF16() const {
325     return isRegOrInlineNoMods(AMDGPU::SReg_32RegClassID, MVT::f16);
326   }
327 
328   bool isSCSrcF32() const {
329     return isRegOrInlineNoMods(AMDGPU::SReg_32RegClassID, MVT::f32);
330   }
331 
332   bool isSCSrcF64() const {
333     return isRegOrInlineNoMods(AMDGPU::SReg_64RegClassID, MVT::f64);
334   }
335 
336   bool isSSrcB32() const {
337     return isSCSrcB32() || isLiteralImm(MVT::i32) || isExpr();
338   }
339 
340   bool isSSrcB16() const {
341     return isSCSrcB16() || isLiteralImm(MVT::i16);
342   }
343 
344   bool isSSrcB64() const {
345     // TODO: Find out how SALU supports extension of 32-bit literals to 64 bits.
346     // See isVSrc64().
347     return isSCSrcB64() || isLiteralImm(MVT::i64);
348   }
349 
350   bool isSSrcF32() const {
351     return isSCSrcB32() || isLiteralImm(MVT::f32) || isExpr();
352   }
353 
354   bool isSSrcF64() const {
355     return isSCSrcB64() || isLiteralImm(MVT::f64);
356   }
357 
358   bool isSSrcF16() const {
359     return isSCSrcB16() || isLiteralImm(MVT::f16);
360   }
361 
362   bool isVCSrcB32() const {
363     return isRegOrInlineNoMods(AMDGPU::VS_32RegClassID, MVT::i32);
364   }
365 
366   bool isVCSrcB64() const {
367     return isRegOrInlineNoMods(AMDGPU::VS_64RegClassID, MVT::i64);
368   }
369 
370   bool isVCSrcB16() const {
371     return isRegOrInlineNoMods(AMDGPU::VS_32RegClassID, MVT::i16);
372   }
373 
374   bool isVCSrcF32() const {
375     return isRegOrInlineNoMods(AMDGPU::VS_32RegClassID, MVT::f32);
376   }
377 
378   bool isVCSrcF64() const {
379     return isRegOrInlineNoMods(AMDGPU::VS_64RegClassID, MVT::f64);
380   }
381 
382   bool isVCSrcF16() const {
383     return isRegOrInlineNoMods(AMDGPU::VS_32RegClassID, MVT::f16);
384   }
385 
386   bool isVSrcB32() const {
387     return isVCSrcF32() || isLiteralImm(MVT::i32);
388   }
389 
390   bool isVSrcB64() const {
391     return isVCSrcF64() || isLiteralImm(MVT::i64);
392   }
393 
394   bool isVSrcB16() const {
395     return isVCSrcF16() || isLiteralImm(MVT::i16);
396   }
397 
398   bool isVSrcF32() const {
399     return isVCSrcF32() || isLiteralImm(MVT::f32);
400   }
401 
402   bool isVSrcF64() const {
403     return isVCSrcF64() || isLiteralImm(MVT::f64);
404   }
405 
406   bool isVSrcF16() const {
407     return isVCSrcF16() || isLiteralImm(MVT::f16);
408   }
409 
410   bool isKImmFP32() const {
411     return isLiteralImm(MVT::f32);
412   }
413 
414   bool isKImmFP16() const {
415     return isLiteralImm(MVT::f16);
416   }
417 
418   bool isMem() const override {
419     return false;
420   }
421 
422   bool isExpr() const {
423     return Kind == Expression;
424   }
425 
426   bool isSoppBrTarget() const {
427     return isExpr() || isImm();
428   }
429 
430   bool isSWaitCnt() const;
431   bool isHwreg() const;
432   bool isSendMsg() const;
433   bool isSMRDOffset8() const;
434   bool isSMRDOffset20() const;
435   bool isSMRDLiteralOffset() const;
436   bool isDPPCtrl() const;
437   bool isGPRIdxMode() const;
438 
439   StringRef getExpressionAsToken() const {
440     assert(isExpr());
441     const MCSymbolRefExpr *S = cast<MCSymbolRefExpr>(Expr);
442     return S->getSymbol().getName();
443   }
444 
445   StringRef getToken() const {
446     assert(isToken());
447 
448     if (Kind == Expression)
449       return getExpressionAsToken();
450 
451     return StringRef(Tok.Data, Tok.Length);
452   }
453 
454   int64_t getImm() const {
455     assert(isImm());
456     return Imm.Val;
457   }
458 
459   ImmTy getImmTy() const {
460     assert(isImm());
461     return Imm.Type;
462   }
463 
464   unsigned getReg() const override {
465     return Reg.RegNo;
466   }
467 
468   SMLoc getStartLoc() const override {
469     return StartLoc;
470   }
471 
472   SMLoc getEndLoc() const override {
473     return EndLoc;
474   }
475 
476   Modifiers getModifiers() const {
477     assert(isRegKind() || isImmTy(ImmTyNone));
478     return isRegKind() ? Reg.Mods : Imm.Mods;
479   }
480 
481   void setModifiers(Modifiers Mods) {
482     assert(isRegKind() || isImmTy(ImmTyNone));
483     if (isRegKind())
484       Reg.Mods = Mods;
485     else
486       Imm.Mods = Mods;
487   }
488 
489   bool hasModifiers() const {
490     return getModifiers().hasModifiers();
491   }
492 
493   bool hasFPModifiers() const {
494     return getModifiers().hasFPModifiers();
495   }
496 
497   bool hasIntModifiers() const {
498     return getModifiers().hasIntModifiers();
499   }
500 
501   void addImmOperands(MCInst &Inst, unsigned N, bool ApplyModifiers = true) const;
502 
503   void addLiteralImmOperand(MCInst &Inst, int64_t Val) const;
504 
505   template <unsigned Bitwidth>
506   void addKImmFPOperands(MCInst &Inst, unsigned N) const;
507 
508   void addKImmFP16Operands(MCInst &Inst, unsigned N) const {
509     addKImmFPOperands<16>(Inst, N);
510   }
511 
512   void addKImmFP32Operands(MCInst &Inst, unsigned N) const {
513     addKImmFPOperands<32>(Inst, N);
514   }
515 
516   void addRegOperands(MCInst &Inst, unsigned N) const;
517 
518   void addRegOrImmOperands(MCInst &Inst, unsigned N) const {
519     if (isRegKind())
520       addRegOperands(Inst, N);
521     else if (isExpr())
522       Inst.addOperand(MCOperand::createExpr(Expr));
523     else
524       addImmOperands(Inst, N);
525   }
526 
527   void addRegOrImmWithInputModsOperands(MCInst &Inst, unsigned N) const {
528     Modifiers Mods = getModifiers();
529     Inst.addOperand(MCOperand::createImm(Mods.getModifiersOperand()));
530     if (isRegKind()) {
531       addRegOperands(Inst, N);
532     } else {
533       addImmOperands(Inst, N, false);
534     }
535   }
536 
537   void addRegOrImmWithFPInputModsOperands(MCInst &Inst, unsigned N) const {
538     assert(!hasIntModifiers());
539     addRegOrImmWithInputModsOperands(Inst, N);
540   }
541 
542   void addRegOrImmWithIntInputModsOperands(MCInst &Inst, unsigned N) const {
543     assert(!hasFPModifiers());
544     addRegOrImmWithInputModsOperands(Inst, N);
545   }
546 
547   void addRegWithInputModsOperands(MCInst &Inst, unsigned N) const {
548     Modifiers Mods = getModifiers();
549     Inst.addOperand(MCOperand::createImm(Mods.getModifiersOperand()));
550     assert(isRegKind());
551     addRegOperands(Inst, N);
552   }
553 
554   void addRegWithFPInputModsOperands(MCInst &Inst, unsigned N) const {
555     assert(!hasIntModifiers());
556     addRegWithInputModsOperands(Inst, N);
557   }
558 
559   void addRegWithIntInputModsOperands(MCInst &Inst, unsigned N) const {
560     assert(!hasFPModifiers());
561     addRegWithInputModsOperands(Inst, N);
562   }
563 
564   void addSoppBrTargetOperands(MCInst &Inst, unsigned N) const {
565     if (isImm())
566       addImmOperands(Inst, N);
567     else {
568       assert(isExpr());
569       Inst.addOperand(MCOperand::createExpr(Expr));
570     }
571   }
572 
573   static void printImmTy(raw_ostream& OS, ImmTy Type) {
574     switch (Type) {
575     case ImmTyNone: OS << "None"; break;
576     case ImmTyGDS: OS << "GDS"; break;
577     case ImmTyOffen: OS << "Offen"; break;
578     case ImmTyIdxen: OS << "Idxen"; break;
579     case ImmTyAddr64: OS << "Addr64"; break;
580     case ImmTyOffset: OS << "Offset"; break;
581     case ImmTyOffset0: OS << "Offset0"; break;
582     case ImmTyOffset1: OS << "Offset1"; break;
583     case ImmTyGLC: OS << "GLC"; break;
584     case ImmTySLC: OS << "SLC"; break;
585     case ImmTyTFE: OS << "TFE"; break;
586     case ImmTyClampSI: OS << "ClampSI"; break;
587     case ImmTyOModSI: OS << "OModSI"; break;
588     case ImmTyDppCtrl: OS << "DppCtrl"; break;
589     case ImmTyDppRowMask: OS << "DppRowMask"; break;
590     case ImmTyDppBankMask: OS << "DppBankMask"; break;
591     case ImmTyDppBoundCtrl: OS << "DppBoundCtrl"; break;
592     case ImmTySdwaDstSel: OS << "SdwaDstSel"; break;
593     case ImmTySdwaSrc0Sel: OS << "SdwaSrc0Sel"; break;
594     case ImmTySdwaSrc1Sel: OS << "SdwaSrc1Sel"; break;
595     case ImmTySdwaDstUnused: OS << "SdwaDstUnused"; break;
596     case ImmTyDMask: OS << "DMask"; break;
597     case ImmTyUNorm: OS << "UNorm"; break;
598     case ImmTyDA: OS << "DA"; break;
599     case ImmTyR128: OS << "R128"; break;
600     case ImmTyLWE: OS << "LWE"; break;
601     case ImmTyOff: OS << "Off"; break;
602     case ImmTyExpTgt: OS << "ExpTgt"; break;
603     case ImmTyExpCompr: OS << "ExpCompr"; break;
604     case ImmTyExpVM: OS << "ExpVM"; break;
605     case ImmTyHwreg: OS << "Hwreg"; break;
606     case ImmTySendMsg: OS << "SendMsg"; break;
607     case ImmTyInterpSlot: OS << "InterpSlot"; break;
608     case ImmTyInterpAttr: OS << "InterpAttr"; break;
609     case ImmTyAttrChan: OS << "AttrChan"; break;
610     }
611   }
612 
613   void print(raw_ostream &OS) const override {
614     switch (Kind) {
615     case Register:
616       OS << "<register " << getReg() << " mods: " << Reg.Mods << '>';
617       break;
618     case Immediate:
619       OS << '<' << getImm();
620       if (getImmTy() != ImmTyNone) {
621         OS << " type: "; printImmTy(OS, getImmTy());
622       }
623       OS << " mods: " << Imm.Mods << '>';
624       break;
625     case Token:
626       OS << '\'' << getToken() << '\'';
627       break;
628     case Expression:
629       OS << "<expr " << *Expr << '>';
630       break;
631     }
632   }
633 
634   static AMDGPUOperand::Ptr CreateImm(const AMDGPUAsmParser *AsmParser,
635                                       int64_t Val, SMLoc Loc,
636                                       ImmTy Type = ImmTyNone,
637                                       bool IsFPImm = false) {
638     auto Op = llvm::make_unique<AMDGPUOperand>(Immediate, AsmParser);
639     Op->Imm.Val = Val;
640     Op->Imm.IsFPImm = IsFPImm;
641     Op->Imm.Type = Type;
642     Op->Imm.Mods = Modifiers();
643     Op->StartLoc = Loc;
644     Op->EndLoc = Loc;
645     return Op;
646   }
647 
648   static AMDGPUOperand::Ptr CreateToken(const AMDGPUAsmParser *AsmParser,
649                                         StringRef Str, SMLoc Loc,
650                                         bool HasExplicitEncodingSize = true) {
651     auto Res = llvm::make_unique<AMDGPUOperand>(Token, AsmParser);
652     Res->Tok.Data = Str.data();
653     Res->Tok.Length = Str.size();
654     Res->StartLoc = Loc;
655     Res->EndLoc = Loc;
656     return Res;
657   }
658 
659   static AMDGPUOperand::Ptr CreateReg(const AMDGPUAsmParser *AsmParser,
660                                       unsigned RegNo, SMLoc S,
661                                       SMLoc E,
662                                       bool ForceVOP3) {
663     auto Op = llvm::make_unique<AMDGPUOperand>(Register, AsmParser);
664     Op->Reg.RegNo = RegNo;
665     Op->Reg.Mods = Modifiers();
666     Op->Reg.IsForcedVOP3 = ForceVOP3;
667     Op->StartLoc = S;
668     Op->EndLoc = E;
669     return Op;
670   }
671 
672   static AMDGPUOperand::Ptr CreateExpr(const AMDGPUAsmParser *AsmParser,
673                                        const class MCExpr *Expr, SMLoc S) {
674     auto Op = llvm::make_unique<AMDGPUOperand>(Expression, AsmParser);
675     Op->Expr = Expr;
676     Op->StartLoc = S;
677     Op->EndLoc = S;
678     return Op;
679   }
680 };
681 
682 raw_ostream &operator <<(raw_ostream &OS, AMDGPUOperand::Modifiers Mods) {
683   OS << "abs:" << Mods.Abs << " neg: " << Mods.Neg << " sext:" << Mods.Sext;
684   return OS;
685 }
686 
687 //===----------------------------------------------------------------------===//
688 // AsmParser
689 //===----------------------------------------------------------------------===//
690 
691 // Holds info related to the current kernel, e.g. count of SGPRs used.
692 // Kernel scope begins at .amdgpu_hsa_kernel directive, ends at next
693 // .amdgpu_hsa_kernel or at EOF.
694 class KernelScopeInfo {
695   int SgprIndexUnusedMin = -1;
696   int VgprIndexUnusedMin = -1;
697   MCContext *Ctx = nullptr;
698 
699   void usesSgprAt(int i) {
700     if (i >= SgprIndexUnusedMin) {
701       SgprIndexUnusedMin = ++i;
702       if (Ctx) {
703         MCSymbol * const Sym = Ctx->getOrCreateSymbol(Twine(".kernel.sgpr_count"));
704         Sym->setVariableValue(MCConstantExpr::create(SgprIndexUnusedMin, *Ctx));
705       }
706     }
707   }
708 
709   void usesVgprAt(int i) {
710     if (i >= VgprIndexUnusedMin) {
711       VgprIndexUnusedMin = ++i;
712       if (Ctx) {
713         MCSymbol * const Sym = Ctx->getOrCreateSymbol(Twine(".kernel.vgpr_count"));
714         Sym->setVariableValue(MCConstantExpr::create(VgprIndexUnusedMin, *Ctx));
715       }
716     }
717   }
718 
719 public:
720   KernelScopeInfo() = default;
721 
722   void initialize(MCContext &Context) {
723     Ctx = &Context;
724     usesSgprAt(SgprIndexUnusedMin = -1);
725     usesVgprAt(VgprIndexUnusedMin = -1);
726   }
727 
728   void usesRegister(RegisterKind RegKind, unsigned DwordRegIndex, unsigned RegWidth) {
729     switch (RegKind) {
730       case IS_SGPR: usesSgprAt(DwordRegIndex + RegWidth - 1); break;
731       case IS_VGPR: usesVgprAt(DwordRegIndex + RegWidth - 1); break;
732       default: break;
733     }
734   }
735 };
736 
737 class AMDGPUAsmParser : public MCTargetAsmParser {
738   const MCInstrInfo &MII;
739   MCAsmParser &Parser;
740 
741   unsigned ForcedEncodingSize = 0;
742   bool ForcedDPP = false;
743   bool ForcedSDWA = false;
744   KernelScopeInfo KernelScope;
745 
746   /// @name Auto-generated Match Functions
747   /// {
748 
749 #define GET_ASSEMBLER_HEADER
750 #include "AMDGPUGenAsmMatcher.inc"
751 
752   /// }
753 
754 private:
755   bool ParseAsAbsoluteExpression(uint32_t &Ret);
756   bool ParseDirectiveMajorMinor(uint32_t &Major, uint32_t &Minor);
757   bool ParseDirectiveHSACodeObjectVersion();
758   bool ParseDirectiveHSACodeObjectISA();
759   bool ParseDirectiveRuntimeMetadata();
760   bool ParseAMDKernelCodeTValue(StringRef ID, amd_kernel_code_t &Header);
761   bool ParseDirectiveAMDKernelCodeT();
762   bool ParseSectionDirectiveHSAText();
763   bool subtargetHasRegister(const MCRegisterInfo &MRI, unsigned RegNo) const;
764   bool ParseDirectiveAMDGPUHsaKernel();
765   bool ParseDirectiveAMDGPUHsaModuleGlobal();
766   bool ParseDirectiveAMDGPUHsaProgramGlobal();
767   bool ParseSectionDirectiveHSADataGlobalAgent();
768   bool ParseSectionDirectiveHSADataGlobalProgram();
769   bool ParseSectionDirectiveHSARodataReadonlyAgent();
770   bool AddNextRegisterToList(unsigned& Reg, unsigned& RegWidth,
771                              RegisterKind RegKind, unsigned Reg1,
772                              unsigned RegNum);
773   bool ParseAMDGPURegister(RegisterKind& RegKind, unsigned& Reg,
774                            unsigned& RegNum, unsigned& RegWidth,
775                            unsigned *DwordRegIndex);
776   void cvtMubufImpl(MCInst &Inst, const OperandVector &Operands,
777                     bool IsAtomic, bool IsAtomicReturn);
778   void cvtDSImpl(MCInst &Inst, const OperandVector &Operands,
779                  bool IsGdsHardcoded);
780 
781 public:
782   enum AMDGPUMatchResultTy {
783     Match_PreferE32 = FIRST_TARGET_MATCH_RESULT_TY
784   };
785 
786   AMDGPUAsmParser(const MCSubtargetInfo &STI, MCAsmParser &_Parser,
787                const MCInstrInfo &MII,
788                const MCTargetOptions &Options)
789       : MCTargetAsmParser(Options, STI), MII(MII), Parser(_Parser) {
790     MCAsmParserExtension::Initialize(Parser);
791 
792     if (getSTI().getFeatureBits().none()) {
793       // Set default features.
794       copySTI().ToggleFeature("SOUTHERN_ISLANDS");
795     }
796 
797     setAvailableFeatures(ComputeAvailableFeatures(getSTI().getFeatureBits()));
798 
799     {
800       // TODO: make those pre-defined variables read-only.
801       // Currently there is none suitable machinery in the core llvm-mc for this.
802       // MCSymbol::isRedefinable is intended for another purpose, and
803       // AsmParser::parseDirectiveSet() cannot be specialized for specific target.
804       AMDGPU::IsaVersion Isa = AMDGPU::getIsaVersion(getSTI().getFeatureBits());
805       MCContext &Ctx = getContext();
806       MCSymbol *Sym = Ctx.getOrCreateSymbol(Twine(".option.machine_version_major"));
807       Sym->setVariableValue(MCConstantExpr::create(Isa.Major, Ctx));
808       Sym = Ctx.getOrCreateSymbol(Twine(".option.machine_version_minor"));
809       Sym->setVariableValue(MCConstantExpr::create(Isa.Minor, Ctx));
810       Sym = Ctx.getOrCreateSymbol(Twine(".option.machine_version_stepping"));
811       Sym->setVariableValue(MCConstantExpr::create(Isa.Stepping, Ctx));
812     }
813     KernelScope.initialize(getContext());
814   }
815 
816   bool isSI() const {
817     return AMDGPU::isSI(getSTI());
818   }
819 
820   bool isCI() const {
821     return AMDGPU::isCI(getSTI());
822   }
823 
824   bool isVI() const {
825     return AMDGPU::isVI(getSTI());
826   }
827 
828   bool hasInv2PiInlineImm() const {
829     return getSTI().getFeatureBits()[AMDGPU::FeatureInv2PiInlineImm];
830   }
831 
832   bool hasSGPR102_SGPR103() const {
833     return !isVI();
834   }
835 
836   AMDGPUTargetStreamer &getTargetStreamer() {
837     MCTargetStreamer &TS = *getParser().getStreamer().getTargetStreamer();
838     return static_cast<AMDGPUTargetStreamer &>(TS);
839   }
840 
841   const MCRegisterInfo *getMRI() const {
842     // We need this const_cast because for some reason getContext() is not const
843     // in MCAsmParser.
844     return const_cast<AMDGPUAsmParser*>(this)->getContext().getRegisterInfo();
845   }
846 
847   const MCInstrInfo *getMII() const {
848     return &MII;
849   }
850 
851   void setForcedEncodingSize(unsigned Size) { ForcedEncodingSize = Size; }
852   void setForcedDPP(bool ForceDPP_) { ForcedDPP = ForceDPP_; }
853   void setForcedSDWA(bool ForceSDWA_) { ForcedSDWA = ForceSDWA_; }
854 
855   unsigned getForcedEncodingSize() const { return ForcedEncodingSize; }
856   bool isForcedVOP3() const { return ForcedEncodingSize == 64; }
857   bool isForcedDPP() const { return ForcedDPP; }
858   bool isForcedSDWA() const { return ForcedSDWA; }
859   ArrayRef<unsigned> getMatchedVariants() const;
860 
861   std::unique_ptr<AMDGPUOperand> parseRegister();
862   bool ParseRegister(unsigned &RegNo, SMLoc &StartLoc, SMLoc &EndLoc) override;
863   unsigned checkTargetMatchPredicate(MCInst &Inst) override;
864   unsigned validateTargetOperandClass(MCParsedAsmOperand &Op,
865                                       unsigned Kind) override;
866   bool MatchAndEmitInstruction(SMLoc IDLoc, unsigned &Opcode,
867                                OperandVector &Operands, MCStreamer &Out,
868                                uint64_t &ErrorInfo,
869                                bool MatchingInlineAsm) override;
870   bool ParseDirective(AsmToken DirectiveID) override;
871   OperandMatchResultTy parseOperand(OperandVector &Operands, StringRef Mnemonic);
872   StringRef parseMnemonicSuffix(StringRef Name);
873   bool ParseInstruction(ParseInstructionInfo &Info, StringRef Name,
874                         SMLoc NameLoc, OperandVector &Operands) override;
875   //bool ProcessInstruction(MCInst &Inst);
876 
877   OperandMatchResultTy parseIntWithPrefix(const char *Prefix, int64_t &Int);
878   OperandMatchResultTy
879   parseIntWithPrefix(const char *Prefix, OperandVector &Operands,
880                      AMDGPUOperand::ImmTy ImmTy = AMDGPUOperand::ImmTyNone,
881                      bool (*ConvertResult)(int64_t &) = nullptr);
882   OperandMatchResultTy
883   parseNamedBit(const char *Name, OperandVector &Operands,
884                 AMDGPUOperand::ImmTy ImmTy = AMDGPUOperand::ImmTyNone);
885   OperandMatchResultTy parseStringWithPrefix(StringRef Prefix,
886                                              StringRef &Value);
887 
888   OperandMatchResultTy parseImm(OperandVector &Operands);
889   OperandMatchResultTy parseReg(OperandVector &Operands);
890   OperandMatchResultTy parseRegOrImm(OperandVector &Operands);
891   OperandMatchResultTy parseRegOrImmWithFPInputMods(OperandVector &Operands, bool AllowImm = true);
892   OperandMatchResultTy parseRegOrImmWithIntInputMods(OperandVector &Operands, bool AllowImm = true);
893   OperandMatchResultTy parseRegWithFPInputMods(OperandVector &Operands);
894   OperandMatchResultTy parseRegWithIntInputMods(OperandVector &Operands);
895   OperandMatchResultTy parseVReg32OrOff(OperandVector &Operands);
896 
897   void cvtDSOffset01(MCInst &Inst, const OperandVector &Operands);
898   void cvtDS(MCInst &Inst, const OperandVector &Operands) { cvtDSImpl(Inst, Operands, false); }
899   void cvtDSGds(MCInst &Inst, const OperandVector &Operands) { cvtDSImpl(Inst, Operands, true); }
900   void cvtExp(MCInst &Inst, const OperandVector &Operands);
901 
902   bool parseCnt(int64_t &IntVal);
903   OperandMatchResultTy parseSWaitCntOps(OperandVector &Operands);
904   OperandMatchResultTy parseHwreg(OperandVector &Operands);
905 
906 private:
907   struct OperandInfoTy {
908     int64_t Id;
909     bool IsSymbolic;
910     OperandInfoTy(int64_t Id_) : Id(Id_), IsSymbolic(false) { }
911   };
912 
913   bool parseSendMsgConstruct(OperandInfoTy &Msg, OperandInfoTy &Operation, int64_t &StreamId);
914   bool parseHwregConstruct(OperandInfoTy &HwReg, int64_t &Offset, int64_t &Width);
915 
916   void errorExpTgt();
917   OperandMatchResultTy parseExpTgtImpl(StringRef Str, uint8_t &Val);
918 
919 public:
920   OperandMatchResultTy parseOptionalOperand(OperandVector &Operands);
921 
922   OperandMatchResultTy parseExpTgt(OperandVector &Operands);
923   OperandMatchResultTy parseSendMsgOp(OperandVector &Operands);
924   OperandMatchResultTy parseInterpSlot(OperandVector &Operands);
925   OperandMatchResultTy parseInterpAttr(OperandVector &Operands);
926   OperandMatchResultTy parseSOppBrTarget(OperandVector &Operands);
927 
928   void cvtMubuf(MCInst &Inst, const OperandVector &Operands) { cvtMubufImpl(Inst, Operands, false, false); }
929   void cvtMubufAtomic(MCInst &Inst, const OperandVector &Operands) { cvtMubufImpl(Inst, Operands, true, false); }
930   void cvtMubufAtomicReturn(MCInst &Inst, const OperandVector &Operands) { cvtMubufImpl(Inst, Operands, true, true); }
931   AMDGPUOperand::Ptr defaultGLC() const;
932   AMDGPUOperand::Ptr defaultSLC() const;
933   AMDGPUOperand::Ptr defaultTFE() const;
934 
935   AMDGPUOperand::Ptr defaultDMask() const;
936   AMDGPUOperand::Ptr defaultUNorm() const;
937   AMDGPUOperand::Ptr defaultDA() const;
938   AMDGPUOperand::Ptr defaultR128() const;
939   AMDGPUOperand::Ptr defaultLWE() const;
940   AMDGPUOperand::Ptr defaultSMRDOffset8() const;
941   AMDGPUOperand::Ptr defaultSMRDOffset20() const;
942   AMDGPUOperand::Ptr defaultSMRDLiteralOffset() const;
943 
944   OperandMatchResultTy parseOModOperand(OperandVector &Operands);
945 
946   void cvtId(MCInst &Inst, const OperandVector &Operands);
947   void cvtVOP3_2_mod(MCInst &Inst, const OperandVector &Operands);
948   void cvtVOP3(MCInst &Inst, const OperandVector &Operands);
949 
950   void cvtMIMG(MCInst &Inst, const OperandVector &Operands);
951   void cvtMIMGAtomic(MCInst &Inst, const OperandVector &Operands);
952 
953   OperandMatchResultTy parseDPPCtrl(OperandVector &Operands);
954   AMDGPUOperand::Ptr defaultRowMask() const;
955   AMDGPUOperand::Ptr defaultBankMask() const;
956   AMDGPUOperand::Ptr defaultBoundCtrl() const;
957   void cvtDPP(MCInst &Inst, const OperandVector &Operands);
958 
959   OperandMatchResultTy parseSDWASel(OperandVector &Operands, StringRef Prefix,
960                                     AMDGPUOperand::ImmTy Type);
961   OperandMatchResultTy parseSDWADstUnused(OperandVector &Operands);
962   void cvtSdwaVOP1(MCInst &Inst, const OperandVector &Operands);
963   void cvtSdwaVOP2(MCInst &Inst, const OperandVector &Operands);
964   void cvtSdwaVOPC(MCInst &Inst, const OperandVector &Operands);
965   void cvtSDWA(MCInst &Inst, const OperandVector &Operands,
966                uint64_t BasicInstType);
967 };
968 
969 struct OptionalOperand {
970   const char *Name;
971   AMDGPUOperand::ImmTy Type;
972   bool IsBit;
973   bool (*ConvertResult)(int64_t&);
974 };
975 
976 } // end anonymous namespace
977 
978 // May be called with integer type with equivalent bitwidth.
979 static const fltSemantics *getFltSemantics(unsigned Size) {
980   switch (Size) {
981   case 4:
982     return &APFloat::IEEEsingle();
983   case 8:
984     return &APFloat::IEEEdouble();
985   case 2:
986     return &APFloat::IEEEhalf();
987   default:
988     llvm_unreachable("unsupported fp type");
989   }
990 }
991 
992 static const fltSemantics *getFltSemantics(MVT VT) {
993   return getFltSemantics(VT.getSizeInBits() / 8);
994 }
995 
996 //===----------------------------------------------------------------------===//
997 // Operand
998 //===----------------------------------------------------------------------===//
999 
1000 static bool canLosslesslyConvertToFPType(APFloat &FPLiteral, MVT VT) {
1001   bool Lost;
1002 
1003   // Convert literal to single precision
1004   APFloat::opStatus Status = FPLiteral.convert(*getFltSemantics(VT),
1005                                                APFloat::rmNearestTiesToEven,
1006                                                &Lost);
1007   // We allow precision lost but not overflow or underflow
1008   if (Status != APFloat::opOK &&
1009       Lost &&
1010       ((Status & APFloat::opOverflow)  != 0 ||
1011        (Status & APFloat::opUnderflow) != 0)) {
1012     return false;
1013   }
1014 
1015   return true;
1016 }
1017 
1018 bool AMDGPUOperand::isInlinableImm(MVT type) const {
1019   if (!isImmTy(ImmTyNone)) {
1020     // Only plain immediates are inlinable (e.g. "clamp" attribute is not)
1021     return false;
1022   }
1023   // TODO: We should avoid using host float here. It would be better to
1024   // check the float bit values which is what a few other places do.
1025   // We've had bot failures before due to weird NaN support on mips hosts.
1026 
1027   APInt Literal(64, Imm.Val);
1028 
1029   if (Imm.IsFPImm) { // We got fp literal token
1030     if (type == MVT::f64 || type == MVT::i64) { // Expected 64-bit operand
1031       return AMDGPU::isInlinableLiteral64(Imm.Val,
1032                                           AsmParser->hasInv2PiInlineImm());
1033     }
1034 
1035     APFloat FPLiteral(APFloat::IEEEdouble(), APInt(64, Imm.Val));
1036     if (!canLosslesslyConvertToFPType(FPLiteral, type))
1037       return false;
1038 
1039     if (type.getScalarSizeInBits() == 16) {
1040       return AMDGPU::isInlinableLiteral16(
1041         static_cast<int32_t>(FPLiteral.bitcastToAPInt().getZExtValue()),
1042         AsmParser->hasInv2PiInlineImm());
1043     }
1044 
1045     // Check if single precision literal is inlinable
1046     return AMDGPU::isInlinableLiteral32(
1047       static_cast<int32_t>(FPLiteral.bitcastToAPInt().getZExtValue()),
1048       AsmParser->hasInv2PiInlineImm());
1049   }
1050 
1051   // We got int literal token.
1052   if (type == MVT::f64 || type == MVT::i64) { // Expected 64-bit operand
1053     return AMDGPU::isInlinableLiteral64(Imm.Val,
1054                                         AsmParser->hasInv2PiInlineImm());
1055   }
1056 
1057   if (type.getScalarSizeInBits() == 16) {
1058     return AMDGPU::isInlinableLiteral16(
1059       static_cast<int16_t>(Literal.getLoBits(16).getSExtValue()),
1060       AsmParser->hasInv2PiInlineImm());
1061   }
1062 
1063   return AMDGPU::isInlinableLiteral32(
1064     static_cast<int32_t>(Literal.getLoBits(32).getZExtValue()),
1065     AsmParser->hasInv2PiInlineImm());
1066 }
1067 
1068 bool AMDGPUOperand::isLiteralImm(MVT type) const {
1069   // Check that this imediate can be added as literal
1070   if (!isImmTy(ImmTyNone)) {
1071     return false;
1072   }
1073 
1074   if (!Imm.IsFPImm) {
1075     // We got int literal token.
1076 
1077     unsigned Size = type.getSizeInBits();
1078     if (Size == 64)
1079       Size = 32;
1080 
1081     // FIXME: 64-bit operands can zero extend, sign extend, or pad zeroes for FP
1082     // types.
1083     return isUIntN(Size, Imm.Val) || isIntN(Size, Imm.Val);
1084   }
1085 
1086   // We got fp literal token
1087   if (type == MVT::f64) { // Expected 64-bit fp operand
1088     // We would set low 64-bits of literal to zeroes but we accept this literals
1089     return true;
1090   }
1091 
1092   if (type == MVT::i64) { // Expected 64-bit int operand
1093     // We don't allow fp literals in 64-bit integer instructions. It is
1094     // unclear how we should encode them.
1095     return false;
1096   }
1097 
1098   APFloat FPLiteral(APFloat::IEEEdouble(), APInt(64, Imm.Val));
1099   return canLosslesslyConvertToFPType(FPLiteral, type);
1100 }
1101 
1102 bool AMDGPUOperand::isRegClass(unsigned RCID) const {
1103   return isRegKind() && AsmParser->getMRI()->getRegClass(RCID).contains(getReg());
1104 }
1105 
1106 void AMDGPUOperand::addImmOperands(MCInst &Inst, unsigned N, bool ApplyModifiers) const {
1107   int64_t Val = Imm.Val;
1108   if (isImmTy(ImmTyNone) && ApplyModifiers && Imm.Mods.hasFPModifiers() && Imm.Mods.Neg) {
1109     // Apply modifiers to immediate value. Only negate can get here
1110     if (Imm.IsFPImm) {
1111       APFloat F(BitsToDouble(Val));
1112       F.changeSign();
1113       Val = F.bitcastToAPInt().getZExtValue();
1114     } else {
1115       Val = -Val;
1116     }
1117   }
1118 
1119   if (AMDGPU::isSISrcOperand(AsmParser->getMII()->get(Inst.getOpcode()),
1120                              Inst.getNumOperands())) {
1121     addLiteralImmOperand(Inst, Val);
1122   } else {
1123     Inst.addOperand(MCOperand::createImm(Val));
1124   }
1125 }
1126 
1127 void AMDGPUOperand::addLiteralImmOperand(MCInst &Inst, int64_t Val) const {
1128   const auto& InstDesc = AsmParser->getMII()->get(Inst.getOpcode());
1129   auto OpNum = Inst.getNumOperands();
1130   // Check that this operand accepts literals
1131   assert(AMDGPU::isSISrcOperand(InstDesc, OpNum));
1132 
1133   auto OpSize = AMDGPU::getOperandSize(InstDesc, OpNum); // expected operand size
1134 
1135   if (Imm.IsFPImm) { // We got fp literal token
1136     APInt Literal(64, Val);
1137 
1138     switch (OpSize) {
1139     case 8:
1140       if (AMDGPU::isInlinableLiteral64(Literal.getZExtValue(),
1141                                        AsmParser->hasInv2PiInlineImm())) {
1142         Inst.addOperand(MCOperand::createImm(Literal.getZExtValue()));
1143         return;
1144       }
1145 
1146       // Non-inlineable
1147       if (AMDGPU::isSISrcFPOperand(InstDesc, OpNum)) { // Expected 64-bit fp operand
1148         // For fp operands we check if low 32 bits are zeros
1149         if (Literal.getLoBits(32) != 0) {
1150           const_cast<AMDGPUAsmParser *>(AsmParser)->Warning(Inst.getLoc(),
1151           "Can't encode literal as exact 64-bit floating-point operand. "
1152           "Low 32-bits will be set to zero");
1153         }
1154 
1155         Inst.addOperand(MCOperand::createImm(Literal.lshr(32).getZExtValue()));
1156         return;
1157       }
1158 
1159       // We don't allow fp literals in 64-bit integer instructions. It is
1160       // unclear how we should encode them. This case should be checked earlier
1161       // in predicate methods (isLiteralImm())
1162       llvm_unreachable("fp literal in 64-bit integer instruction.");
1163 
1164     case 4:
1165     case 2: {
1166       bool lost;
1167       APFloat FPLiteral(APFloat::IEEEdouble(), Literal);
1168       // Convert literal to single precision
1169       FPLiteral.convert(*getFltSemantics(OpSize),
1170                         APFloat::rmNearestTiesToEven, &lost);
1171       // We allow precision lost but not overflow or underflow. This should be
1172       // checked earlier in isLiteralImm()
1173       Inst.addOperand(MCOperand::createImm(FPLiteral.bitcastToAPInt().getZExtValue()));
1174       return;
1175     }
1176     default:
1177       llvm_unreachable("invalid operand size");
1178     }
1179 
1180     return;
1181   }
1182 
1183    // We got int literal token.
1184   // Only sign extend inline immediates.
1185   // FIXME: No errors on truncation
1186   switch (OpSize) {
1187   case 4:
1188     if (isInt<32>(Val) &&
1189         AMDGPU::isInlinableLiteral32(static_cast<int32_t>(Val),
1190                                      AsmParser->hasInv2PiInlineImm())) {
1191       Inst.addOperand(MCOperand::createImm(Val));
1192       return;
1193     }
1194 
1195     Inst.addOperand(MCOperand::createImm(Val & 0xffffffff));
1196     return;
1197 
1198   case 8:
1199     if (AMDGPU::isInlinableLiteral64(Val,
1200                                      AsmParser->hasInv2PiInlineImm())) {
1201       Inst.addOperand(MCOperand::createImm(Val));
1202       return;
1203     }
1204 
1205     Inst.addOperand(MCOperand::createImm(Lo_32(Val)));
1206     return;
1207 
1208   case 2:
1209     if (isInt<16>(Val) &&
1210         AMDGPU::isInlinableLiteral16(static_cast<int16_t>(Val),
1211                                      AsmParser->hasInv2PiInlineImm())) {
1212       Inst.addOperand(MCOperand::createImm(Val));
1213       return;
1214     }
1215 
1216     Inst.addOperand(MCOperand::createImm(Val & 0xffff));
1217     return;
1218 
1219   default:
1220     llvm_unreachable("invalid operand size");
1221   }
1222 }
1223 
1224 template <unsigned Bitwidth>
1225 void AMDGPUOperand::addKImmFPOperands(MCInst &Inst, unsigned N) const {
1226   APInt Literal(64, Imm.Val);
1227 
1228   if (!Imm.IsFPImm) {
1229     // We got int literal token.
1230     Inst.addOperand(MCOperand::createImm(Literal.getLoBits(Bitwidth).getZExtValue()));
1231     return;
1232   }
1233 
1234   bool Lost;
1235   APFloat FPLiteral(APFloat::IEEEdouble(), Literal);
1236   FPLiteral.convert(*getFltSemantics(Bitwidth / 8),
1237                     APFloat::rmNearestTiesToEven, &Lost);
1238   Inst.addOperand(MCOperand::createImm(FPLiteral.bitcastToAPInt().getZExtValue()));
1239 }
1240 
1241 void AMDGPUOperand::addRegOperands(MCInst &Inst, unsigned N) const {
1242   Inst.addOperand(MCOperand::createReg(AMDGPU::getMCReg(getReg(), AsmParser->getSTI())));
1243 }
1244 
1245 //===----------------------------------------------------------------------===//
1246 // AsmParser
1247 //===----------------------------------------------------------------------===//
1248 
1249 static int getRegClass(RegisterKind Is, unsigned RegWidth) {
1250   if (Is == IS_VGPR) {
1251     switch (RegWidth) {
1252       default: return -1;
1253       case 1: return AMDGPU::VGPR_32RegClassID;
1254       case 2: return AMDGPU::VReg_64RegClassID;
1255       case 3: return AMDGPU::VReg_96RegClassID;
1256       case 4: return AMDGPU::VReg_128RegClassID;
1257       case 8: return AMDGPU::VReg_256RegClassID;
1258       case 16: return AMDGPU::VReg_512RegClassID;
1259     }
1260   } else if (Is == IS_TTMP) {
1261     switch (RegWidth) {
1262       default: return -1;
1263       case 1: return AMDGPU::TTMP_32RegClassID;
1264       case 2: return AMDGPU::TTMP_64RegClassID;
1265       case 4: return AMDGPU::TTMP_128RegClassID;
1266     }
1267   } else if (Is == IS_SGPR) {
1268     switch (RegWidth) {
1269       default: return -1;
1270       case 1: return AMDGPU::SGPR_32RegClassID;
1271       case 2: return AMDGPU::SGPR_64RegClassID;
1272       case 4: return AMDGPU::SGPR_128RegClassID;
1273       case 8: return AMDGPU::SReg_256RegClassID;
1274       case 16: return AMDGPU::SReg_512RegClassID;
1275     }
1276   }
1277   return -1;
1278 }
1279 
1280 static unsigned getSpecialRegForName(StringRef RegName) {
1281   return StringSwitch<unsigned>(RegName)
1282     .Case("exec", AMDGPU::EXEC)
1283     .Case("vcc", AMDGPU::VCC)
1284     .Case("flat_scratch", AMDGPU::FLAT_SCR)
1285     .Case("m0", AMDGPU::M0)
1286     .Case("scc", AMDGPU::SCC)
1287     .Case("tba", AMDGPU::TBA)
1288     .Case("tma", AMDGPU::TMA)
1289     .Case("flat_scratch_lo", AMDGPU::FLAT_SCR_LO)
1290     .Case("flat_scratch_hi", AMDGPU::FLAT_SCR_HI)
1291     .Case("vcc_lo", AMDGPU::VCC_LO)
1292     .Case("vcc_hi", AMDGPU::VCC_HI)
1293     .Case("exec_lo", AMDGPU::EXEC_LO)
1294     .Case("exec_hi", AMDGPU::EXEC_HI)
1295     .Case("tma_lo", AMDGPU::TMA_LO)
1296     .Case("tma_hi", AMDGPU::TMA_HI)
1297     .Case("tba_lo", AMDGPU::TBA_LO)
1298     .Case("tba_hi", AMDGPU::TBA_HI)
1299     .Default(0);
1300 }
1301 
1302 bool AMDGPUAsmParser::ParseRegister(unsigned &RegNo, SMLoc &StartLoc,
1303                                     SMLoc &EndLoc) {
1304   auto R = parseRegister();
1305   if (!R) return true;
1306   assert(R->isReg());
1307   RegNo = R->getReg();
1308   StartLoc = R->getStartLoc();
1309   EndLoc = R->getEndLoc();
1310   return false;
1311 }
1312 
1313 bool AMDGPUAsmParser::AddNextRegisterToList(unsigned &Reg, unsigned &RegWidth,
1314                                             RegisterKind RegKind, unsigned Reg1,
1315                                             unsigned RegNum) {
1316   switch (RegKind) {
1317   case IS_SPECIAL:
1318     if (Reg == AMDGPU::EXEC_LO && Reg1 == AMDGPU::EXEC_HI) {
1319       Reg = AMDGPU::EXEC;
1320       RegWidth = 2;
1321       return true;
1322     }
1323     if (Reg == AMDGPU::FLAT_SCR_LO && Reg1 == AMDGPU::FLAT_SCR_HI) {
1324       Reg = AMDGPU::FLAT_SCR;
1325       RegWidth = 2;
1326       return true;
1327     }
1328     if (Reg == AMDGPU::VCC_LO && Reg1 == AMDGPU::VCC_HI) {
1329       Reg = AMDGPU::VCC;
1330       RegWidth = 2;
1331       return true;
1332     }
1333     if (Reg == AMDGPU::TBA_LO && Reg1 == AMDGPU::TBA_HI) {
1334       Reg = AMDGPU::TBA;
1335       RegWidth = 2;
1336       return true;
1337     }
1338     if (Reg == AMDGPU::TMA_LO && Reg1 == AMDGPU::TMA_HI) {
1339       Reg = AMDGPU::TMA;
1340       RegWidth = 2;
1341       return true;
1342     }
1343     return false;
1344   case IS_VGPR:
1345   case IS_SGPR:
1346   case IS_TTMP:
1347     if (Reg1 != Reg + RegWidth) {
1348       return false;
1349     }
1350     RegWidth++;
1351     return true;
1352   default:
1353     llvm_unreachable("unexpected register kind");
1354   }
1355 }
1356 
1357 bool AMDGPUAsmParser::ParseAMDGPURegister(RegisterKind &RegKind, unsigned &Reg,
1358                                           unsigned &RegNum, unsigned &RegWidth,
1359                                           unsigned *DwordRegIndex) {
1360   if (DwordRegIndex) { *DwordRegIndex = 0; }
1361   const MCRegisterInfo *TRI = getContext().getRegisterInfo();
1362   if (getLexer().is(AsmToken::Identifier)) {
1363     StringRef RegName = Parser.getTok().getString();
1364     if ((Reg = getSpecialRegForName(RegName))) {
1365       Parser.Lex();
1366       RegKind = IS_SPECIAL;
1367     } else {
1368       unsigned RegNumIndex = 0;
1369       if (RegName[0] == 'v') {
1370         RegNumIndex = 1;
1371         RegKind = IS_VGPR;
1372       } else if (RegName[0] == 's') {
1373         RegNumIndex = 1;
1374         RegKind = IS_SGPR;
1375       } else if (RegName.startswith("ttmp")) {
1376         RegNumIndex = strlen("ttmp");
1377         RegKind = IS_TTMP;
1378       } else {
1379         return false;
1380       }
1381       if (RegName.size() > RegNumIndex) {
1382         // Single 32-bit register: vXX.
1383         if (RegName.substr(RegNumIndex).getAsInteger(10, RegNum))
1384           return false;
1385         Parser.Lex();
1386         RegWidth = 1;
1387       } else {
1388         // Range of registers: v[XX:YY]. ":YY" is optional.
1389         Parser.Lex();
1390         int64_t RegLo, RegHi;
1391         if (getLexer().isNot(AsmToken::LBrac))
1392           return false;
1393         Parser.Lex();
1394 
1395         if (getParser().parseAbsoluteExpression(RegLo))
1396           return false;
1397 
1398         const bool isRBrace = getLexer().is(AsmToken::RBrac);
1399         if (!isRBrace && getLexer().isNot(AsmToken::Colon))
1400           return false;
1401         Parser.Lex();
1402 
1403         if (isRBrace) {
1404           RegHi = RegLo;
1405         } else {
1406           if (getParser().parseAbsoluteExpression(RegHi))
1407             return false;
1408 
1409           if (getLexer().isNot(AsmToken::RBrac))
1410             return false;
1411           Parser.Lex();
1412         }
1413         RegNum = (unsigned) RegLo;
1414         RegWidth = (RegHi - RegLo) + 1;
1415       }
1416     }
1417   } else if (getLexer().is(AsmToken::LBrac)) {
1418     // List of consecutive registers: [s0,s1,s2,s3]
1419     Parser.Lex();
1420     if (!ParseAMDGPURegister(RegKind, Reg, RegNum, RegWidth, nullptr))
1421       return false;
1422     if (RegWidth != 1)
1423       return false;
1424     RegisterKind RegKind1;
1425     unsigned Reg1, RegNum1, RegWidth1;
1426     do {
1427       if (getLexer().is(AsmToken::Comma)) {
1428         Parser.Lex();
1429       } else if (getLexer().is(AsmToken::RBrac)) {
1430         Parser.Lex();
1431         break;
1432       } else if (ParseAMDGPURegister(RegKind1, Reg1, RegNum1, RegWidth1, nullptr)) {
1433         if (RegWidth1 != 1) {
1434           return false;
1435         }
1436         if (RegKind1 != RegKind) {
1437           return false;
1438         }
1439         if (!AddNextRegisterToList(Reg, RegWidth, RegKind1, Reg1, RegNum1)) {
1440           return false;
1441         }
1442       } else {
1443         return false;
1444       }
1445     } while (true);
1446   } else {
1447     return false;
1448   }
1449   switch (RegKind) {
1450   case IS_SPECIAL:
1451     RegNum = 0;
1452     RegWidth = 1;
1453     break;
1454   case IS_VGPR:
1455   case IS_SGPR:
1456   case IS_TTMP:
1457   {
1458     unsigned Size = 1;
1459     if (RegKind == IS_SGPR || RegKind == IS_TTMP) {
1460       // SGPR and TTMP registers must be aligned. Max required alignment is 4 dwords.
1461       Size = std::min(RegWidth, 4u);
1462     }
1463     if (RegNum % Size != 0)
1464       return false;
1465     if (DwordRegIndex) { *DwordRegIndex = RegNum; }
1466     RegNum = RegNum / Size;
1467     int RCID = getRegClass(RegKind, RegWidth);
1468     if (RCID == -1)
1469       return false;
1470     const MCRegisterClass RC = TRI->getRegClass(RCID);
1471     if (RegNum >= RC.getNumRegs())
1472       return false;
1473     Reg = RC.getRegister(RegNum);
1474     break;
1475   }
1476 
1477   default:
1478     llvm_unreachable("unexpected register kind");
1479   }
1480 
1481   if (!subtargetHasRegister(*TRI, Reg))
1482     return false;
1483   return true;
1484 }
1485 
1486 std::unique_ptr<AMDGPUOperand> AMDGPUAsmParser::parseRegister() {
1487   const auto &Tok = Parser.getTok();
1488   SMLoc StartLoc = Tok.getLoc();
1489   SMLoc EndLoc = Tok.getEndLoc();
1490   RegisterKind RegKind;
1491   unsigned Reg, RegNum, RegWidth, DwordRegIndex;
1492 
1493   if (!ParseAMDGPURegister(RegKind, Reg, RegNum, RegWidth, &DwordRegIndex)) {
1494     return nullptr;
1495   }
1496   KernelScope.usesRegister(RegKind, DwordRegIndex, RegWidth);
1497   return AMDGPUOperand::CreateReg(this, Reg, StartLoc, EndLoc, false);
1498 }
1499 
1500 OperandMatchResultTy
1501 AMDGPUAsmParser::parseImm(OperandVector &Operands) {
1502   // TODO: add syntactic sugar for 1/(2*PI)
1503   bool Minus = false;
1504   if (getLexer().getKind() == AsmToken::Minus) {
1505     Minus = true;
1506     Parser.Lex();
1507   }
1508 
1509   SMLoc S = Parser.getTok().getLoc();
1510   switch(getLexer().getKind()) {
1511   case AsmToken::Integer: {
1512     int64_t IntVal;
1513     if (getParser().parseAbsoluteExpression(IntVal))
1514       return MatchOperand_ParseFail;
1515     if (Minus)
1516       IntVal *= -1;
1517     Operands.push_back(AMDGPUOperand::CreateImm(this, IntVal, S));
1518     return MatchOperand_Success;
1519   }
1520   case AsmToken::Real: {
1521     int64_t IntVal;
1522     if (getParser().parseAbsoluteExpression(IntVal))
1523       return MatchOperand_ParseFail;
1524 
1525     APFloat F(BitsToDouble(IntVal));
1526     if (Minus)
1527       F.changeSign();
1528     Operands.push_back(
1529         AMDGPUOperand::CreateImm(this, F.bitcastToAPInt().getZExtValue(), S,
1530                                  AMDGPUOperand::ImmTyNone, true));
1531     return MatchOperand_Success;
1532   }
1533   default:
1534     return Minus ? MatchOperand_ParseFail : MatchOperand_NoMatch;
1535   }
1536 }
1537 
1538 OperandMatchResultTy
1539 AMDGPUAsmParser::parseReg(OperandVector &Operands) {
1540   if (auto R = parseRegister()) {
1541     assert(R->isReg());
1542     R->Reg.IsForcedVOP3 = isForcedVOP3();
1543     Operands.push_back(std::move(R));
1544     return MatchOperand_Success;
1545   }
1546   return MatchOperand_NoMatch;
1547 }
1548 
1549 OperandMatchResultTy
1550 AMDGPUAsmParser::parseRegOrImm(OperandVector &Operands) {
1551   auto res = parseImm(Operands);
1552   if (res != MatchOperand_NoMatch) {
1553     return res;
1554   }
1555 
1556   return parseReg(Operands);
1557 }
1558 
1559 OperandMatchResultTy
1560 AMDGPUAsmParser::parseRegOrImmWithFPInputMods(OperandVector &Operands,
1561                                               bool AllowImm) {
1562   // XXX: During parsing we can't determine if minus sign means
1563   // negate-modifier or negative immediate value.
1564   // By default we suppose it is modifier.
1565   bool Negate = false, Abs = false, Abs2 = false;
1566 
1567   if (getLexer().getKind()== AsmToken::Minus) {
1568     Parser.Lex();
1569     Negate = true;
1570   }
1571 
1572   if (getLexer().getKind() == AsmToken::Identifier &&
1573       Parser.getTok().getString() == "abs") {
1574     Parser.Lex();
1575     Abs2 = true;
1576     if (getLexer().isNot(AsmToken::LParen)) {
1577       Error(Parser.getTok().getLoc(), "expected left paren after abs");
1578       return MatchOperand_ParseFail;
1579     }
1580     Parser.Lex();
1581   }
1582 
1583   if (getLexer().getKind() == AsmToken::Pipe) {
1584     if (Abs2) {
1585       Error(Parser.getTok().getLoc(), "expected register or immediate");
1586       return MatchOperand_ParseFail;
1587     }
1588     Parser.Lex();
1589     Abs = true;
1590   }
1591 
1592   OperandMatchResultTy Res;
1593   if (AllowImm) {
1594     Res = parseRegOrImm(Operands);
1595   } else {
1596     Res = parseReg(Operands);
1597   }
1598   if (Res != MatchOperand_Success) {
1599     return Res;
1600   }
1601 
1602   AMDGPUOperand::Modifiers Mods;
1603   if (Negate) {
1604     Mods.Neg = true;
1605   }
1606   if (Abs) {
1607     if (getLexer().getKind() != AsmToken::Pipe) {
1608       Error(Parser.getTok().getLoc(), "expected vertical bar");
1609       return MatchOperand_ParseFail;
1610     }
1611     Parser.Lex();
1612     Mods.Abs = true;
1613   }
1614   if (Abs2) {
1615     if (getLexer().isNot(AsmToken::RParen)) {
1616       Error(Parser.getTok().getLoc(), "expected closing parentheses");
1617       return MatchOperand_ParseFail;
1618     }
1619     Parser.Lex();
1620     Mods.Abs = true;
1621   }
1622 
1623   if (Mods.hasFPModifiers()) {
1624     AMDGPUOperand &Op = static_cast<AMDGPUOperand &>(*Operands.back());
1625     Op.setModifiers(Mods);
1626   }
1627   return MatchOperand_Success;
1628 }
1629 
1630 OperandMatchResultTy
1631 AMDGPUAsmParser::parseRegOrImmWithIntInputMods(OperandVector &Operands,
1632                                                bool AllowImm) {
1633   bool Sext = false;
1634 
1635   if (getLexer().getKind() == AsmToken::Identifier &&
1636       Parser.getTok().getString() == "sext") {
1637     Parser.Lex();
1638     Sext = true;
1639     if (getLexer().isNot(AsmToken::LParen)) {
1640       Error(Parser.getTok().getLoc(), "expected left paren after sext");
1641       return MatchOperand_ParseFail;
1642     }
1643     Parser.Lex();
1644   }
1645 
1646   OperandMatchResultTy Res;
1647   if (AllowImm) {
1648     Res = parseRegOrImm(Operands);
1649   } else {
1650     Res = parseReg(Operands);
1651   }
1652   if (Res != MatchOperand_Success) {
1653     return Res;
1654   }
1655 
1656   AMDGPUOperand::Modifiers Mods;
1657   if (Sext) {
1658     if (getLexer().isNot(AsmToken::RParen)) {
1659       Error(Parser.getTok().getLoc(), "expected closing parentheses");
1660       return MatchOperand_ParseFail;
1661     }
1662     Parser.Lex();
1663     Mods.Sext = true;
1664   }
1665 
1666   if (Mods.hasIntModifiers()) {
1667     AMDGPUOperand &Op = static_cast<AMDGPUOperand &>(*Operands.back());
1668     Op.setModifiers(Mods);
1669   }
1670 
1671   return MatchOperand_Success;
1672 }
1673 
1674 OperandMatchResultTy
1675 AMDGPUAsmParser::parseRegWithFPInputMods(OperandVector &Operands) {
1676   return parseRegOrImmWithFPInputMods(Operands, false);
1677 }
1678 
1679 OperandMatchResultTy
1680 AMDGPUAsmParser::parseRegWithIntInputMods(OperandVector &Operands) {
1681   return parseRegOrImmWithIntInputMods(Operands, false);
1682 }
1683 
1684 OperandMatchResultTy AMDGPUAsmParser::parseVReg32OrOff(OperandVector &Operands) {
1685   std::unique_ptr<AMDGPUOperand> Reg = parseRegister();
1686   if (Reg) {
1687     Operands.push_back(std::move(Reg));
1688     return MatchOperand_Success;
1689   }
1690 
1691   const AsmToken &Tok = Parser.getTok();
1692   if (Tok.getString() == "off") {
1693     Operands.push_back(AMDGPUOperand::CreateImm(this, 0, Tok.getLoc(),
1694                                                 AMDGPUOperand::ImmTyOff, false));
1695     Parser.Lex();
1696     return MatchOperand_Success;
1697   }
1698 
1699   return MatchOperand_NoMatch;
1700 }
1701 
1702 unsigned AMDGPUAsmParser::checkTargetMatchPredicate(MCInst &Inst) {
1703   uint64_t TSFlags = MII.get(Inst.getOpcode()).TSFlags;
1704 
1705   if ((getForcedEncodingSize() == 32 && (TSFlags & SIInstrFlags::VOP3)) ||
1706       (getForcedEncodingSize() == 64 && !(TSFlags & SIInstrFlags::VOP3)) ||
1707       (isForcedDPP() && !(TSFlags & SIInstrFlags::DPP)) ||
1708       (isForcedSDWA() && !(TSFlags & SIInstrFlags::SDWA)) )
1709     return Match_InvalidOperand;
1710 
1711   if ((TSFlags & SIInstrFlags::VOP3) &&
1712       (TSFlags & SIInstrFlags::VOPAsmPrefer32Bit) &&
1713       getForcedEncodingSize() != 64)
1714     return Match_PreferE32;
1715 
1716   if (Inst.getOpcode() == AMDGPU::V_MAC_F32_sdwa_vi ||
1717       Inst.getOpcode() == AMDGPU::V_MAC_F16_sdwa_vi) {
1718     // v_mac_f32/16 allow only dst_sel == DWORD;
1719     auto OpNum =
1720         AMDGPU::getNamedOperandIdx(Inst.getOpcode(), AMDGPU::OpName::dst_sel);
1721     const auto &Op = Inst.getOperand(OpNum);
1722     if (!Op.isImm() || Op.getImm() != AMDGPU::SDWA::SdwaSel::DWORD) {
1723       return Match_InvalidOperand;
1724     }
1725   }
1726 
1727   return Match_Success;
1728 }
1729 
1730 // What asm variants we should check
1731 ArrayRef<unsigned> AMDGPUAsmParser::getMatchedVariants() const {
1732   if (getForcedEncodingSize() == 32) {
1733     static const unsigned Variants[] = {AMDGPUAsmVariants::DEFAULT};
1734     return makeArrayRef(Variants);
1735   }
1736 
1737   if (isForcedVOP3()) {
1738     static const unsigned Variants[] = {AMDGPUAsmVariants::VOP3};
1739     return makeArrayRef(Variants);
1740   }
1741 
1742   if (isForcedSDWA()) {
1743     static const unsigned Variants[] = {AMDGPUAsmVariants::SDWA};
1744     return makeArrayRef(Variants);
1745   }
1746 
1747   if (isForcedDPP()) {
1748     static const unsigned Variants[] = {AMDGPUAsmVariants::DPP};
1749     return makeArrayRef(Variants);
1750   }
1751 
1752   static const unsigned Variants[] = {
1753     AMDGPUAsmVariants::DEFAULT, AMDGPUAsmVariants::VOP3,
1754     AMDGPUAsmVariants::SDWA, AMDGPUAsmVariants::DPP
1755   };
1756 
1757   return makeArrayRef(Variants);
1758 }
1759 
1760 bool AMDGPUAsmParser::MatchAndEmitInstruction(SMLoc IDLoc, unsigned &Opcode,
1761                                               OperandVector &Operands,
1762                                               MCStreamer &Out,
1763                                               uint64_t &ErrorInfo,
1764                                               bool MatchingInlineAsm) {
1765   MCInst Inst;
1766   unsigned Result = Match_Success;
1767   for (auto Variant : getMatchedVariants()) {
1768     uint64_t EI;
1769     auto R = MatchInstructionImpl(Operands, Inst, EI, MatchingInlineAsm,
1770                                   Variant);
1771     // We order match statuses from least to most specific. We use most specific
1772     // status as resulting
1773     // Match_MnemonicFail < Match_InvalidOperand < Match_MissingFeature < Match_PreferE32
1774     if ((R == Match_Success) ||
1775         (R == Match_PreferE32) ||
1776         (R == Match_MissingFeature && Result != Match_PreferE32) ||
1777         (R == Match_InvalidOperand && Result != Match_MissingFeature
1778                                    && Result != Match_PreferE32) ||
1779         (R == Match_MnemonicFail   && Result != Match_InvalidOperand
1780                                    && Result != Match_MissingFeature
1781                                    && Result != Match_PreferE32)) {
1782       Result = R;
1783       ErrorInfo = EI;
1784     }
1785     if (R == Match_Success)
1786       break;
1787   }
1788 
1789   switch (Result) {
1790   default: break;
1791   case Match_Success:
1792     Inst.setLoc(IDLoc);
1793     Out.EmitInstruction(Inst, getSTI());
1794     return false;
1795 
1796   case Match_MissingFeature:
1797     return Error(IDLoc, "instruction not supported on this GPU");
1798 
1799   case Match_MnemonicFail:
1800     return Error(IDLoc, "unrecognized instruction mnemonic");
1801 
1802   case Match_InvalidOperand: {
1803     SMLoc ErrorLoc = IDLoc;
1804     if (ErrorInfo != ~0ULL) {
1805       if (ErrorInfo >= Operands.size()) {
1806         return Error(IDLoc, "too few operands for instruction");
1807       }
1808       ErrorLoc = ((AMDGPUOperand &)*Operands[ErrorInfo]).getStartLoc();
1809       if (ErrorLoc == SMLoc())
1810         ErrorLoc = IDLoc;
1811     }
1812     return Error(ErrorLoc, "invalid operand for instruction");
1813   }
1814 
1815   case Match_PreferE32:
1816     return Error(IDLoc, "internal error: instruction without _e64 suffix "
1817                         "should be encoded as e32");
1818   }
1819   llvm_unreachable("Implement any new match types added!");
1820 }
1821 
1822 bool AMDGPUAsmParser::ParseAsAbsoluteExpression(uint32_t &Ret) {
1823   int64_t Tmp = -1;
1824   if (getLexer().isNot(AsmToken::Integer) && getLexer().isNot(AsmToken::Identifier)) {
1825     return true;
1826   }
1827   if (getParser().parseAbsoluteExpression(Tmp)) {
1828     return true;
1829   }
1830   Ret = static_cast<uint32_t>(Tmp);
1831   return false;
1832 }
1833 
1834 bool AMDGPUAsmParser::ParseDirectiveMajorMinor(uint32_t &Major,
1835                                                uint32_t &Minor) {
1836   if (ParseAsAbsoluteExpression(Major))
1837     return TokError("invalid major version");
1838 
1839   if (getLexer().isNot(AsmToken::Comma))
1840     return TokError("minor version number required, comma expected");
1841   Lex();
1842 
1843   if (ParseAsAbsoluteExpression(Minor))
1844     return TokError("invalid minor version");
1845 
1846   return false;
1847 }
1848 
1849 bool AMDGPUAsmParser::ParseDirectiveHSACodeObjectVersion() {
1850   uint32_t Major;
1851   uint32_t Minor;
1852 
1853   if (ParseDirectiveMajorMinor(Major, Minor))
1854     return true;
1855 
1856   getTargetStreamer().EmitDirectiveHSACodeObjectVersion(Major, Minor);
1857   return false;
1858 }
1859 
1860 bool AMDGPUAsmParser::ParseDirectiveHSACodeObjectISA() {
1861   uint32_t Major;
1862   uint32_t Minor;
1863   uint32_t Stepping;
1864   StringRef VendorName;
1865   StringRef ArchName;
1866 
1867   // If this directive has no arguments, then use the ISA version for the
1868   // targeted GPU.
1869   if (getLexer().is(AsmToken::EndOfStatement)) {
1870     AMDGPU::IsaVersion Isa = AMDGPU::getIsaVersion(getSTI().getFeatureBits());
1871     getTargetStreamer().EmitDirectiveHSACodeObjectISA(Isa.Major, Isa.Minor,
1872                                                       Isa.Stepping,
1873                                                       "AMD", "AMDGPU");
1874     return false;
1875   }
1876 
1877   if (ParseDirectiveMajorMinor(Major, Minor))
1878     return true;
1879 
1880   if (getLexer().isNot(AsmToken::Comma))
1881     return TokError("stepping version number required, comma expected");
1882   Lex();
1883 
1884   if (ParseAsAbsoluteExpression(Stepping))
1885     return TokError("invalid stepping version");
1886 
1887   if (getLexer().isNot(AsmToken::Comma))
1888     return TokError("vendor name required, comma expected");
1889   Lex();
1890 
1891   if (getLexer().isNot(AsmToken::String))
1892     return TokError("invalid vendor name");
1893 
1894   VendorName = getLexer().getTok().getStringContents();
1895   Lex();
1896 
1897   if (getLexer().isNot(AsmToken::Comma))
1898     return TokError("arch name required, comma expected");
1899   Lex();
1900 
1901   if (getLexer().isNot(AsmToken::String))
1902     return TokError("invalid arch name");
1903 
1904   ArchName = getLexer().getTok().getStringContents();
1905   Lex();
1906 
1907   getTargetStreamer().EmitDirectiveHSACodeObjectISA(Major, Minor, Stepping,
1908                                                     VendorName, ArchName);
1909   return false;
1910 }
1911 
1912 bool AMDGPUAsmParser::ParseDirectiveRuntimeMetadata() {
1913   std::string Metadata;
1914   raw_string_ostream MS(Metadata);
1915 
1916   getLexer().setSkipSpace(false);
1917 
1918   bool FoundEnd = false;
1919   while (!getLexer().is(AsmToken::Eof)) {
1920     while (getLexer().is(AsmToken::Space)) {
1921       MS << ' ';
1922       Lex();
1923     }
1924 
1925     if (getLexer().is(AsmToken::Identifier)) {
1926       StringRef ID = getLexer().getTok().getIdentifier();
1927       if (ID == ".end_amdgpu_runtime_metadata") {
1928         Lex();
1929         FoundEnd = true;
1930         break;
1931       }
1932     }
1933 
1934     MS << Parser.parseStringToEndOfStatement()
1935        << getContext().getAsmInfo()->getSeparatorString();
1936 
1937     Parser.eatToEndOfStatement();
1938   }
1939 
1940   getLexer().setSkipSpace(true);
1941 
1942   if (getLexer().is(AsmToken::Eof) && !FoundEnd)
1943     return TokError("expected directive .end_amdgpu_runtime_metadata not found");
1944 
1945   MS.flush();
1946 
1947   getTargetStreamer().EmitRuntimeMetadata(Metadata);
1948 
1949   return false;
1950 }
1951 
1952 bool AMDGPUAsmParser::ParseAMDKernelCodeTValue(StringRef ID,
1953                                                amd_kernel_code_t &Header) {
1954   SmallString<40> ErrStr;
1955   raw_svector_ostream Err(ErrStr);
1956   if (!parseAmdKernelCodeField(ID, getParser(), Header, Err)) {
1957     return TokError(Err.str());
1958   }
1959   Lex();
1960   return false;
1961 }
1962 
1963 bool AMDGPUAsmParser::ParseDirectiveAMDKernelCodeT() {
1964   amd_kernel_code_t Header;
1965   AMDGPU::initDefaultAMDKernelCodeT(Header, getSTI().getFeatureBits());
1966 
1967   while (true) {
1968     // Lex EndOfStatement.  This is in a while loop, because lexing a comment
1969     // will set the current token to EndOfStatement.
1970     while(getLexer().is(AsmToken::EndOfStatement))
1971       Lex();
1972 
1973     if (getLexer().isNot(AsmToken::Identifier))
1974       return TokError("expected value identifier or .end_amd_kernel_code_t");
1975 
1976     StringRef ID = getLexer().getTok().getIdentifier();
1977     Lex();
1978 
1979     if (ID == ".end_amd_kernel_code_t")
1980       break;
1981 
1982     if (ParseAMDKernelCodeTValue(ID, Header))
1983       return true;
1984   }
1985 
1986   getTargetStreamer().EmitAMDKernelCodeT(Header);
1987 
1988   return false;
1989 }
1990 
1991 bool AMDGPUAsmParser::ParseSectionDirectiveHSAText() {
1992   getParser().getStreamer().SwitchSection(
1993       AMDGPU::getHSATextSection(getContext()));
1994   return false;
1995 }
1996 
1997 bool AMDGPUAsmParser::ParseDirectiveAMDGPUHsaKernel() {
1998   if (getLexer().isNot(AsmToken::Identifier))
1999     return TokError("expected symbol name");
2000 
2001   StringRef KernelName = Parser.getTok().getString();
2002 
2003   getTargetStreamer().EmitAMDGPUSymbolType(KernelName,
2004                                            ELF::STT_AMDGPU_HSA_KERNEL);
2005   Lex();
2006   KernelScope.initialize(getContext());
2007   return false;
2008 }
2009 
2010 bool AMDGPUAsmParser::ParseDirectiveAMDGPUHsaModuleGlobal() {
2011   if (getLexer().isNot(AsmToken::Identifier))
2012     return TokError("expected symbol name");
2013 
2014   StringRef GlobalName = Parser.getTok().getIdentifier();
2015 
2016   getTargetStreamer().EmitAMDGPUHsaModuleScopeGlobal(GlobalName);
2017   Lex();
2018   return false;
2019 }
2020 
2021 bool AMDGPUAsmParser::ParseDirectiveAMDGPUHsaProgramGlobal() {
2022   if (getLexer().isNot(AsmToken::Identifier))
2023     return TokError("expected symbol name");
2024 
2025   StringRef GlobalName = Parser.getTok().getIdentifier();
2026 
2027   getTargetStreamer().EmitAMDGPUHsaProgramScopeGlobal(GlobalName);
2028   Lex();
2029   return false;
2030 }
2031 
2032 bool AMDGPUAsmParser::ParseSectionDirectiveHSADataGlobalAgent() {
2033   getParser().getStreamer().SwitchSection(
2034       AMDGPU::getHSADataGlobalAgentSection(getContext()));
2035   return false;
2036 }
2037 
2038 bool AMDGPUAsmParser::ParseSectionDirectiveHSADataGlobalProgram() {
2039   getParser().getStreamer().SwitchSection(
2040       AMDGPU::getHSADataGlobalProgramSection(getContext()));
2041   return false;
2042 }
2043 
2044 bool AMDGPUAsmParser::ParseSectionDirectiveHSARodataReadonlyAgent() {
2045   getParser().getStreamer().SwitchSection(
2046       AMDGPU::getHSARodataReadonlyAgentSection(getContext()));
2047   return false;
2048 }
2049 
2050 bool AMDGPUAsmParser::ParseDirective(AsmToken DirectiveID) {
2051   StringRef IDVal = DirectiveID.getString();
2052 
2053   if (IDVal == ".hsa_code_object_version")
2054     return ParseDirectiveHSACodeObjectVersion();
2055 
2056   if (IDVal == ".hsa_code_object_isa")
2057     return ParseDirectiveHSACodeObjectISA();
2058 
2059   if (IDVal == ".amdgpu_runtime_metadata")
2060     return ParseDirectiveRuntimeMetadata();
2061 
2062   if (IDVal == ".amd_kernel_code_t")
2063     return ParseDirectiveAMDKernelCodeT();
2064 
2065   if (IDVal == ".hsatext")
2066     return ParseSectionDirectiveHSAText();
2067 
2068   if (IDVal == ".amdgpu_hsa_kernel")
2069     return ParseDirectiveAMDGPUHsaKernel();
2070 
2071   if (IDVal == ".amdgpu_hsa_module_global")
2072     return ParseDirectiveAMDGPUHsaModuleGlobal();
2073 
2074   if (IDVal == ".amdgpu_hsa_program_global")
2075     return ParseDirectiveAMDGPUHsaProgramGlobal();
2076 
2077   if (IDVal == ".hsadata_global_agent")
2078     return ParseSectionDirectiveHSADataGlobalAgent();
2079 
2080   if (IDVal == ".hsadata_global_program")
2081     return ParseSectionDirectiveHSADataGlobalProgram();
2082 
2083   if (IDVal == ".hsarodata_readonly_agent")
2084     return ParseSectionDirectiveHSARodataReadonlyAgent();
2085 
2086   return true;
2087 }
2088 
2089 bool AMDGPUAsmParser::subtargetHasRegister(const MCRegisterInfo &MRI,
2090                                            unsigned RegNo) const {
2091   if (isCI())
2092     return true;
2093 
2094   if (isSI()) {
2095     // No flat_scr
2096     switch (RegNo) {
2097     case AMDGPU::FLAT_SCR:
2098     case AMDGPU::FLAT_SCR_LO:
2099     case AMDGPU::FLAT_SCR_HI:
2100       return false;
2101     default:
2102       return true;
2103     }
2104   }
2105 
2106   // VI only has 102 SGPRs, so make sure we aren't trying to use the 2 more that
2107   // SI/CI have.
2108   for (MCRegAliasIterator R(AMDGPU::SGPR102_SGPR103, &MRI, true);
2109        R.isValid(); ++R) {
2110     if (*R == RegNo)
2111       return false;
2112   }
2113 
2114   return true;
2115 }
2116 
2117 OperandMatchResultTy
2118 AMDGPUAsmParser::parseOperand(OperandVector &Operands, StringRef Mnemonic) {
2119   // Try to parse with a custom parser
2120   OperandMatchResultTy ResTy = MatchOperandParserImpl(Operands, Mnemonic);
2121 
2122   // If we successfully parsed the operand or if there as an error parsing,
2123   // we are done.
2124   //
2125   // If we are parsing after we reach EndOfStatement then this means we
2126   // are appending default values to the Operands list.  This is only done
2127   // by custom parser, so we shouldn't continue on to the generic parsing.
2128   if (ResTy == MatchOperand_Success || ResTy == MatchOperand_ParseFail ||
2129       getLexer().is(AsmToken::EndOfStatement))
2130     return ResTy;
2131 
2132   ResTy = parseRegOrImm(Operands);
2133 
2134   if (ResTy == MatchOperand_Success)
2135     return ResTy;
2136 
2137   if (getLexer().getKind() == AsmToken::Identifier) {
2138     // If this identifier is a symbol, we want to create an expression for it.
2139     // It is a little difficult to distinguish between a symbol name, and
2140     // an instruction flag like 'gds'.  In order to do this, we parse
2141     // all tokens as expressions and then treate the symbol name as the token
2142     // string when we want to interpret the operand as a token.
2143     const auto &Tok = Parser.getTok();
2144     SMLoc S = Tok.getLoc();
2145     const MCExpr *Expr = nullptr;
2146     if (!Parser.parseExpression(Expr)) {
2147       Operands.push_back(AMDGPUOperand::CreateExpr(this, Expr, S));
2148       return MatchOperand_Success;
2149     }
2150 
2151     Operands.push_back(AMDGPUOperand::CreateToken(this, Tok.getString(), Tok.getLoc()));
2152     Parser.Lex();
2153     return MatchOperand_Success;
2154   }
2155   return MatchOperand_NoMatch;
2156 }
2157 
2158 StringRef AMDGPUAsmParser::parseMnemonicSuffix(StringRef Name) {
2159   // Clear any forced encodings from the previous instruction.
2160   setForcedEncodingSize(0);
2161   setForcedDPP(false);
2162   setForcedSDWA(false);
2163 
2164   if (Name.endswith("_e64")) {
2165     setForcedEncodingSize(64);
2166     return Name.substr(0, Name.size() - 4);
2167   } else if (Name.endswith("_e32")) {
2168     setForcedEncodingSize(32);
2169     return Name.substr(0, Name.size() - 4);
2170   } else if (Name.endswith("_dpp")) {
2171     setForcedDPP(true);
2172     return Name.substr(0, Name.size() - 4);
2173   } else if (Name.endswith("_sdwa")) {
2174     setForcedSDWA(true);
2175     return Name.substr(0, Name.size() - 5);
2176   }
2177   return Name;
2178 }
2179 
2180 bool AMDGPUAsmParser::ParseInstruction(ParseInstructionInfo &Info,
2181                                        StringRef Name,
2182                                        SMLoc NameLoc, OperandVector &Operands) {
2183   // Add the instruction mnemonic
2184   Name = parseMnemonicSuffix(Name);
2185   Operands.push_back(AMDGPUOperand::CreateToken(this, Name, NameLoc));
2186 
2187   while (!getLexer().is(AsmToken::EndOfStatement)) {
2188     OperandMatchResultTy Res = parseOperand(Operands, Name);
2189 
2190     // Eat the comma or space if there is one.
2191     if (getLexer().is(AsmToken::Comma))
2192       Parser.Lex();
2193 
2194     switch (Res) {
2195       case MatchOperand_Success: break;
2196       case MatchOperand_ParseFail:
2197         Error(getLexer().getLoc(), "failed parsing operand.");
2198         while (!getLexer().is(AsmToken::EndOfStatement)) {
2199           Parser.Lex();
2200         }
2201         return true;
2202       case MatchOperand_NoMatch:
2203         Error(getLexer().getLoc(), "not a valid operand.");
2204         while (!getLexer().is(AsmToken::EndOfStatement)) {
2205           Parser.Lex();
2206         }
2207         return true;
2208     }
2209   }
2210 
2211   return false;
2212 }
2213 
2214 //===----------------------------------------------------------------------===//
2215 // Utility functions
2216 //===----------------------------------------------------------------------===//
2217 
2218 OperandMatchResultTy
2219 AMDGPUAsmParser::parseIntWithPrefix(const char *Prefix, int64_t &Int) {
2220   switch(getLexer().getKind()) {
2221     default: return MatchOperand_NoMatch;
2222     case AsmToken::Identifier: {
2223       StringRef Name = Parser.getTok().getString();
2224       if (!Name.equals(Prefix)) {
2225         return MatchOperand_NoMatch;
2226       }
2227 
2228       Parser.Lex();
2229       if (getLexer().isNot(AsmToken::Colon))
2230         return MatchOperand_ParseFail;
2231 
2232       Parser.Lex();
2233       if (getLexer().isNot(AsmToken::Integer))
2234         return MatchOperand_ParseFail;
2235 
2236       if (getParser().parseAbsoluteExpression(Int))
2237         return MatchOperand_ParseFail;
2238       break;
2239     }
2240   }
2241   return MatchOperand_Success;
2242 }
2243 
2244 OperandMatchResultTy
2245 AMDGPUAsmParser::parseIntWithPrefix(const char *Prefix, OperandVector &Operands,
2246                                     AMDGPUOperand::ImmTy ImmTy,
2247                                     bool (*ConvertResult)(int64_t&)) {
2248   SMLoc S = Parser.getTok().getLoc();
2249   int64_t Value = 0;
2250 
2251   OperandMatchResultTy Res = parseIntWithPrefix(Prefix, Value);
2252   if (Res != MatchOperand_Success)
2253     return Res;
2254 
2255   if (ConvertResult && !ConvertResult(Value)) {
2256     return MatchOperand_ParseFail;
2257   }
2258 
2259   Operands.push_back(AMDGPUOperand::CreateImm(this, Value, S, ImmTy));
2260   return MatchOperand_Success;
2261 }
2262 
2263 OperandMatchResultTy
2264 AMDGPUAsmParser::parseNamedBit(const char *Name, OperandVector &Operands,
2265                                AMDGPUOperand::ImmTy ImmTy) {
2266   int64_t Bit = 0;
2267   SMLoc S = Parser.getTok().getLoc();
2268 
2269   // We are at the end of the statement, and this is a default argument, so
2270   // use a default value.
2271   if (getLexer().isNot(AsmToken::EndOfStatement)) {
2272     switch(getLexer().getKind()) {
2273       case AsmToken::Identifier: {
2274         StringRef Tok = Parser.getTok().getString();
2275         if (Tok == Name) {
2276           Bit = 1;
2277           Parser.Lex();
2278         } else if (Tok.startswith("no") && Tok.endswith(Name)) {
2279           Bit = 0;
2280           Parser.Lex();
2281         } else {
2282           return MatchOperand_NoMatch;
2283         }
2284         break;
2285       }
2286       default:
2287         return MatchOperand_NoMatch;
2288     }
2289   }
2290 
2291   Operands.push_back(AMDGPUOperand::CreateImm(this, Bit, S, ImmTy));
2292   return MatchOperand_Success;
2293 }
2294 
2295 typedef std::map<enum AMDGPUOperand::ImmTy, unsigned> OptionalImmIndexMap;
2296 
2297 static void addOptionalImmOperand(MCInst& Inst, const OperandVector& Operands,
2298                                   OptionalImmIndexMap& OptionalIdx,
2299                                   AMDGPUOperand::ImmTy ImmT,
2300                                   int64_t Default = 0) {
2301   auto i = OptionalIdx.find(ImmT);
2302   if (i != OptionalIdx.end()) {
2303     unsigned Idx = i->second;
2304     ((AMDGPUOperand &)*Operands[Idx]).addImmOperands(Inst, 1);
2305   } else {
2306     Inst.addOperand(MCOperand::createImm(Default));
2307   }
2308 }
2309 
2310 OperandMatchResultTy
2311 AMDGPUAsmParser::parseStringWithPrefix(StringRef Prefix, StringRef &Value) {
2312   if (getLexer().isNot(AsmToken::Identifier)) {
2313     return MatchOperand_NoMatch;
2314   }
2315   StringRef Tok = Parser.getTok().getString();
2316   if (Tok != Prefix) {
2317     return MatchOperand_NoMatch;
2318   }
2319 
2320   Parser.Lex();
2321   if (getLexer().isNot(AsmToken::Colon)) {
2322     return MatchOperand_ParseFail;
2323   }
2324 
2325   Parser.Lex();
2326   if (getLexer().isNot(AsmToken::Identifier)) {
2327     return MatchOperand_ParseFail;
2328   }
2329 
2330   Value = Parser.getTok().getString();
2331   return MatchOperand_Success;
2332 }
2333 
2334 //===----------------------------------------------------------------------===//
2335 // ds
2336 //===----------------------------------------------------------------------===//
2337 
2338 void AMDGPUAsmParser::cvtDSOffset01(MCInst &Inst,
2339                                     const OperandVector &Operands) {
2340   OptionalImmIndexMap OptionalIdx;
2341 
2342   for (unsigned i = 1, e = Operands.size(); i != e; ++i) {
2343     AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[i]);
2344 
2345     // Add the register arguments
2346     if (Op.isReg()) {
2347       Op.addRegOperands(Inst, 1);
2348       continue;
2349     }
2350 
2351     // Handle optional arguments
2352     OptionalIdx[Op.getImmTy()] = i;
2353   }
2354 
2355   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyOffset0);
2356   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyOffset1);
2357   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyGDS);
2358 
2359   Inst.addOperand(MCOperand::createReg(AMDGPU::M0)); // m0
2360 }
2361 
2362 void AMDGPUAsmParser::cvtDSImpl(MCInst &Inst, const OperandVector &Operands,
2363                                 bool IsGdsHardcoded) {
2364   OptionalImmIndexMap OptionalIdx;
2365 
2366   for (unsigned i = 1, e = Operands.size(); i != e; ++i) {
2367     AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[i]);
2368 
2369     // Add the register arguments
2370     if (Op.isReg()) {
2371       Op.addRegOperands(Inst, 1);
2372       continue;
2373     }
2374 
2375     if (Op.isToken() && Op.getToken() == "gds") {
2376       IsGdsHardcoded = true;
2377       continue;
2378     }
2379 
2380     // Handle optional arguments
2381     OptionalIdx[Op.getImmTy()] = i;
2382   }
2383 
2384   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyOffset);
2385   if (!IsGdsHardcoded) {
2386     addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyGDS);
2387   }
2388   Inst.addOperand(MCOperand::createReg(AMDGPU::M0)); // m0
2389 }
2390 
2391 void AMDGPUAsmParser::cvtExp(MCInst &Inst, const OperandVector &Operands) {
2392   OptionalImmIndexMap OptionalIdx;
2393 
2394   unsigned EnMask = 0;
2395   int SrcIdx = 0;
2396 
2397   for (unsigned i = 1, e = Operands.size(); i != e; ++i) {
2398     AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[i]);
2399 
2400     // Add the register arguments
2401     if (Op.isReg()) {
2402       EnMask |= (1 << SrcIdx);
2403       Op.addRegOperands(Inst, 1);
2404       ++SrcIdx;
2405       continue;
2406     }
2407 
2408     if (Op.isOff()) {
2409       ++SrcIdx;
2410       Inst.addOperand(MCOperand::createReg(AMDGPU::NoRegister));
2411       continue;
2412     }
2413 
2414     if (Op.isImm() && Op.getImmTy() == AMDGPUOperand::ImmTyExpTgt) {
2415       Op.addImmOperands(Inst, 1);
2416       continue;
2417     }
2418 
2419     if (Op.isToken() && Op.getToken() == "done")
2420       continue;
2421 
2422     // Handle optional arguments
2423     OptionalIdx[Op.getImmTy()] = i;
2424   }
2425 
2426   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyExpVM);
2427   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyExpCompr);
2428 
2429   Inst.addOperand(MCOperand::createImm(EnMask));
2430 }
2431 
2432 //===----------------------------------------------------------------------===//
2433 // s_waitcnt
2434 //===----------------------------------------------------------------------===//
2435 
2436 bool AMDGPUAsmParser::parseCnt(int64_t &IntVal) {
2437   StringRef CntName = Parser.getTok().getString();
2438   int64_t CntVal;
2439 
2440   Parser.Lex();
2441   if (getLexer().isNot(AsmToken::LParen))
2442     return true;
2443 
2444   Parser.Lex();
2445   if (getLexer().isNot(AsmToken::Integer))
2446     return true;
2447 
2448   if (getParser().parseAbsoluteExpression(CntVal))
2449     return true;
2450 
2451   if (getLexer().isNot(AsmToken::RParen))
2452     return true;
2453 
2454   Parser.Lex();
2455   if (getLexer().is(AsmToken::Amp) || getLexer().is(AsmToken::Comma))
2456     Parser.Lex();
2457 
2458   IsaVersion IV = getIsaVersion(getSTI().getFeatureBits());
2459   if (CntName == "vmcnt")
2460     IntVal = encodeVmcnt(IV, IntVal, CntVal);
2461   else if (CntName == "expcnt")
2462     IntVal = encodeExpcnt(IV, IntVal, CntVal);
2463   else if (CntName == "lgkmcnt")
2464     IntVal = encodeLgkmcnt(IV, IntVal, CntVal);
2465   else
2466     return true;
2467 
2468   return false;
2469 }
2470 
2471 OperandMatchResultTy
2472 AMDGPUAsmParser::parseSWaitCntOps(OperandVector &Operands) {
2473   IsaVersion IV = getIsaVersion(getSTI().getFeatureBits());
2474   int64_t Waitcnt = getWaitcntBitMask(IV);
2475   SMLoc S = Parser.getTok().getLoc();
2476 
2477   switch(getLexer().getKind()) {
2478     default: return MatchOperand_ParseFail;
2479     case AsmToken::Integer:
2480       // The operand can be an integer value.
2481       if (getParser().parseAbsoluteExpression(Waitcnt))
2482         return MatchOperand_ParseFail;
2483       break;
2484 
2485     case AsmToken::Identifier:
2486       do {
2487         if (parseCnt(Waitcnt))
2488           return MatchOperand_ParseFail;
2489       } while(getLexer().isNot(AsmToken::EndOfStatement));
2490       break;
2491   }
2492   Operands.push_back(AMDGPUOperand::CreateImm(this, Waitcnt, S));
2493   return MatchOperand_Success;
2494 }
2495 
2496 bool AMDGPUAsmParser::parseHwregConstruct(OperandInfoTy &HwReg, int64_t &Offset,
2497                                           int64_t &Width) {
2498   using namespace llvm::AMDGPU::Hwreg;
2499 
2500   if (Parser.getTok().getString() != "hwreg")
2501     return true;
2502   Parser.Lex();
2503 
2504   if (getLexer().isNot(AsmToken::LParen))
2505     return true;
2506   Parser.Lex();
2507 
2508   if (getLexer().is(AsmToken::Identifier)) {
2509     HwReg.IsSymbolic = true;
2510     HwReg.Id = ID_UNKNOWN_;
2511     const StringRef tok = Parser.getTok().getString();
2512     for (int i = ID_SYMBOLIC_FIRST_; i < ID_SYMBOLIC_LAST_; ++i) {
2513       if (tok == IdSymbolic[i]) {
2514         HwReg.Id = i;
2515         break;
2516       }
2517     }
2518     Parser.Lex();
2519   } else {
2520     HwReg.IsSymbolic = false;
2521     if (getLexer().isNot(AsmToken::Integer))
2522       return true;
2523     if (getParser().parseAbsoluteExpression(HwReg.Id))
2524       return true;
2525   }
2526 
2527   if (getLexer().is(AsmToken::RParen)) {
2528     Parser.Lex();
2529     return false;
2530   }
2531 
2532   // optional params
2533   if (getLexer().isNot(AsmToken::Comma))
2534     return true;
2535   Parser.Lex();
2536 
2537   if (getLexer().isNot(AsmToken::Integer))
2538     return true;
2539   if (getParser().parseAbsoluteExpression(Offset))
2540     return true;
2541 
2542   if (getLexer().isNot(AsmToken::Comma))
2543     return true;
2544   Parser.Lex();
2545 
2546   if (getLexer().isNot(AsmToken::Integer))
2547     return true;
2548   if (getParser().parseAbsoluteExpression(Width))
2549     return true;
2550 
2551   if (getLexer().isNot(AsmToken::RParen))
2552     return true;
2553   Parser.Lex();
2554 
2555   return false;
2556 }
2557 
2558 OperandMatchResultTy AMDGPUAsmParser::parseHwreg(OperandVector &Operands) {
2559   using namespace llvm::AMDGPU::Hwreg;
2560 
2561   int64_t Imm16Val = 0;
2562   SMLoc S = Parser.getTok().getLoc();
2563 
2564   switch(getLexer().getKind()) {
2565     default: return MatchOperand_NoMatch;
2566     case AsmToken::Integer:
2567       // The operand can be an integer value.
2568       if (getParser().parseAbsoluteExpression(Imm16Val))
2569         return MatchOperand_NoMatch;
2570       if (Imm16Val < 0 || !isUInt<16>(Imm16Val)) {
2571         Error(S, "invalid immediate: only 16-bit values are legal");
2572         // Do not return error code, but create an imm operand anyway and proceed
2573         // to the next operand, if any. That avoids unneccessary error messages.
2574       }
2575       break;
2576 
2577     case AsmToken::Identifier: {
2578         OperandInfoTy HwReg(ID_UNKNOWN_);
2579         int64_t Offset = OFFSET_DEFAULT_;
2580         int64_t Width = WIDTH_M1_DEFAULT_ + 1;
2581         if (parseHwregConstruct(HwReg, Offset, Width))
2582           return MatchOperand_ParseFail;
2583         if (HwReg.Id < 0 || !isUInt<ID_WIDTH_>(HwReg.Id)) {
2584           if (HwReg.IsSymbolic)
2585             Error(S, "invalid symbolic name of hardware register");
2586           else
2587             Error(S, "invalid code of hardware register: only 6-bit values are legal");
2588         }
2589         if (Offset < 0 || !isUInt<OFFSET_WIDTH_>(Offset))
2590           Error(S, "invalid bit offset: only 5-bit values are legal");
2591         if ((Width-1) < 0 || !isUInt<WIDTH_M1_WIDTH_>(Width-1))
2592           Error(S, "invalid bitfield width: only values from 1 to 32 are legal");
2593         Imm16Val = (HwReg.Id << ID_SHIFT_) | (Offset << OFFSET_SHIFT_) | ((Width-1) << WIDTH_M1_SHIFT_);
2594       }
2595       break;
2596   }
2597   Operands.push_back(AMDGPUOperand::CreateImm(this, Imm16Val, S, AMDGPUOperand::ImmTyHwreg));
2598   return MatchOperand_Success;
2599 }
2600 
2601 bool AMDGPUOperand::isSWaitCnt() const {
2602   return isImm();
2603 }
2604 
2605 bool AMDGPUOperand::isHwreg() const {
2606   return isImmTy(ImmTyHwreg);
2607 }
2608 
2609 bool AMDGPUAsmParser::parseSendMsgConstruct(OperandInfoTy &Msg, OperandInfoTy &Operation, int64_t &StreamId) {
2610   using namespace llvm::AMDGPU::SendMsg;
2611 
2612   if (Parser.getTok().getString() != "sendmsg")
2613     return true;
2614   Parser.Lex();
2615 
2616   if (getLexer().isNot(AsmToken::LParen))
2617     return true;
2618   Parser.Lex();
2619 
2620   if (getLexer().is(AsmToken::Identifier)) {
2621     Msg.IsSymbolic = true;
2622     Msg.Id = ID_UNKNOWN_;
2623     const std::string tok = Parser.getTok().getString();
2624     for (int i = ID_GAPS_FIRST_; i < ID_GAPS_LAST_; ++i) {
2625       switch(i) {
2626         default: continue; // Omit gaps.
2627         case ID_INTERRUPT: case ID_GS: case ID_GS_DONE:  case ID_SYSMSG: break;
2628       }
2629       if (tok == IdSymbolic[i]) {
2630         Msg.Id = i;
2631         break;
2632       }
2633     }
2634     Parser.Lex();
2635   } else {
2636     Msg.IsSymbolic = false;
2637     if (getLexer().isNot(AsmToken::Integer))
2638       return true;
2639     if (getParser().parseAbsoluteExpression(Msg.Id))
2640       return true;
2641     if (getLexer().is(AsmToken::Integer))
2642       if (getParser().parseAbsoluteExpression(Msg.Id))
2643         Msg.Id = ID_UNKNOWN_;
2644   }
2645   if (Msg.Id == ID_UNKNOWN_) // Don't know how to parse the rest.
2646     return false;
2647 
2648   if (!(Msg.Id == ID_GS || Msg.Id == ID_GS_DONE || Msg.Id == ID_SYSMSG)) {
2649     if (getLexer().isNot(AsmToken::RParen))
2650       return true;
2651     Parser.Lex();
2652     return false;
2653   }
2654 
2655   if (getLexer().isNot(AsmToken::Comma))
2656     return true;
2657   Parser.Lex();
2658 
2659   assert(Msg.Id == ID_GS || Msg.Id == ID_GS_DONE || Msg.Id == ID_SYSMSG);
2660   Operation.Id = ID_UNKNOWN_;
2661   if (getLexer().is(AsmToken::Identifier)) {
2662     Operation.IsSymbolic = true;
2663     const char* const *S = (Msg.Id == ID_SYSMSG) ? OpSysSymbolic : OpGsSymbolic;
2664     const int F = (Msg.Id == ID_SYSMSG) ? OP_SYS_FIRST_ : OP_GS_FIRST_;
2665     const int L = (Msg.Id == ID_SYSMSG) ? OP_SYS_LAST_ : OP_GS_LAST_;
2666     const StringRef Tok = Parser.getTok().getString();
2667     for (int i = F; i < L; ++i) {
2668       if (Tok == S[i]) {
2669         Operation.Id = i;
2670         break;
2671       }
2672     }
2673     Parser.Lex();
2674   } else {
2675     Operation.IsSymbolic = false;
2676     if (getLexer().isNot(AsmToken::Integer))
2677       return true;
2678     if (getParser().parseAbsoluteExpression(Operation.Id))
2679       return true;
2680   }
2681 
2682   if ((Msg.Id == ID_GS || Msg.Id == ID_GS_DONE) && Operation.Id != OP_GS_NOP) {
2683     // Stream id is optional.
2684     if (getLexer().is(AsmToken::RParen)) {
2685       Parser.Lex();
2686       return false;
2687     }
2688 
2689     if (getLexer().isNot(AsmToken::Comma))
2690       return true;
2691     Parser.Lex();
2692 
2693     if (getLexer().isNot(AsmToken::Integer))
2694       return true;
2695     if (getParser().parseAbsoluteExpression(StreamId))
2696       return true;
2697   }
2698 
2699   if (getLexer().isNot(AsmToken::RParen))
2700     return true;
2701   Parser.Lex();
2702   return false;
2703 }
2704 
2705 OperandMatchResultTy AMDGPUAsmParser::parseInterpSlot(OperandVector &Operands) {
2706   if (getLexer().getKind() != AsmToken::Identifier)
2707     return MatchOperand_NoMatch;
2708 
2709   StringRef Str = Parser.getTok().getString();
2710   int Slot = StringSwitch<int>(Str)
2711     .Case("p10", 0)
2712     .Case("p20", 1)
2713     .Case("p0", 2)
2714     .Default(-1);
2715 
2716   SMLoc S = Parser.getTok().getLoc();
2717   if (Slot == -1)
2718     return MatchOperand_ParseFail;
2719 
2720   Parser.Lex();
2721   Operands.push_back(AMDGPUOperand::CreateImm(this, Slot, S,
2722                                               AMDGPUOperand::ImmTyInterpSlot));
2723   return MatchOperand_Success;
2724 }
2725 
2726 OperandMatchResultTy AMDGPUAsmParser::parseInterpAttr(OperandVector &Operands) {
2727   if (getLexer().getKind() != AsmToken::Identifier)
2728     return MatchOperand_NoMatch;
2729 
2730   StringRef Str = Parser.getTok().getString();
2731   if (!Str.startswith("attr"))
2732     return MatchOperand_NoMatch;
2733 
2734   StringRef Chan = Str.take_back(2);
2735   int AttrChan = StringSwitch<int>(Chan)
2736     .Case(".x", 0)
2737     .Case(".y", 1)
2738     .Case(".z", 2)
2739     .Case(".w", 3)
2740     .Default(-1);
2741   if (AttrChan == -1)
2742     return MatchOperand_ParseFail;
2743 
2744   Str = Str.drop_back(2).drop_front(4);
2745 
2746   uint8_t Attr;
2747   if (Str.getAsInteger(10, Attr))
2748     return MatchOperand_ParseFail;
2749 
2750   SMLoc S = Parser.getTok().getLoc();
2751   Parser.Lex();
2752   if (Attr > 63) {
2753     Error(S, "out of bounds attr");
2754     return MatchOperand_Success;
2755   }
2756 
2757   SMLoc SChan = SMLoc::getFromPointer(Chan.data());
2758 
2759   Operands.push_back(AMDGPUOperand::CreateImm(this, Attr, S,
2760                                               AMDGPUOperand::ImmTyInterpAttr));
2761   Operands.push_back(AMDGPUOperand::CreateImm(this, AttrChan, SChan,
2762                                               AMDGPUOperand::ImmTyAttrChan));
2763   return MatchOperand_Success;
2764 }
2765 
2766 void AMDGPUAsmParser::errorExpTgt() {
2767   Error(Parser.getTok().getLoc(), "invalid exp target");
2768 }
2769 
2770 OperandMatchResultTy AMDGPUAsmParser::parseExpTgtImpl(StringRef Str,
2771                                                       uint8_t &Val) {
2772   if (Str == "null") {
2773     Val = 9;
2774     return MatchOperand_Success;
2775   }
2776 
2777   if (Str.startswith("mrt")) {
2778     Str = Str.drop_front(3);
2779     if (Str == "z") { // == mrtz
2780       Val = 8;
2781       return MatchOperand_Success;
2782     }
2783 
2784     if (Str.getAsInteger(10, Val))
2785       return MatchOperand_ParseFail;
2786 
2787     if (Val > 7)
2788       errorExpTgt();
2789 
2790     return MatchOperand_Success;
2791   }
2792 
2793   if (Str.startswith("pos")) {
2794     Str = Str.drop_front(3);
2795     if (Str.getAsInteger(10, Val))
2796       return MatchOperand_ParseFail;
2797 
2798     if (Val > 3)
2799       errorExpTgt();
2800 
2801     Val += 12;
2802     return MatchOperand_Success;
2803   }
2804 
2805   if (Str.startswith("param")) {
2806     Str = Str.drop_front(5);
2807     if (Str.getAsInteger(10, Val))
2808       return MatchOperand_ParseFail;
2809 
2810     if (Val >= 32)
2811       errorExpTgt();
2812 
2813     Val += 32;
2814     return MatchOperand_Success;
2815   }
2816 
2817   if (Str.startswith("invalid_target_")) {
2818     Str = Str.drop_front(15);
2819     if (Str.getAsInteger(10, Val))
2820       return MatchOperand_ParseFail;
2821 
2822     errorExpTgt();
2823     return MatchOperand_Success;
2824   }
2825 
2826   return MatchOperand_NoMatch;
2827 }
2828 
2829 OperandMatchResultTy AMDGPUAsmParser::parseExpTgt(OperandVector &Operands) {
2830   uint8_t Val;
2831   StringRef Str = Parser.getTok().getString();
2832 
2833   auto Res = parseExpTgtImpl(Str, Val);
2834   if (Res != MatchOperand_Success)
2835     return Res;
2836 
2837   SMLoc S = Parser.getTok().getLoc();
2838   Parser.Lex();
2839 
2840   Operands.push_back(AMDGPUOperand::CreateImm(this, Val, S,
2841                                               AMDGPUOperand::ImmTyExpTgt));
2842   return MatchOperand_Success;
2843 }
2844 
2845 OperandMatchResultTy
2846 AMDGPUAsmParser::parseSendMsgOp(OperandVector &Operands) {
2847   using namespace llvm::AMDGPU::SendMsg;
2848 
2849   int64_t Imm16Val = 0;
2850   SMLoc S = Parser.getTok().getLoc();
2851 
2852   switch(getLexer().getKind()) {
2853   default:
2854     return MatchOperand_NoMatch;
2855   case AsmToken::Integer:
2856     // The operand can be an integer value.
2857     if (getParser().parseAbsoluteExpression(Imm16Val))
2858       return MatchOperand_NoMatch;
2859     if (Imm16Val < 0 || !isUInt<16>(Imm16Val)) {
2860       Error(S, "invalid immediate: only 16-bit values are legal");
2861       // Do not return error code, but create an imm operand anyway and proceed
2862       // to the next operand, if any. That avoids unneccessary error messages.
2863     }
2864     break;
2865   case AsmToken::Identifier: {
2866       OperandInfoTy Msg(ID_UNKNOWN_);
2867       OperandInfoTy Operation(OP_UNKNOWN_);
2868       int64_t StreamId = STREAM_ID_DEFAULT_;
2869       if (parseSendMsgConstruct(Msg, Operation, StreamId))
2870         return MatchOperand_ParseFail;
2871       do {
2872         // Validate and encode message ID.
2873         if (! ((ID_INTERRUPT <= Msg.Id && Msg.Id <= ID_GS_DONE)
2874                 || Msg.Id == ID_SYSMSG)) {
2875           if (Msg.IsSymbolic)
2876             Error(S, "invalid/unsupported symbolic name of message");
2877           else
2878             Error(S, "invalid/unsupported code of message");
2879           break;
2880         }
2881         Imm16Val = (Msg.Id << ID_SHIFT_);
2882         // Validate and encode operation ID.
2883         if (Msg.Id == ID_GS || Msg.Id == ID_GS_DONE) {
2884           if (! (OP_GS_FIRST_ <= Operation.Id && Operation.Id < OP_GS_LAST_)) {
2885             if (Operation.IsSymbolic)
2886               Error(S, "invalid symbolic name of GS_OP");
2887             else
2888               Error(S, "invalid code of GS_OP: only 2-bit values are legal");
2889             break;
2890           }
2891           if (Operation.Id == OP_GS_NOP
2892               && Msg.Id != ID_GS_DONE) {
2893             Error(S, "invalid GS_OP: NOP is for GS_DONE only");
2894             break;
2895           }
2896           Imm16Val |= (Operation.Id << OP_SHIFT_);
2897         }
2898         if (Msg.Id == ID_SYSMSG) {
2899           if (! (OP_SYS_FIRST_ <= Operation.Id && Operation.Id < OP_SYS_LAST_)) {
2900             if (Operation.IsSymbolic)
2901               Error(S, "invalid/unsupported symbolic name of SYSMSG_OP");
2902             else
2903               Error(S, "invalid/unsupported code of SYSMSG_OP");
2904             break;
2905           }
2906           Imm16Val |= (Operation.Id << OP_SHIFT_);
2907         }
2908         // Validate and encode stream ID.
2909         if ((Msg.Id == ID_GS || Msg.Id == ID_GS_DONE) && Operation.Id != OP_GS_NOP) {
2910           if (! (STREAM_ID_FIRST_ <= StreamId && StreamId < STREAM_ID_LAST_)) {
2911             Error(S, "invalid stream id: only 2-bit values are legal");
2912             break;
2913           }
2914           Imm16Val |= (StreamId << STREAM_ID_SHIFT_);
2915         }
2916       } while (false);
2917     }
2918     break;
2919   }
2920   Operands.push_back(AMDGPUOperand::CreateImm(this, Imm16Val, S, AMDGPUOperand::ImmTySendMsg));
2921   return MatchOperand_Success;
2922 }
2923 
2924 bool AMDGPUOperand::isSendMsg() const {
2925   return isImmTy(ImmTySendMsg);
2926 }
2927 
2928 //===----------------------------------------------------------------------===//
2929 // sopp branch targets
2930 //===----------------------------------------------------------------------===//
2931 
2932 OperandMatchResultTy
2933 AMDGPUAsmParser::parseSOppBrTarget(OperandVector &Operands) {
2934   SMLoc S = Parser.getTok().getLoc();
2935 
2936   switch (getLexer().getKind()) {
2937     default: return MatchOperand_ParseFail;
2938     case AsmToken::Integer: {
2939       int64_t Imm;
2940       if (getParser().parseAbsoluteExpression(Imm))
2941         return MatchOperand_ParseFail;
2942       Operands.push_back(AMDGPUOperand::CreateImm(this, Imm, S));
2943       return MatchOperand_Success;
2944     }
2945 
2946     case AsmToken::Identifier:
2947       Operands.push_back(AMDGPUOperand::CreateExpr(this,
2948           MCSymbolRefExpr::create(getContext().getOrCreateSymbol(
2949                                   Parser.getTok().getString()), getContext()), S));
2950       Parser.Lex();
2951       return MatchOperand_Success;
2952   }
2953 }
2954 
2955 //===----------------------------------------------------------------------===//
2956 // mubuf
2957 //===----------------------------------------------------------------------===//
2958 
2959 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultGLC() const {
2960   return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTyGLC);
2961 }
2962 
2963 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultSLC() const {
2964   return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTySLC);
2965 }
2966 
2967 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultTFE() const {
2968   return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTyTFE);
2969 }
2970 
2971 void AMDGPUAsmParser::cvtMubufImpl(MCInst &Inst,
2972                                const OperandVector &Operands,
2973                                bool IsAtomic, bool IsAtomicReturn) {
2974   OptionalImmIndexMap OptionalIdx;
2975   assert(IsAtomicReturn ? IsAtomic : true);
2976 
2977   for (unsigned i = 1, e = Operands.size(); i != e; ++i) {
2978     AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[i]);
2979 
2980     // Add the register arguments
2981     if (Op.isReg()) {
2982       Op.addRegOperands(Inst, 1);
2983       continue;
2984     }
2985 
2986     // Handle the case where soffset is an immediate
2987     if (Op.isImm() && Op.getImmTy() == AMDGPUOperand::ImmTyNone) {
2988       Op.addImmOperands(Inst, 1);
2989       continue;
2990     }
2991 
2992     // Handle tokens like 'offen' which are sometimes hard-coded into the
2993     // asm string.  There are no MCInst operands for these.
2994     if (Op.isToken()) {
2995       continue;
2996     }
2997     assert(Op.isImm());
2998 
2999     // Handle optional arguments
3000     OptionalIdx[Op.getImmTy()] = i;
3001   }
3002 
3003   // Copy $vdata_in operand and insert as $vdata for MUBUF_Atomic RTN insns.
3004   if (IsAtomicReturn) {
3005     MCInst::iterator I = Inst.begin(); // $vdata_in is always at the beginning.
3006     Inst.insert(I, *I);
3007   }
3008 
3009   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyOffset);
3010   if (!IsAtomic) { // glc is hard-coded.
3011     addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyGLC);
3012   }
3013   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySLC);
3014   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyTFE);
3015 }
3016 
3017 //===----------------------------------------------------------------------===//
3018 // mimg
3019 //===----------------------------------------------------------------------===//
3020 
3021 void AMDGPUAsmParser::cvtMIMG(MCInst &Inst, const OperandVector &Operands) {
3022   unsigned I = 1;
3023   const MCInstrDesc &Desc = MII.get(Inst.getOpcode());
3024   for (unsigned J = 0; J < Desc.getNumDefs(); ++J) {
3025     ((AMDGPUOperand &)*Operands[I++]).addRegOperands(Inst, 1);
3026   }
3027 
3028   OptionalImmIndexMap OptionalIdx;
3029 
3030   for (unsigned E = Operands.size(); I != E; ++I) {
3031     AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[I]);
3032 
3033     // Add the register arguments
3034     if (Op.isRegOrImm()) {
3035       Op.addRegOrImmOperands(Inst, 1);
3036       continue;
3037     } else if (Op.isImmModifier()) {
3038       OptionalIdx[Op.getImmTy()] = I;
3039     } else {
3040       llvm_unreachable("unexpected operand type");
3041     }
3042   }
3043 
3044   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyDMask);
3045   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyUNorm);
3046   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyGLC);
3047   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyDA);
3048   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyR128);
3049   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyTFE);
3050   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyLWE);
3051   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySLC);
3052 }
3053 
3054 void AMDGPUAsmParser::cvtMIMGAtomic(MCInst &Inst, const OperandVector &Operands) {
3055   unsigned I = 1;
3056   const MCInstrDesc &Desc = MII.get(Inst.getOpcode());
3057   for (unsigned J = 0; J < Desc.getNumDefs(); ++J) {
3058     ((AMDGPUOperand &)*Operands[I++]).addRegOperands(Inst, 1);
3059   }
3060 
3061   // Add src, same as dst
3062   ((AMDGPUOperand &)*Operands[I]).addRegOperands(Inst, 1);
3063 
3064   OptionalImmIndexMap OptionalIdx;
3065 
3066   for (unsigned E = Operands.size(); I != E; ++I) {
3067     AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[I]);
3068 
3069     // Add the register arguments
3070     if (Op.isRegOrImm()) {
3071       Op.addRegOrImmOperands(Inst, 1);
3072       continue;
3073     } else if (Op.isImmModifier()) {
3074       OptionalIdx[Op.getImmTy()] = I;
3075     } else {
3076       llvm_unreachable("unexpected operand type");
3077     }
3078   }
3079 
3080   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyDMask);
3081   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyUNorm);
3082   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyGLC);
3083   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyDA);
3084   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyR128);
3085   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyTFE);
3086   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyLWE);
3087   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySLC);
3088 }
3089 
3090 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultDMask() const {
3091   return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTyDMask);
3092 }
3093 
3094 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultUNorm() const {
3095   return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTyUNorm);
3096 }
3097 
3098 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultDA() const {
3099   return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTyDA);
3100 }
3101 
3102 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultR128() const {
3103   return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTyR128);
3104 }
3105 
3106 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultLWE() const {
3107   return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTyLWE);
3108 }
3109 
3110 //===----------------------------------------------------------------------===//
3111 // smrd
3112 //===----------------------------------------------------------------------===//
3113 
3114 bool AMDGPUOperand::isSMRDOffset8() const {
3115   return isImm() && isUInt<8>(getImm());
3116 }
3117 
3118 bool AMDGPUOperand::isSMRDOffset20() const {
3119   return isImm() && isUInt<20>(getImm());
3120 }
3121 
3122 bool AMDGPUOperand::isSMRDLiteralOffset() const {
3123   // 32-bit literals are only supported on CI and we only want to use them
3124   // when the offset is > 8-bits.
3125   return isImm() && !isUInt<8>(getImm()) && isUInt<32>(getImm());
3126 }
3127 
3128 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultSMRDOffset8() const {
3129   return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTyOffset);
3130 }
3131 
3132 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultSMRDOffset20() const {
3133   return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTyOffset);
3134 }
3135 
3136 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultSMRDLiteralOffset() const {
3137   return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTyOffset);
3138 }
3139 
3140 //===----------------------------------------------------------------------===//
3141 // vop3
3142 //===----------------------------------------------------------------------===//
3143 
3144 static bool ConvertOmodMul(int64_t &Mul) {
3145   if (Mul != 1 && Mul != 2 && Mul != 4)
3146     return false;
3147 
3148   Mul >>= 1;
3149   return true;
3150 }
3151 
3152 static bool ConvertOmodDiv(int64_t &Div) {
3153   if (Div == 1) {
3154     Div = 0;
3155     return true;
3156   }
3157 
3158   if (Div == 2) {
3159     Div = 3;
3160     return true;
3161   }
3162 
3163   return false;
3164 }
3165 
3166 static bool ConvertBoundCtrl(int64_t &BoundCtrl) {
3167   if (BoundCtrl == 0) {
3168     BoundCtrl = 1;
3169     return true;
3170   }
3171 
3172   if (BoundCtrl == -1) {
3173     BoundCtrl = 0;
3174     return true;
3175   }
3176 
3177   return false;
3178 }
3179 
3180 // Note: the order in this table matches the order of operands in AsmString.
3181 static const OptionalOperand AMDGPUOptionalOperandTable[] = {
3182   {"offen",   AMDGPUOperand::ImmTyOffen, true, nullptr},
3183   {"idxen",   AMDGPUOperand::ImmTyIdxen, true, nullptr},
3184   {"addr64",  AMDGPUOperand::ImmTyAddr64, true, nullptr},
3185   {"offset0", AMDGPUOperand::ImmTyOffset0, false, nullptr},
3186   {"offset1", AMDGPUOperand::ImmTyOffset1, false, nullptr},
3187   {"gds",     AMDGPUOperand::ImmTyGDS, true, nullptr},
3188   {"offset",  AMDGPUOperand::ImmTyOffset, false, nullptr},
3189   {"glc",     AMDGPUOperand::ImmTyGLC, true, nullptr},
3190   {"slc",     AMDGPUOperand::ImmTySLC, true, nullptr},
3191   {"tfe",     AMDGPUOperand::ImmTyTFE, true, nullptr},
3192   {"clamp",   AMDGPUOperand::ImmTyClampSI, true, nullptr},
3193   {"omod",    AMDGPUOperand::ImmTyOModSI, false, ConvertOmodMul},
3194   {"unorm",   AMDGPUOperand::ImmTyUNorm, true, nullptr},
3195   {"da",      AMDGPUOperand::ImmTyDA,    true, nullptr},
3196   {"r128",    AMDGPUOperand::ImmTyR128,  true, nullptr},
3197   {"lwe",     AMDGPUOperand::ImmTyLWE,   true, nullptr},
3198   {"dmask",   AMDGPUOperand::ImmTyDMask, false, nullptr},
3199   {"row_mask",   AMDGPUOperand::ImmTyDppRowMask, false, nullptr},
3200   {"bank_mask",  AMDGPUOperand::ImmTyDppBankMask, false, nullptr},
3201   {"bound_ctrl", AMDGPUOperand::ImmTyDppBoundCtrl, false, ConvertBoundCtrl},
3202   {"dst_sel",    AMDGPUOperand::ImmTySdwaDstSel, false, nullptr},
3203   {"src0_sel",   AMDGPUOperand::ImmTySdwaSrc0Sel, false, nullptr},
3204   {"src1_sel",   AMDGPUOperand::ImmTySdwaSrc1Sel, false, nullptr},
3205   {"dst_unused", AMDGPUOperand::ImmTySdwaDstUnused, false, nullptr},
3206   {"vm", AMDGPUOperand::ImmTyExpVM, true, nullptr},
3207 };
3208 
3209 OperandMatchResultTy AMDGPUAsmParser::parseOptionalOperand(OperandVector &Operands) {
3210   OperandMatchResultTy res;
3211   for (const OptionalOperand &Op : AMDGPUOptionalOperandTable) {
3212     // try to parse any optional operand here
3213     if (Op.IsBit) {
3214       res = parseNamedBit(Op.Name, Operands, Op.Type);
3215     } else if (Op.Type == AMDGPUOperand::ImmTyOModSI) {
3216       res = parseOModOperand(Operands);
3217     } else if (Op.Type == AMDGPUOperand::ImmTySdwaDstSel ||
3218                Op.Type == AMDGPUOperand::ImmTySdwaSrc0Sel ||
3219                Op.Type == AMDGPUOperand::ImmTySdwaSrc1Sel) {
3220       res = parseSDWASel(Operands, Op.Name, Op.Type);
3221     } else if (Op.Type == AMDGPUOperand::ImmTySdwaDstUnused) {
3222       res = parseSDWADstUnused(Operands);
3223     } else {
3224       res = parseIntWithPrefix(Op.Name, Operands, Op.Type, Op.ConvertResult);
3225     }
3226     if (res != MatchOperand_NoMatch) {
3227       return res;
3228     }
3229   }
3230   return MatchOperand_NoMatch;
3231 }
3232 
3233 OperandMatchResultTy AMDGPUAsmParser::parseOModOperand(OperandVector &Operands) {
3234   StringRef Name = Parser.getTok().getString();
3235   if (Name == "mul") {
3236     return parseIntWithPrefix("mul", Operands,
3237                               AMDGPUOperand::ImmTyOModSI, ConvertOmodMul);
3238   }
3239 
3240   if (Name == "div") {
3241     return parseIntWithPrefix("div", Operands,
3242                               AMDGPUOperand::ImmTyOModSI, ConvertOmodDiv);
3243   }
3244 
3245   return MatchOperand_NoMatch;
3246 }
3247 
3248 void AMDGPUAsmParser::cvtId(MCInst &Inst, const OperandVector &Operands) {
3249   unsigned I = 1;
3250   const MCInstrDesc &Desc = MII.get(Inst.getOpcode());
3251   for (unsigned J = 0; J < Desc.getNumDefs(); ++J) {
3252     ((AMDGPUOperand &)*Operands[I++]).addRegOperands(Inst, 1);
3253   }
3254   for (unsigned E = Operands.size(); I != E; ++I)
3255     ((AMDGPUOperand &)*Operands[I]).addRegOrImmOperands(Inst, 1);
3256 }
3257 
3258 void AMDGPUAsmParser::cvtVOP3_2_mod(MCInst &Inst, const OperandVector &Operands) {
3259   uint64_t TSFlags = MII.get(Inst.getOpcode()).TSFlags;
3260   if (TSFlags & SIInstrFlags::VOP3) {
3261     cvtVOP3(Inst, Operands);
3262   } else {
3263     cvtId(Inst, Operands);
3264   }
3265 }
3266 
3267 static bool isRegOrImmWithInputMods(const MCInstrDesc &Desc, unsigned OpNum) {
3268       // 1. This operand is input modifiers
3269   return Desc.OpInfo[OpNum].OperandType == AMDGPU::OPERAND_INPUT_MODS
3270       // 2. This is not last operand
3271       && Desc.NumOperands > (OpNum + 1)
3272       // 3. Next operand is register class
3273       && Desc.OpInfo[OpNum + 1].RegClass != -1
3274       // 4. Next register is not tied to any other operand
3275       && Desc.getOperandConstraint(OpNum + 1, MCOI::OperandConstraint::TIED_TO) == -1;
3276 }
3277 
3278 void AMDGPUAsmParser::cvtVOP3(MCInst &Inst, const OperandVector &Operands) {
3279   OptionalImmIndexMap OptionalIdx;
3280   unsigned I = 1;
3281   const MCInstrDesc &Desc = MII.get(Inst.getOpcode());
3282   for (unsigned J = 0; J < Desc.getNumDefs(); ++J) {
3283     ((AMDGPUOperand &)*Operands[I++]).addRegOperands(Inst, 1);
3284   }
3285 
3286   for (unsigned E = Operands.size(); I != E; ++I) {
3287     AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[I]);
3288     if (isRegOrImmWithInputMods(Desc, Inst.getNumOperands())) {
3289       Op.addRegOrImmWithFPInputModsOperands(Inst, 2);
3290     } else if (Op.isImm()) {
3291       OptionalIdx[Op.getImmTy()] = I;
3292     } else {
3293       llvm_unreachable("unhandled operand type");
3294     }
3295   }
3296 
3297   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyClampSI);
3298   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyOModSI);
3299 
3300   // special case v_mac_{f16, f32}:
3301   // it has src2 register operand that is tied to dst operand
3302   // we don't allow modifiers for this operand in assembler so src2_modifiers
3303   // should be 0
3304   if (Inst.getOpcode() == AMDGPU::V_MAC_F32_e64_si ||
3305       Inst.getOpcode() == AMDGPU::V_MAC_F32_e64_vi ||
3306       Inst.getOpcode() == AMDGPU::V_MAC_F16_e64_vi) {
3307     auto it = Inst.begin();
3308     std::advance(
3309       it,
3310       AMDGPU::getNamedOperandIdx(Inst.getOpcode() == AMDGPU::V_MAC_F16_e64_vi ?
3311                                      AMDGPU::V_MAC_F16_e64 :
3312                                      AMDGPU::V_MAC_F32_e64,
3313                                  AMDGPU::OpName::src2_modifiers));
3314     it = Inst.insert(it, MCOperand::createImm(0)); // no modifiers for src2
3315     ++it;
3316     Inst.insert(it, Inst.getOperand(0)); // src2 = dst
3317   }
3318 }
3319 
3320 //===----------------------------------------------------------------------===//
3321 // dpp
3322 //===----------------------------------------------------------------------===//
3323 
3324 bool AMDGPUOperand::isDPPCtrl() const {
3325   bool result = isImm() && getImmTy() == ImmTyDppCtrl && isUInt<9>(getImm());
3326   if (result) {
3327     int64_t Imm = getImm();
3328     return ((Imm >= 0x000) && (Imm <= 0x0ff)) ||
3329            ((Imm >= 0x101) && (Imm <= 0x10f)) ||
3330            ((Imm >= 0x111) && (Imm <= 0x11f)) ||
3331            ((Imm >= 0x121) && (Imm <= 0x12f)) ||
3332            (Imm == 0x130) ||
3333            (Imm == 0x134) ||
3334            (Imm == 0x138) ||
3335            (Imm == 0x13c) ||
3336            (Imm == 0x140) ||
3337            (Imm == 0x141) ||
3338            (Imm == 0x142) ||
3339            (Imm == 0x143);
3340   }
3341   return false;
3342 }
3343 
3344 bool AMDGPUOperand::isGPRIdxMode() const {
3345   return isImm() && isUInt<4>(getImm());
3346 }
3347 
3348 OperandMatchResultTy
3349 AMDGPUAsmParser::parseDPPCtrl(OperandVector &Operands) {
3350   SMLoc S = Parser.getTok().getLoc();
3351   StringRef Prefix;
3352   int64_t Int;
3353 
3354   if (getLexer().getKind() == AsmToken::Identifier) {
3355     Prefix = Parser.getTok().getString();
3356   } else {
3357     return MatchOperand_NoMatch;
3358   }
3359 
3360   if (Prefix == "row_mirror") {
3361     Int = 0x140;
3362     Parser.Lex();
3363   } else if (Prefix == "row_half_mirror") {
3364     Int = 0x141;
3365     Parser.Lex();
3366   } else {
3367     // Check to prevent parseDPPCtrlOps from eating invalid tokens
3368     if (Prefix != "quad_perm"
3369         && Prefix != "row_shl"
3370         && Prefix != "row_shr"
3371         && Prefix != "row_ror"
3372         && Prefix != "wave_shl"
3373         && Prefix != "wave_rol"
3374         && Prefix != "wave_shr"
3375         && Prefix != "wave_ror"
3376         && Prefix != "row_bcast") {
3377       return MatchOperand_NoMatch;
3378     }
3379 
3380     Parser.Lex();
3381     if (getLexer().isNot(AsmToken::Colon))
3382       return MatchOperand_ParseFail;
3383 
3384     if (Prefix == "quad_perm") {
3385       // quad_perm:[%d,%d,%d,%d]
3386       Parser.Lex();
3387       if (getLexer().isNot(AsmToken::LBrac))
3388         return MatchOperand_ParseFail;
3389       Parser.Lex();
3390 
3391       if (getParser().parseAbsoluteExpression(Int) || !(0 <= Int && Int <=3))
3392         return MatchOperand_ParseFail;
3393 
3394       for (int i = 0; i < 3; ++i) {
3395         if (getLexer().isNot(AsmToken::Comma))
3396           return MatchOperand_ParseFail;
3397         Parser.Lex();
3398 
3399         int64_t Temp;
3400         if (getParser().parseAbsoluteExpression(Temp) || !(0 <= Temp && Temp <=3))
3401           return MatchOperand_ParseFail;
3402         const int shift = i*2 + 2;
3403         Int += (Temp << shift);
3404       }
3405 
3406       if (getLexer().isNot(AsmToken::RBrac))
3407         return MatchOperand_ParseFail;
3408       Parser.Lex();
3409 
3410     } else {
3411       // sel:%d
3412       Parser.Lex();
3413       if (getParser().parseAbsoluteExpression(Int))
3414         return MatchOperand_ParseFail;
3415 
3416       if (Prefix == "row_shl" && 1 <= Int && Int <= 15) {
3417         Int |= 0x100;
3418       } else if (Prefix == "row_shr" && 1 <= Int && Int <= 15) {
3419         Int |= 0x110;
3420       } else if (Prefix == "row_ror" && 1 <= Int && Int <= 15) {
3421         Int |= 0x120;
3422       } else if (Prefix == "wave_shl" && 1 == Int) {
3423         Int = 0x130;
3424       } else if (Prefix == "wave_rol" && 1 == Int) {
3425         Int = 0x134;
3426       } else if (Prefix == "wave_shr" && 1 == Int) {
3427         Int = 0x138;
3428       } else if (Prefix == "wave_ror" && 1 == Int) {
3429         Int = 0x13C;
3430       } else if (Prefix == "row_bcast") {
3431         if (Int == 15) {
3432           Int = 0x142;
3433         } else if (Int == 31) {
3434           Int = 0x143;
3435         } else {
3436           return MatchOperand_ParseFail;
3437         }
3438       } else {
3439         return MatchOperand_ParseFail;
3440       }
3441     }
3442   }
3443 
3444   Operands.push_back(AMDGPUOperand::CreateImm(this, Int, S, AMDGPUOperand::ImmTyDppCtrl));
3445   return MatchOperand_Success;
3446 }
3447 
3448 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultRowMask() const {
3449   return AMDGPUOperand::CreateImm(this, 0xf, SMLoc(), AMDGPUOperand::ImmTyDppRowMask);
3450 }
3451 
3452 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultBankMask() const {
3453   return AMDGPUOperand::CreateImm(this, 0xf, SMLoc(), AMDGPUOperand::ImmTyDppBankMask);
3454 }
3455 
3456 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultBoundCtrl() const {
3457   return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTyDppBoundCtrl);
3458 }
3459 
3460 void AMDGPUAsmParser::cvtDPP(MCInst &Inst, const OperandVector &Operands) {
3461   OptionalImmIndexMap OptionalIdx;
3462 
3463   unsigned I = 1;
3464   const MCInstrDesc &Desc = MII.get(Inst.getOpcode());
3465   for (unsigned J = 0; J < Desc.getNumDefs(); ++J) {
3466     ((AMDGPUOperand &)*Operands[I++]).addRegOperands(Inst, 1);
3467   }
3468 
3469   for (unsigned E = Operands.size(); I != E; ++I) {
3470     AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[I]);
3471     // Add the register arguments
3472     if (Op.isReg() && Op.Reg.RegNo == AMDGPU::VCC) {
3473       // VOP2b (v_add_u32, v_sub_u32 ...) dpp use "vcc" token.
3474       // Skip it.
3475       continue;
3476     } if (isRegOrImmWithInputMods(Desc, Inst.getNumOperands())) {
3477       Op.addRegWithFPInputModsOperands(Inst, 2);
3478     } else if (Op.isDPPCtrl()) {
3479       Op.addImmOperands(Inst, 1);
3480     } else if (Op.isImm()) {
3481       // Handle optional arguments
3482       OptionalIdx[Op.getImmTy()] = I;
3483     } else {
3484       llvm_unreachable("Invalid operand type");
3485     }
3486   }
3487 
3488   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyDppRowMask, 0xf);
3489   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyDppBankMask, 0xf);
3490   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyDppBoundCtrl);
3491 
3492   // special case v_mac_{f16, f32}:
3493   // it has src2 register operand that is tied to dst operand
3494   if (Inst.getOpcode() == AMDGPU::V_MAC_F32_dpp ||
3495       Inst.getOpcode() == AMDGPU::V_MAC_F16_dpp) {
3496     auto it = Inst.begin();
3497     std::advance(
3498         it, AMDGPU::getNamedOperandIdx(Inst.getOpcode(), AMDGPU::OpName::src2));
3499     Inst.insert(it, Inst.getOperand(0)); // src2 = dst
3500   }
3501 }
3502 
3503 //===----------------------------------------------------------------------===//
3504 // sdwa
3505 //===----------------------------------------------------------------------===//
3506 
3507 OperandMatchResultTy
3508 AMDGPUAsmParser::parseSDWASel(OperandVector &Operands, StringRef Prefix,
3509                               AMDGPUOperand::ImmTy Type) {
3510   using namespace llvm::AMDGPU::SDWA;
3511 
3512   SMLoc S = Parser.getTok().getLoc();
3513   StringRef Value;
3514   OperandMatchResultTy res;
3515 
3516   res = parseStringWithPrefix(Prefix, Value);
3517   if (res != MatchOperand_Success) {
3518     return res;
3519   }
3520 
3521   int64_t Int;
3522   Int = StringSwitch<int64_t>(Value)
3523         .Case("BYTE_0", SdwaSel::BYTE_0)
3524         .Case("BYTE_1", SdwaSel::BYTE_1)
3525         .Case("BYTE_2", SdwaSel::BYTE_2)
3526         .Case("BYTE_3", SdwaSel::BYTE_3)
3527         .Case("WORD_0", SdwaSel::WORD_0)
3528         .Case("WORD_1", SdwaSel::WORD_1)
3529         .Case("DWORD", SdwaSel::DWORD)
3530         .Default(0xffffffff);
3531   Parser.Lex(); // eat last token
3532 
3533   if (Int == 0xffffffff) {
3534     return MatchOperand_ParseFail;
3535   }
3536 
3537   Operands.push_back(AMDGPUOperand::CreateImm(this, Int, S, Type));
3538   return MatchOperand_Success;
3539 }
3540 
3541 OperandMatchResultTy
3542 AMDGPUAsmParser::parseSDWADstUnused(OperandVector &Operands) {
3543   using namespace llvm::AMDGPU::SDWA;
3544 
3545   SMLoc S = Parser.getTok().getLoc();
3546   StringRef Value;
3547   OperandMatchResultTy res;
3548 
3549   res = parseStringWithPrefix("dst_unused", Value);
3550   if (res != MatchOperand_Success) {
3551     return res;
3552   }
3553 
3554   int64_t Int;
3555   Int = StringSwitch<int64_t>(Value)
3556         .Case("UNUSED_PAD", DstUnused::UNUSED_PAD)
3557         .Case("UNUSED_SEXT", DstUnused::UNUSED_SEXT)
3558         .Case("UNUSED_PRESERVE", DstUnused::UNUSED_PRESERVE)
3559         .Default(0xffffffff);
3560   Parser.Lex(); // eat last token
3561 
3562   if (Int == 0xffffffff) {
3563     return MatchOperand_ParseFail;
3564   }
3565 
3566   Operands.push_back(AMDGPUOperand::CreateImm(this, Int, S, AMDGPUOperand::ImmTySdwaDstUnused));
3567   return MatchOperand_Success;
3568 }
3569 
3570 void AMDGPUAsmParser::cvtSdwaVOP1(MCInst &Inst, const OperandVector &Operands) {
3571   cvtSDWA(Inst, Operands, SIInstrFlags::VOP1);
3572 }
3573 
3574 void AMDGPUAsmParser::cvtSdwaVOP2(MCInst &Inst, const OperandVector &Operands) {
3575   cvtSDWA(Inst, Operands, SIInstrFlags::VOP2);
3576 }
3577 
3578 void AMDGPUAsmParser::cvtSdwaVOPC(MCInst &Inst, const OperandVector &Operands) {
3579   cvtSDWA(Inst, Operands, SIInstrFlags::VOPC);
3580 }
3581 
3582 void AMDGPUAsmParser::cvtSDWA(MCInst &Inst, const OperandVector &Operands,
3583                               uint64_t BasicInstType) {
3584   using namespace llvm::AMDGPU::SDWA;
3585   OptionalImmIndexMap OptionalIdx;
3586 
3587   unsigned I = 1;
3588   const MCInstrDesc &Desc = MII.get(Inst.getOpcode());
3589   for (unsigned J = 0; J < Desc.getNumDefs(); ++J) {
3590     ((AMDGPUOperand &)*Operands[I++]).addRegOperands(Inst, 1);
3591   }
3592 
3593   for (unsigned E = Operands.size(); I != E; ++I) {
3594     AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[I]);
3595     // Add the register arguments
3596     if ((BasicInstType == SIInstrFlags::VOPC ||
3597          BasicInstType == SIInstrFlags::VOP2)&&
3598         Op.isReg() &&
3599         Op.Reg.RegNo == AMDGPU::VCC) {
3600       // VOPC and VOP2b (v_add_u32, v_sub_u32 ...) sdwa use "vcc" token as dst.
3601       // Skip it.
3602       continue;
3603     } else if (isRegOrImmWithInputMods(Desc, Inst.getNumOperands())) {
3604       Op.addRegWithInputModsOperands(Inst, 2);
3605     } else if (Op.isImm()) {
3606       // Handle optional arguments
3607       OptionalIdx[Op.getImmTy()] = I;
3608     } else {
3609       llvm_unreachable("Invalid operand type");
3610     }
3611   }
3612 
3613   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyClampSI, 0);
3614 
3615   if (Inst.getOpcode() != AMDGPU::V_NOP_sdwa_vi) {
3616     // V_NOP_sdwa_vi has no optional sdwa arguments
3617     switch (BasicInstType) {
3618     case SIInstrFlags::VOP1:
3619       addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySdwaDstSel, SdwaSel::DWORD);
3620       addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySdwaDstUnused, DstUnused::UNUSED_PRESERVE);
3621       addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySdwaSrc0Sel, SdwaSel::DWORD);
3622       break;
3623 
3624     case SIInstrFlags::VOP2:
3625       addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySdwaDstSel, SdwaSel::DWORD);
3626       addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySdwaDstUnused, DstUnused::UNUSED_PRESERVE);
3627       addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySdwaSrc0Sel, SdwaSel::DWORD);
3628       addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySdwaSrc1Sel, SdwaSel::DWORD);
3629       break;
3630 
3631     case SIInstrFlags::VOPC:
3632       addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySdwaSrc0Sel, SdwaSel::DWORD);
3633       addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySdwaSrc1Sel, SdwaSel::DWORD);
3634       break;
3635 
3636     default:
3637       llvm_unreachable("Invalid instruction type. Only VOP1, VOP2 and VOPC allowed");
3638     }
3639   }
3640 
3641   // special case v_mac_{f16, f32}:
3642   // it has src2 register operand that is tied to dst operand
3643   if (Inst.getOpcode() == AMDGPU::V_MAC_F32_sdwa_vi ||
3644       Inst.getOpcode() == AMDGPU::V_MAC_F16_sdwa_vi)  {
3645     auto it = Inst.begin();
3646     std::advance(
3647         it, AMDGPU::getNamedOperandIdx(Inst.getOpcode(), AMDGPU::OpName::src2));
3648     Inst.insert(it, Inst.getOperand(0)); // src2 = dst
3649   }
3650 
3651 }
3652 
3653 /// Force static initialization.
3654 extern "C" void LLVMInitializeAMDGPUAsmParser() {
3655   RegisterMCAsmParser<AMDGPUAsmParser> A(getTheAMDGPUTarget());
3656   RegisterMCAsmParser<AMDGPUAsmParser> B(getTheGCNTarget());
3657 }
3658 
3659 #define GET_REGISTER_MATCHER
3660 #define GET_MATCHER_IMPLEMENTATION
3661 #include "AMDGPUGenAsmMatcher.inc"
3662 
3663 // This fuction should be defined after auto-generated include so that we have
3664 // MatchClassKind enum defined
3665 unsigned AMDGPUAsmParser::validateTargetOperandClass(MCParsedAsmOperand &Op,
3666                                                      unsigned Kind) {
3667   // Tokens like "glc" would be parsed as immediate operands in ParseOperand().
3668   // But MatchInstructionImpl() expects to meet token and fails to validate
3669   // operand. This method checks if we are given immediate operand but expect to
3670   // get corresponding token.
3671   AMDGPUOperand &Operand = (AMDGPUOperand&)Op;
3672   switch (Kind) {
3673   case MCK_addr64:
3674     return Operand.isAddr64() ? Match_Success : Match_InvalidOperand;
3675   case MCK_gds:
3676     return Operand.isGDS() ? Match_Success : Match_InvalidOperand;
3677   case MCK_glc:
3678     return Operand.isGLC() ? Match_Success : Match_InvalidOperand;
3679   case MCK_idxen:
3680     return Operand.isIdxen() ? Match_Success : Match_InvalidOperand;
3681   case MCK_offen:
3682     return Operand.isOffen() ? Match_Success : Match_InvalidOperand;
3683   case MCK_SSrcB32:
3684     // When operands have expression values, they will return true for isToken,
3685     // because it is not possible to distinguish between a token and an
3686     // expression at parse time. MatchInstructionImpl() will always try to
3687     // match an operand as a token, when isToken returns true, and when the
3688     // name of the expression is not a valid token, the match will fail,
3689     // so we need to handle it here.
3690     return Operand.isSSrcB32() ? Match_Success : Match_InvalidOperand;
3691   case MCK_SSrcF32:
3692     return Operand.isSSrcF32() ? Match_Success : Match_InvalidOperand;
3693   case MCK_SoppBrTarget:
3694     return Operand.isSoppBrTarget() ? Match_Success : Match_InvalidOperand;
3695   case MCK_VReg32OrOff:
3696     return Operand.isVReg32OrOff() ? Match_Success : Match_InvalidOperand;
3697   case MCK_InterpSlot:
3698     return Operand.isInterpSlot() ? Match_Success : Match_InvalidOperand;
3699   case MCK_Attr:
3700     return Operand.isInterpAttr() ? Match_Success : Match_InvalidOperand;
3701   case MCK_AttrChan:
3702     return Operand.isAttrChan() ? Match_Success : Match_InvalidOperand;
3703   default:
3704     return Match_InvalidOperand;
3705   }
3706 }
3707