1 //===-- AMDGPUAsmParser.cpp - Parse SI asm to MCInst instructions ---------===//
2 //
3 //                     The LLVM Compiler Infrastructure
4 //
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
7 //
8 //===----------------------------------------------------------------------===//
9 
10 #include "AMDKernelCodeT.h"
11 #include "MCTargetDesc/AMDGPUMCTargetDesc.h"
12 #include "MCTargetDesc/AMDGPUTargetStreamer.h"
13 #include "SIDefines.h"
14 #include "Utils/AMDGPUBaseInfo.h"
15 #include "Utils/AMDKernelCodeTUtils.h"
16 #include "Utils/AMDGPUAsmUtils.h"
17 #include "llvm/ADT/APFloat.h"
18 #include "llvm/ADT/APInt.h"
19 #include "llvm/ADT/ArrayRef.h"
20 #include "llvm/ADT/SmallBitVector.h"
21 #include "llvm/ADT/SmallString.h"
22 #include "llvm/ADT/STLExtras.h"
23 #include "llvm/ADT/StringRef.h"
24 #include "llvm/ADT/StringSwitch.h"
25 #include "llvm/ADT/Twine.h"
26 #include "llvm/CodeGen/MachineValueType.h"
27 #include "llvm/MC/MCAsmInfo.h"
28 #include "llvm/MC/MCContext.h"
29 #include "llvm/MC/MCExpr.h"
30 #include "llvm/MC/MCInst.h"
31 #include "llvm/MC/MCInstrDesc.h"
32 #include "llvm/MC/MCInstrInfo.h"
33 #include "llvm/MC/MCParser/MCAsmLexer.h"
34 #include "llvm/MC/MCParser/MCAsmParser.h"
35 #include "llvm/MC/MCParser/MCAsmParserExtension.h"
36 #include "llvm/MC/MCParser/MCParsedAsmOperand.h"
37 #include "llvm/MC/MCParser/MCTargetAsmParser.h"
38 #include "llvm/MC/MCRegisterInfo.h"
39 #include "llvm/MC/MCStreamer.h"
40 #include "llvm/MC/MCSubtargetInfo.h"
41 #include "llvm/MC/MCSymbol.h"
42 #include "llvm/Support/Casting.h"
43 #include "llvm/Support/ELF.h"
44 #include "llvm/Support/ErrorHandling.h"
45 #include "llvm/Support/MathExtras.h"
46 #include "llvm/Support/raw_ostream.h"
47 #include "llvm/Support/SMLoc.h"
48 #include "llvm/Support/TargetRegistry.h"
49 #include <algorithm>
50 #include <cassert>
51 #include <cstdint>
52 #include <cstring>
53 #include <iterator>
54 #include <map>
55 #include <memory>
56 #include <string>
57 
58 using namespace llvm;
59 using namespace llvm::AMDGPU;
60 
61 namespace {
62 
63 class AMDGPUAsmParser;
64 
65 enum RegisterKind { IS_UNKNOWN, IS_VGPR, IS_SGPR, IS_TTMP, IS_SPECIAL };
66 
67 //===----------------------------------------------------------------------===//
68 // Operand
69 //===----------------------------------------------------------------------===//
70 
71 class AMDGPUOperand : public MCParsedAsmOperand {
72   enum KindTy {
73     Token,
74     Immediate,
75     Register,
76     Expression
77   } Kind;
78 
79   SMLoc StartLoc, EndLoc;
80   const AMDGPUAsmParser *AsmParser;
81 
82 public:
83   AMDGPUOperand(KindTy Kind_, const AMDGPUAsmParser *AsmParser_)
84     : MCParsedAsmOperand(), Kind(Kind_), AsmParser(AsmParser_) {}
85 
86   typedef std::unique_ptr<AMDGPUOperand> Ptr;
87 
88   struct Modifiers {
89     bool Abs = false;
90     bool Neg = false;
91     bool Sext = false;
92 
93     bool hasFPModifiers() const { return Abs || Neg; }
94     bool hasIntModifiers() const { return Sext; }
95     bool hasModifiers() const { return hasFPModifiers() || hasIntModifiers(); }
96 
97     int64_t getFPModifiersOperand() const {
98       int64_t Operand = 0;
99       Operand |= Abs ? SISrcMods::ABS : 0;
100       Operand |= Neg ? SISrcMods::NEG : 0;
101       return Operand;
102     }
103 
104     int64_t getIntModifiersOperand() const {
105       int64_t Operand = 0;
106       Operand |= Sext ? SISrcMods::SEXT : 0;
107       return Operand;
108     }
109 
110     int64_t getModifiersOperand() const {
111       assert(!(hasFPModifiers() && hasIntModifiers())
112            && "fp and int modifiers should not be used simultaneously");
113       if (hasFPModifiers()) {
114         return getFPModifiersOperand();
115       } else if (hasIntModifiers()) {
116         return getIntModifiersOperand();
117       } else {
118         return 0;
119       }
120     }
121 
122     friend raw_ostream &operator <<(raw_ostream &OS, AMDGPUOperand::Modifiers Mods);
123   };
124 
125   enum ImmTy {
126     ImmTyNone,
127     ImmTyGDS,
128     ImmTyOffen,
129     ImmTyIdxen,
130     ImmTyAddr64,
131     ImmTyOffset,
132     ImmTyOffset0,
133     ImmTyOffset1,
134     ImmTyGLC,
135     ImmTySLC,
136     ImmTyTFE,
137     ImmTyClampSI,
138     ImmTyOModSI,
139     ImmTyDppCtrl,
140     ImmTyDppRowMask,
141     ImmTyDppBankMask,
142     ImmTyDppBoundCtrl,
143     ImmTySdwaDstSel,
144     ImmTySdwaSrc0Sel,
145     ImmTySdwaSrc1Sel,
146     ImmTySdwaDstUnused,
147     ImmTyDMask,
148     ImmTyUNorm,
149     ImmTyDA,
150     ImmTyR128,
151     ImmTyLWE,
152     ImmTyExpTgt,
153     ImmTyExpCompr,
154     ImmTyExpVM,
155     ImmTyHwreg,
156     ImmTyOff,
157     ImmTySendMsg,
158     ImmTyInterpSlot,
159     ImmTyInterpAttr,
160     ImmTyAttrChan,
161     ImmTyOpSel,
162     ImmTyOpSelHi,
163     ImmTyNegLo,
164     ImmTyNegHi
165   };
166 
167   struct TokOp {
168     const char *Data;
169     unsigned Length;
170   };
171 
172   struct ImmOp {
173     int64_t Val;
174     ImmTy Type;
175     bool IsFPImm;
176     Modifiers Mods;
177   };
178 
179   struct RegOp {
180     unsigned RegNo;
181     bool IsForcedVOP3;
182     Modifiers Mods;
183   };
184 
185   union {
186     TokOp Tok;
187     ImmOp Imm;
188     RegOp Reg;
189     const MCExpr *Expr;
190   };
191 
192   bool isToken() const override {
193     if (Kind == Token)
194       return true;
195 
196     if (Kind != Expression || !Expr)
197       return false;
198 
199     // When parsing operands, we can't always tell if something was meant to be
200     // a token, like 'gds', or an expression that references a global variable.
201     // In this case, we assume the string is an expression, and if we need to
202     // interpret is a token, then we treat the symbol name as the token.
203     return isa<MCSymbolRefExpr>(Expr);
204   }
205 
206   bool isImm() const override {
207     return Kind == Immediate;
208   }
209 
210   bool isInlinableImm(MVT type) const;
211   bool isLiteralImm(MVT type) const;
212 
213   bool isRegKind() const {
214     return Kind == Register;
215   }
216 
217   bool isReg() const override {
218     return isRegKind() && !hasModifiers();
219   }
220 
221   bool isRegOrImmWithInputMods(MVT type) const {
222     return isRegKind() || isInlinableImm(type);
223   }
224 
225   bool isRegOrImmWithInt16InputMods() const {
226     return isRegOrImmWithInputMods(MVT::i16);
227   }
228 
229   bool isRegOrImmWithInt32InputMods() const {
230     return isRegOrImmWithInputMods(MVT::i32);
231   }
232 
233   bool isRegOrImmWithInt64InputMods() const {
234     return isRegOrImmWithInputMods(MVT::i64);
235   }
236 
237   bool isRegOrImmWithFP16InputMods() const {
238     return isRegOrImmWithInputMods(MVT::f16);
239   }
240 
241   bool isRegOrImmWithFP32InputMods() const {
242     return isRegOrImmWithInputMods(MVT::f32);
243   }
244 
245   bool isRegOrImmWithFP64InputMods() const {
246     return isRegOrImmWithInputMods(MVT::f64);
247   }
248 
249   bool isVReg() const {
250     return isRegClass(AMDGPU::VGPR_32RegClassID) ||
251            isRegClass(AMDGPU::VReg_64RegClassID) ||
252            isRegClass(AMDGPU::VReg_96RegClassID) ||
253            isRegClass(AMDGPU::VReg_128RegClassID) ||
254            isRegClass(AMDGPU::VReg_256RegClassID) ||
255            isRegClass(AMDGPU::VReg_512RegClassID);
256   }
257 
258   bool isVReg32OrOff() const {
259     return isOff() || isRegClass(AMDGPU::VGPR_32RegClassID);
260   }
261 
262   bool isImmTy(ImmTy ImmT) const {
263     return isImm() && Imm.Type == ImmT;
264   }
265 
266   bool isImmModifier() const {
267     return isImm() && Imm.Type != ImmTyNone;
268   }
269 
270   bool isClampSI() const { return isImmTy(ImmTyClampSI); }
271   bool isOModSI() const { return isImmTy(ImmTyOModSI); }
272   bool isDMask() const { return isImmTy(ImmTyDMask); }
273   bool isUNorm() const { return isImmTy(ImmTyUNorm); }
274   bool isDA() const { return isImmTy(ImmTyDA); }
275   bool isR128() const { return isImmTy(ImmTyUNorm); }
276   bool isLWE() const { return isImmTy(ImmTyLWE); }
277   bool isOff() const { return isImmTy(ImmTyOff); }
278   bool isExpTgt() const { return isImmTy(ImmTyExpTgt); }
279   bool isExpVM() const { return isImmTy(ImmTyExpVM); }
280   bool isExpCompr() const { return isImmTy(ImmTyExpCompr); }
281   bool isOffen() const { return isImmTy(ImmTyOffen); }
282   bool isIdxen() const { return isImmTy(ImmTyIdxen); }
283   bool isAddr64() const { return isImmTy(ImmTyAddr64); }
284   bool isOffset() const { return isImmTy(ImmTyOffset) && isUInt<16>(getImm()); }
285   bool isOffset0() const { return isImmTy(ImmTyOffset0) && isUInt<16>(getImm()); }
286   bool isOffset1() const { return isImmTy(ImmTyOffset1) && isUInt<8>(getImm()); }
287   bool isGDS() const { return isImmTy(ImmTyGDS); }
288   bool isGLC() const { return isImmTy(ImmTyGLC); }
289   bool isSLC() const { return isImmTy(ImmTySLC); }
290   bool isTFE() const { return isImmTy(ImmTyTFE); }
291   bool isBankMask() const { return isImmTy(ImmTyDppBankMask); }
292   bool isRowMask() const { return isImmTy(ImmTyDppRowMask); }
293   bool isBoundCtrl() const { return isImmTy(ImmTyDppBoundCtrl); }
294   bool isSDWADstSel() const { return isImmTy(ImmTySdwaDstSel); }
295   bool isSDWASrc0Sel() const { return isImmTy(ImmTySdwaSrc0Sel); }
296   bool isSDWASrc1Sel() const { return isImmTy(ImmTySdwaSrc1Sel); }
297   bool isSDWADstUnused() const { return isImmTy(ImmTySdwaDstUnused); }
298   bool isInterpSlot() const { return isImmTy(ImmTyInterpSlot); }
299   bool isInterpAttr() const { return isImmTy(ImmTyInterpAttr); }
300   bool isAttrChan() const { return isImmTy(ImmTyAttrChan); }
301   bool isOpSel() const { return isImmTy(ImmTyOpSel); }
302   bool isOpSelHi() const { return isImmTy(ImmTyOpSelHi); }
303   bool isNegLo() const { return isImmTy(ImmTyNegLo); }
304   bool isNegHi() const { return isImmTy(ImmTyNegHi); }
305 
306   bool isMod() const {
307     return isClampSI() || isOModSI();
308   }
309 
310   bool isRegOrImm() const {
311     return isReg() || isImm();
312   }
313 
314   bool isRegClass(unsigned RCID) const;
315 
316   bool isRegOrInlineNoMods(unsigned RCID, MVT type) const {
317     return (isRegClass(RCID) || isInlinableImm(type)) && !hasModifiers();
318   }
319 
320   bool isSCSrcB16() const {
321     return isRegOrInlineNoMods(AMDGPU::SReg_32RegClassID, MVT::i16);
322   }
323 
324   bool isSCSrcV2B16() const {
325     return isSCSrcB16();
326   }
327 
328   bool isSCSrcB32() const {
329     return isRegOrInlineNoMods(AMDGPU::SReg_32RegClassID, MVT::i32);
330   }
331 
332   bool isSCSrcB64() const {
333     return isRegOrInlineNoMods(AMDGPU::SReg_64RegClassID, MVT::i64);
334   }
335 
336   bool isSCSrcF16() const {
337     return isRegOrInlineNoMods(AMDGPU::SReg_32RegClassID, MVT::f16);
338   }
339 
340   bool isSCSrcV2F16() const {
341     return isSCSrcF16();
342   }
343 
344   bool isSCSrcF32() const {
345     return isRegOrInlineNoMods(AMDGPU::SReg_32RegClassID, MVT::f32);
346   }
347 
348   bool isSCSrcF64() const {
349     return isRegOrInlineNoMods(AMDGPU::SReg_64RegClassID, MVT::f64);
350   }
351 
352   bool isSSrcB32() const {
353     return isSCSrcB32() || isLiteralImm(MVT::i32) || isExpr();
354   }
355 
356   bool isSSrcB16() const {
357     return isSCSrcB16() || isLiteralImm(MVT::i16);
358   }
359 
360   bool isSSrcV2B16() const {
361     llvm_unreachable("cannot happen");
362     return isSSrcB16();
363   }
364 
365   bool isSSrcB64() const {
366     // TODO: Find out how SALU supports extension of 32-bit literals to 64 bits.
367     // See isVSrc64().
368     return isSCSrcB64() || isLiteralImm(MVT::i64);
369   }
370 
371   bool isSSrcF32() const {
372     return isSCSrcB32() || isLiteralImm(MVT::f32) || isExpr();
373   }
374 
375   bool isSSrcF64() const {
376     return isSCSrcB64() || isLiteralImm(MVT::f64);
377   }
378 
379   bool isSSrcF16() const {
380     return isSCSrcB16() || isLiteralImm(MVT::f16);
381   }
382 
383   bool isSSrcV2F16() const {
384     llvm_unreachable("cannot happen");
385     return isSSrcF16();
386   }
387 
388   bool isVCSrcB32() const {
389     return isRegOrInlineNoMods(AMDGPU::VS_32RegClassID, MVT::i32);
390   }
391 
392   bool isVCSrcB64() const {
393     return isRegOrInlineNoMods(AMDGPU::VS_64RegClassID, MVT::i64);
394   }
395 
396   bool isVCSrcB16() const {
397     return isRegOrInlineNoMods(AMDGPU::VS_32RegClassID, MVT::i16);
398   }
399 
400   bool isVCSrcV2B16() const {
401     return isVCSrcB16();
402   }
403 
404   bool isVCSrcF32() const {
405     return isRegOrInlineNoMods(AMDGPU::VS_32RegClassID, MVT::f32);
406   }
407 
408   bool isVCSrcF64() const {
409     return isRegOrInlineNoMods(AMDGPU::VS_64RegClassID, MVT::f64);
410   }
411 
412   bool isVCSrcF16() const {
413     return isRegOrInlineNoMods(AMDGPU::VS_32RegClassID, MVT::f16);
414   }
415 
416   bool isVCSrcV2F16() const {
417     return isVCSrcF16();
418   }
419 
420   bool isVSrcB32() const {
421     return isVCSrcF32() || isLiteralImm(MVT::i32);
422   }
423 
424   bool isVSrcB64() const {
425     return isVCSrcF64() || isLiteralImm(MVT::i64);
426   }
427 
428   bool isVSrcB16() const {
429     return isVCSrcF16() || isLiteralImm(MVT::i16);
430   }
431 
432   bool isVSrcV2B16() const {
433     llvm_unreachable("cannot happen");
434     return isVSrcB16();
435   }
436 
437   bool isVSrcF32() const {
438     return isVCSrcF32() || isLiteralImm(MVT::f32);
439   }
440 
441   bool isVSrcF64() const {
442     return isVCSrcF64() || isLiteralImm(MVT::f64);
443   }
444 
445   bool isVSrcF16() const {
446     return isVCSrcF16() || isLiteralImm(MVT::f16);
447   }
448 
449   bool isVSrcV2F16() const {
450     llvm_unreachable("cannot happen");
451     return isVSrcF16();
452   }
453 
454   bool isKImmFP32() const {
455     return isLiteralImm(MVT::f32);
456   }
457 
458   bool isKImmFP16() const {
459     return isLiteralImm(MVT::f16);
460   }
461 
462   bool isMem() const override {
463     return false;
464   }
465 
466   bool isExpr() const {
467     return Kind == Expression;
468   }
469 
470   bool isSoppBrTarget() const {
471     return isExpr() || isImm();
472   }
473 
474   bool isSWaitCnt() const;
475   bool isHwreg() const;
476   bool isSendMsg() const;
477   bool isSMRDOffset8() const;
478   bool isSMRDOffset20() const;
479   bool isSMRDLiteralOffset() const;
480   bool isDPPCtrl() const;
481   bool isGPRIdxMode() const;
482 
483   StringRef getExpressionAsToken() const {
484     assert(isExpr());
485     const MCSymbolRefExpr *S = cast<MCSymbolRefExpr>(Expr);
486     return S->getSymbol().getName();
487   }
488 
489   StringRef getToken() const {
490     assert(isToken());
491 
492     if (Kind == Expression)
493       return getExpressionAsToken();
494 
495     return StringRef(Tok.Data, Tok.Length);
496   }
497 
498   int64_t getImm() const {
499     assert(isImm());
500     return Imm.Val;
501   }
502 
503   ImmTy getImmTy() const {
504     assert(isImm());
505     return Imm.Type;
506   }
507 
508   unsigned getReg() const override {
509     return Reg.RegNo;
510   }
511 
512   SMLoc getStartLoc() const override {
513     return StartLoc;
514   }
515 
516   SMLoc getEndLoc() const override {
517     return EndLoc;
518   }
519 
520   Modifiers getModifiers() const {
521     assert(isRegKind() || isImmTy(ImmTyNone));
522     return isRegKind() ? Reg.Mods : Imm.Mods;
523   }
524 
525   void setModifiers(Modifiers Mods) {
526     assert(isRegKind() || isImmTy(ImmTyNone));
527     if (isRegKind())
528       Reg.Mods = Mods;
529     else
530       Imm.Mods = Mods;
531   }
532 
533   bool hasModifiers() const {
534     return getModifiers().hasModifiers();
535   }
536 
537   bool hasFPModifiers() const {
538     return getModifiers().hasFPModifiers();
539   }
540 
541   bool hasIntModifiers() const {
542     return getModifiers().hasIntModifiers();
543   }
544 
545   void addImmOperands(MCInst &Inst, unsigned N, bool ApplyModifiers = true) const;
546 
547   void addLiteralImmOperand(MCInst &Inst, int64_t Val) const;
548 
549   template <unsigned Bitwidth>
550   void addKImmFPOperands(MCInst &Inst, unsigned N) const;
551 
552   void addKImmFP16Operands(MCInst &Inst, unsigned N) const {
553     addKImmFPOperands<16>(Inst, N);
554   }
555 
556   void addKImmFP32Operands(MCInst &Inst, unsigned N) const {
557     addKImmFPOperands<32>(Inst, N);
558   }
559 
560   void addRegOperands(MCInst &Inst, unsigned N) const;
561 
562   void addRegOrImmOperands(MCInst &Inst, unsigned N) const {
563     if (isRegKind())
564       addRegOperands(Inst, N);
565     else if (isExpr())
566       Inst.addOperand(MCOperand::createExpr(Expr));
567     else
568       addImmOperands(Inst, N);
569   }
570 
571   void addRegOrImmWithInputModsOperands(MCInst &Inst, unsigned N) const {
572     Modifiers Mods = getModifiers();
573     Inst.addOperand(MCOperand::createImm(Mods.getModifiersOperand()));
574     if (isRegKind()) {
575       addRegOperands(Inst, N);
576     } else {
577       addImmOperands(Inst, N, false);
578     }
579   }
580 
581   void addRegOrImmWithFPInputModsOperands(MCInst &Inst, unsigned N) const {
582     assert(!hasIntModifiers());
583     addRegOrImmWithInputModsOperands(Inst, N);
584   }
585 
586   void addRegOrImmWithIntInputModsOperands(MCInst &Inst, unsigned N) const {
587     assert(!hasFPModifiers());
588     addRegOrImmWithInputModsOperands(Inst, N);
589   }
590 
591   void addRegWithInputModsOperands(MCInst &Inst, unsigned N) const {
592     Modifiers Mods = getModifiers();
593     Inst.addOperand(MCOperand::createImm(Mods.getModifiersOperand()));
594     assert(isRegKind());
595     addRegOperands(Inst, N);
596   }
597 
598   void addRegWithFPInputModsOperands(MCInst &Inst, unsigned N) const {
599     assert(!hasIntModifiers());
600     addRegWithInputModsOperands(Inst, N);
601   }
602 
603   void addRegWithIntInputModsOperands(MCInst &Inst, unsigned N) const {
604     assert(!hasFPModifiers());
605     addRegWithInputModsOperands(Inst, N);
606   }
607 
608   void addSoppBrTargetOperands(MCInst &Inst, unsigned N) const {
609     if (isImm())
610       addImmOperands(Inst, N);
611     else {
612       assert(isExpr());
613       Inst.addOperand(MCOperand::createExpr(Expr));
614     }
615   }
616 
617   static void printImmTy(raw_ostream& OS, ImmTy Type) {
618     switch (Type) {
619     case ImmTyNone: OS << "None"; break;
620     case ImmTyGDS: OS << "GDS"; break;
621     case ImmTyOffen: OS << "Offen"; break;
622     case ImmTyIdxen: OS << "Idxen"; break;
623     case ImmTyAddr64: OS << "Addr64"; break;
624     case ImmTyOffset: OS << "Offset"; break;
625     case ImmTyOffset0: OS << "Offset0"; break;
626     case ImmTyOffset1: OS << "Offset1"; break;
627     case ImmTyGLC: OS << "GLC"; break;
628     case ImmTySLC: OS << "SLC"; break;
629     case ImmTyTFE: OS << "TFE"; break;
630     case ImmTyClampSI: OS << "ClampSI"; break;
631     case ImmTyOModSI: OS << "OModSI"; break;
632     case ImmTyDppCtrl: OS << "DppCtrl"; break;
633     case ImmTyDppRowMask: OS << "DppRowMask"; break;
634     case ImmTyDppBankMask: OS << "DppBankMask"; break;
635     case ImmTyDppBoundCtrl: OS << "DppBoundCtrl"; break;
636     case ImmTySdwaDstSel: OS << "SdwaDstSel"; break;
637     case ImmTySdwaSrc0Sel: OS << "SdwaSrc0Sel"; break;
638     case ImmTySdwaSrc1Sel: OS << "SdwaSrc1Sel"; break;
639     case ImmTySdwaDstUnused: OS << "SdwaDstUnused"; break;
640     case ImmTyDMask: OS << "DMask"; break;
641     case ImmTyUNorm: OS << "UNorm"; break;
642     case ImmTyDA: OS << "DA"; break;
643     case ImmTyR128: OS << "R128"; break;
644     case ImmTyLWE: OS << "LWE"; break;
645     case ImmTyOff: OS << "Off"; break;
646     case ImmTyExpTgt: OS << "ExpTgt"; break;
647     case ImmTyExpCompr: OS << "ExpCompr"; break;
648     case ImmTyExpVM: OS << "ExpVM"; break;
649     case ImmTyHwreg: OS << "Hwreg"; break;
650     case ImmTySendMsg: OS << "SendMsg"; break;
651     case ImmTyInterpSlot: OS << "InterpSlot"; break;
652     case ImmTyInterpAttr: OS << "InterpAttr"; break;
653     case ImmTyAttrChan: OS << "AttrChan"; break;
654     case ImmTyOpSel: OS << "OpSel"; break;
655     case ImmTyOpSelHi: OS << "OpSelHi"; break;
656     case ImmTyNegLo: OS << "NegLo"; break;
657     case ImmTyNegHi: OS << "NegHi"; break;
658     }
659   }
660 
661   void print(raw_ostream &OS) const override {
662     switch (Kind) {
663     case Register:
664       OS << "<register " << getReg() << " mods: " << Reg.Mods << '>';
665       break;
666     case Immediate:
667       OS << '<' << getImm();
668       if (getImmTy() != ImmTyNone) {
669         OS << " type: "; printImmTy(OS, getImmTy());
670       }
671       OS << " mods: " << Imm.Mods << '>';
672       break;
673     case Token:
674       OS << '\'' << getToken() << '\'';
675       break;
676     case Expression:
677       OS << "<expr " << *Expr << '>';
678       break;
679     }
680   }
681 
682   static AMDGPUOperand::Ptr CreateImm(const AMDGPUAsmParser *AsmParser,
683                                       int64_t Val, SMLoc Loc,
684                                       ImmTy Type = ImmTyNone,
685                                       bool IsFPImm = false) {
686     auto Op = llvm::make_unique<AMDGPUOperand>(Immediate, AsmParser);
687     Op->Imm.Val = Val;
688     Op->Imm.IsFPImm = IsFPImm;
689     Op->Imm.Type = Type;
690     Op->Imm.Mods = Modifiers();
691     Op->StartLoc = Loc;
692     Op->EndLoc = Loc;
693     return Op;
694   }
695 
696   static AMDGPUOperand::Ptr CreateToken(const AMDGPUAsmParser *AsmParser,
697                                         StringRef Str, SMLoc Loc,
698                                         bool HasExplicitEncodingSize = true) {
699     auto Res = llvm::make_unique<AMDGPUOperand>(Token, AsmParser);
700     Res->Tok.Data = Str.data();
701     Res->Tok.Length = Str.size();
702     Res->StartLoc = Loc;
703     Res->EndLoc = Loc;
704     return Res;
705   }
706 
707   static AMDGPUOperand::Ptr CreateReg(const AMDGPUAsmParser *AsmParser,
708                                       unsigned RegNo, SMLoc S,
709                                       SMLoc E,
710                                       bool ForceVOP3) {
711     auto Op = llvm::make_unique<AMDGPUOperand>(Register, AsmParser);
712     Op->Reg.RegNo = RegNo;
713     Op->Reg.Mods = Modifiers();
714     Op->Reg.IsForcedVOP3 = ForceVOP3;
715     Op->StartLoc = S;
716     Op->EndLoc = E;
717     return Op;
718   }
719 
720   static AMDGPUOperand::Ptr CreateExpr(const AMDGPUAsmParser *AsmParser,
721                                        const class MCExpr *Expr, SMLoc S) {
722     auto Op = llvm::make_unique<AMDGPUOperand>(Expression, AsmParser);
723     Op->Expr = Expr;
724     Op->StartLoc = S;
725     Op->EndLoc = S;
726     return Op;
727   }
728 };
729 
730 raw_ostream &operator <<(raw_ostream &OS, AMDGPUOperand::Modifiers Mods) {
731   OS << "abs:" << Mods.Abs << " neg: " << Mods.Neg << " sext:" << Mods.Sext;
732   return OS;
733 }
734 
735 //===----------------------------------------------------------------------===//
736 // AsmParser
737 //===----------------------------------------------------------------------===//
738 
739 // Holds info related to the current kernel, e.g. count of SGPRs used.
740 // Kernel scope begins at .amdgpu_hsa_kernel directive, ends at next
741 // .amdgpu_hsa_kernel or at EOF.
742 class KernelScopeInfo {
743   int SgprIndexUnusedMin = -1;
744   int VgprIndexUnusedMin = -1;
745   MCContext *Ctx = nullptr;
746 
747   void usesSgprAt(int i) {
748     if (i >= SgprIndexUnusedMin) {
749       SgprIndexUnusedMin = ++i;
750       if (Ctx) {
751         MCSymbol * const Sym = Ctx->getOrCreateSymbol(Twine(".kernel.sgpr_count"));
752         Sym->setVariableValue(MCConstantExpr::create(SgprIndexUnusedMin, *Ctx));
753       }
754     }
755   }
756 
757   void usesVgprAt(int i) {
758     if (i >= VgprIndexUnusedMin) {
759       VgprIndexUnusedMin = ++i;
760       if (Ctx) {
761         MCSymbol * const Sym = Ctx->getOrCreateSymbol(Twine(".kernel.vgpr_count"));
762         Sym->setVariableValue(MCConstantExpr::create(VgprIndexUnusedMin, *Ctx));
763       }
764     }
765   }
766 
767 public:
768   KernelScopeInfo() = default;
769 
770   void initialize(MCContext &Context) {
771     Ctx = &Context;
772     usesSgprAt(SgprIndexUnusedMin = -1);
773     usesVgprAt(VgprIndexUnusedMin = -1);
774   }
775 
776   void usesRegister(RegisterKind RegKind, unsigned DwordRegIndex, unsigned RegWidth) {
777     switch (RegKind) {
778       case IS_SGPR: usesSgprAt(DwordRegIndex + RegWidth - 1); break;
779       case IS_VGPR: usesVgprAt(DwordRegIndex + RegWidth - 1); break;
780       default: break;
781     }
782   }
783 };
784 
785 class AMDGPUAsmParser : public MCTargetAsmParser {
786   const MCInstrInfo &MII;
787   MCAsmParser &Parser;
788 
789   unsigned ForcedEncodingSize = 0;
790   bool ForcedDPP = false;
791   bool ForcedSDWA = false;
792   KernelScopeInfo KernelScope;
793 
794   /// @name Auto-generated Match Functions
795   /// {
796 
797 #define GET_ASSEMBLER_HEADER
798 #include "AMDGPUGenAsmMatcher.inc"
799 
800   /// }
801 
802 private:
803   bool ParseAsAbsoluteExpression(uint32_t &Ret);
804   bool ParseDirectiveMajorMinor(uint32_t &Major, uint32_t &Minor);
805   bool ParseDirectiveHSACodeObjectVersion();
806   bool ParseDirectiveHSACodeObjectISA();
807   bool ParseDirectiveRuntimeMetadata();
808   bool ParseAMDKernelCodeTValue(StringRef ID, amd_kernel_code_t &Header);
809   bool ParseDirectiveAMDKernelCodeT();
810   bool ParseSectionDirectiveHSAText();
811   bool subtargetHasRegister(const MCRegisterInfo &MRI, unsigned RegNo) const;
812   bool ParseDirectiveAMDGPUHsaKernel();
813   bool ParseDirectiveAMDGPUHsaModuleGlobal();
814   bool ParseDirectiveAMDGPUHsaProgramGlobal();
815   bool ParseSectionDirectiveHSADataGlobalAgent();
816   bool ParseSectionDirectiveHSADataGlobalProgram();
817   bool ParseSectionDirectiveHSARodataReadonlyAgent();
818   bool AddNextRegisterToList(unsigned& Reg, unsigned& RegWidth,
819                              RegisterKind RegKind, unsigned Reg1,
820                              unsigned RegNum);
821   bool ParseAMDGPURegister(RegisterKind& RegKind, unsigned& Reg,
822                            unsigned& RegNum, unsigned& RegWidth,
823                            unsigned *DwordRegIndex);
824   void cvtMubufImpl(MCInst &Inst, const OperandVector &Operands,
825                     bool IsAtomic, bool IsAtomicReturn);
826   void cvtDSImpl(MCInst &Inst, const OperandVector &Operands,
827                  bool IsGdsHardcoded);
828 
829 public:
830   enum AMDGPUMatchResultTy {
831     Match_PreferE32 = FIRST_TARGET_MATCH_RESULT_TY
832   };
833 
834   typedef std::map<AMDGPUOperand::ImmTy, unsigned> OptionalImmIndexMap;
835 
836   AMDGPUAsmParser(const MCSubtargetInfo &STI, MCAsmParser &_Parser,
837                const MCInstrInfo &MII,
838                const MCTargetOptions &Options)
839       : MCTargetAsmParser(Options, STI), MII(MII), Parser(_Parser) {
840     MCAsmParserExtension::Initialize(Parser);
841 
842     if (getFeatureBits().none()) {
843       // Set default features.
844       copySTI().ToggleFeature("SOUTHERN_ISLANDS");
845     }
846 
847     setAvailableFeatures(ComputeAvailableFeatures(getFeatureBits()));
848 
849     {
850       // TODO: make those pre-defined variables read-only.
851       // Currently there is none suitable machinery in the core llvm-mc for this.
852       // MCSymbol::isRedefinable is intended for another purpose, and
853       // AsmParser::parseDirectiveSet() cannot be specialized for specific target.
854       AMDGPU::IsaInfo::IsaVersion ISA =
855           AMDGPU::IsaInfo::getIsaVersion(getFeatureBits());
856       MCContext &Ctx = getContext();
857       MCSymbol *Sym =
858           Ctx.getOrCreateSymbol(Twine(".option.machine_version_major"));
859       Sym->setVariableValue(MCConstantExpr::create(ISA.Major, Ctx));
860       Sym = Ctx.getOrCreateSymbol(Twine(".option.machine_version_minor"));
861       Sym->setVariableValue(MCConstantExpr::create(ISA.Minor, Ctx));
862       Sym = Ctx.getOrCreateSymbol(Twine(".option.machine_version_stepping"));
863       Sym->setVariableValue(MCConstantExpr::create(ISA.Stepping, Ctx));
864     }
865     KernelScope.initialize(getContext());
866   }
867 
868   bool isSI() const {
869     return AMDGPU::isSI(getSTI());
870   }
871 
872   bool isCI() const {
873     return AMDGPU::isCI(getSTI());
874   }
875 
876   bool isVI() const {
877     return AMDGPU::isVI(getSTI());
878   }
879 
880   bool hasInv2PiInlineImm() const {
881     return getFeatureBits()[AMDGPU::FeatureInv2PiInlineImm];
882   }
883 
884   bool hasSGPR102_SGPR103() const {
885     return !isVI();
886   }
887 
888   AMDGPUTargetStreamer &getTargetStreamer() {
889     MCTargetStreamer &TS = *getParser().getStreamer().getTargetStreamer();
890     return static_cast<AMDGPUTargetStreamer &>(TS);
891   }
892 
893   const MCRegisterInfo *getMRI() const {
894     // We need this const_cast because for some reason getContext() is not const
895     // in MCAsmParser.
896     return const_cast<AMDGPUAsmParser*>(this)->getContext().getRegisterInfo();
897   }
898 
899   const MCInstrInfo *getMII() const {
900     return &MII;
901   }
902 
903   const FeatureBitset &getFeatureBits() const {
904     return getSTI().getFeatureBits();
905   }
906 
907   void setForcedEncodingSize(unsigned Size) { ForcedEncodingSize = Size; }
908   void setForcedDPP(bool ForceDPP_) { ForcedDPP = ForceDPP_; }
909   void setForcedSDWA(bool ForceSDWA_) { ForcedSDWA = ForceSDWA_; }
910 
911   unsigned getForcedEncodingSize() const { return ForcedEncodingSize; }
912   bool isForcedVOP3() const { return ForcedEncodingSize == 64; }
913   bool isForcedDPP() const { return ForcedDPP; }
914   bool isForcedSDWA() const { return ForcedSDWA; }
915   ArrayRef<unsigned> getMatchedVariants() const;
916 
917   std::unique_ptr<AMDGPUOperand> parseRegister();
918   bool ParseRegister(unsigned &RegNo, SMLoc &StartLoc, SMLoc &EndLoc) override;
919   unsigned checkTargetMatchPredicate(MCInst &Inst) override;
920   unsigned validateTargetOperandClass(MCParsedAsmOperand &Op,
921                                       unsigned Kind) override;
922   bool MatchAndEmitInstruction(SMLoc IDLoc, unsigned &Opcode,
923                                OperandVector &Operands, MCStreamer &Out,
924                                uint64_t &ErrorInfo,
925                                bool MatchingInlineAsm) override;
926   bool ParseDirective(AsmToken DirectiveID) override;
927   OperandMatchResultTy parseOperand(OperandVector &Operands, StringRef Mnemonic);
928   StringRef parseMnemonicSuffix(StringRef Name);
929   bool ParseInstruction(ParseInstructionInfo &Info, StringRef Name,
930                         SMLoc NameLoc, OperandVector &Operands) override;
931   //bool ProcessInstruction(MCInst &Inst);
932 
933   OperandMatchResultTy parseIntWithPrefix(const char *Prefix, int64_t &Int);
934 
935   OperandMatchResultTy
936   parseIntWithPrefix(const char *Prefix, OperandVector &Operands,
937                      AMDGPUOperand::ImmTy ImmTy = AMDGPUOperand::ImmTyNone,
938                      bool (*ConvertResult)(int64_t &) = nullptr);
939 
940   OperandMatchResultTy parseOperandArrayWithPrefix(
941     const char *Prefix,
942     OperandVector &Operands,
943     AMDGPUOperand::ImmTy ImmTy = AMDGPUOperand::ImmTyNone,
944     bool (*ConvertResult)(int64_t&) = nullptr);
945 
946   OperandMatchResultTy
947   parseNamedBit(const char *Name, OperandVector &Operands,
948                 AMDGPUOperand::ImmTy ImmTy = AMDGPUOperand::ImmTyNone);
949   OperandMatchResultTy parseStringWithPrefix(StringRef Prefix,
950                                              StringRef &Value);
951 
952   OperandMatchResultTy parseImm(OperandVector &Operands);
953   OperandMatchResultTy parseReg(OperandVector &Operands);
954   OperandMatchResultTy parseRegOrImm(OperandVector &Operands);
955   OperandMatchResultTy parseRegOrImmWithFPInputMods(OperandVector &Operands, bool AllowImm = true);
956   OperandMatchResultTy parseRegOrImmWithIntInputMods(OperandVector &Operands, bool AllowImm = true);
957   OperandMatchResultTy parseRegWithFPInputMods(OperandVector &Operands);
958   OperandMatchResultTy parseRegWithIntInputMods(OperandVector &Operands);
959   OperandMatchResultTy parseVReg32OrOff(OperandVector &Operands);
960 
961   void cvtDSOffset01(MCInst &Inst, const OperandVector &Operands);
962   void cvtDS(MCInst &Inst, const OperandVector &Operands) { cvtDSImpl(Inst, Operands, false); }
963   void cvtDSGds(MCInst &Inst, const OperandVector &Operands) { cvtDSImpl(Inst, Operands, true); }
964   void cvtExp(MCInst &Inst, const OperandVector &Operands);
965 
966   bool parseCnt(int64_t &IntVal);
967   OperandMatchResultTy parseSWaitCntOps(OperandVector &Operands);
968   OperandMatchResultTy parseHwreg(OperandVector &Operands);
969 
970 private:
971   struct OperandInfoTy {
972     int64_t Id;
973     bool IsSymbolic;
974     OperandInfoTy(int64_t Id_) : Id(Id_), IsSymbolic(false) { }
975   };
976 
977   bool parseSendMsgConstruct(OperandInfoTy &Msg, OperandInfoTy &Operation, int64_t &StreamId);
978   bool parseHwregConstruct(OperandInfoTy &HwReg, int64_t &Offset, int64_t &Width);
979 
980   void errorExpTgt();
981   OperandMatchResultTy parseExpTgtImpl(StringRef Str, uint8_t &Val);
982 
983 public:
984   OperandMatchResultTy parseOptionalOperand(OperandVector &Operands);
985 
986   OperandMatchResultTy parseExpTgt(OperandVector &Operands);
987   OperandMatchResultTy parseSendMsgOp(OperandVector &Operands);
988   OperandMatchResultTy parseInterpSlot(OperandVector &Operands);
989   OperandMatchResultTy parseInterpAttr(OperandVector &Operands);
990   OperandMatchResultTy parseSOppBrTarget(OperandVector &Operands);
991 
992   void cvtMubuf(MCInst &Inst, const OperandVector &Operands) { cvtMubufImpl(Inst, Operands, false, false); }
993   void cvtMubufAtomic(MCInst &Inst, const OperandVector &Operands) { cvtMubufImpl(Inst, Operands, true, false); }
994   void cvtMubufAtomicReturn(MCInst &Inst, const OperandVector &Operands) { cvtMubufImpl(Inst, Operands, true, true); }
995   AMDGPUOperand::Ptr defaultGLC() const;
996   AMDGPUOperand::Ptr defaultSLC() const;
997   AMDGPUOperand::Ptr defaultTFE() const;
998 
999   AMDGPUOperand::Ptr defaultDMask() const;
1000   AMDGPUOperand::Ptr defaultUNorm() const;
1001   AMDGPUOperand::Ptr defaultDA() const;
1002   AMDGPUOperand::Ptr defaultR128() const;
1003   AMDGPUOperand::Ptr defaultLWE() const;
1004   AMDGPUOperand::Ptr defaultSMRDOffset8() const;
1005   AMDGPUOperand::Ptr defaultSMRDOffset20() const;
1006   AMDGPUOperand::Ptr defaultSMRDLiteralOffset() const;
1007 
1008   OperandMatchResultTy parseOModOperand(OperandVector &Operands);
1009 
1010   void cvtId(MCInst &Inst, const OperandVector &Operands);
1011   void cvtVOP3_2_mod(MCInst &Inst, const OperandVector &Operands);
1012 
1013   void cvtVOP3Impl(MCInst &Inst,
1014                    const OperandVector &Operands,
1015                    OptionalImmIndexMap &OptionalIdx);
1016   void cvtVOP3(MCInst &Inst, const OperandVector &Operands);
1017   void cvtVOP3P(MCInst &Inst, const OperandVector &Operands);
1018 
1019   void cvtMIMG(MCInst &Inst, const OperandVector &Operands);
1020   void cvtMIMGAtomic(MCInst &Inst, const OperandVector &Operands);
1021 
1022   OperandMatchResultTy parseDPPCtrl(OperandVector &Operands);
1023   AMDGPUOperand::Ptr defaultRowMask() const;
1024   AMDGPUOperand::Ptr defaultBankMask() const;
1025   AMDGPUOperand::Ptr defaultBoundCtrl() const;
1026   void cvtDPP(MCInst &Inst, const OperandVector &Operands);
1027 
1028   OperandMatchResultTy parseSDWASel(OperandVector &Operands, StringRef Prefix,
1029                                     AMDGPUOperand::ImmTy Type);
1030   OperandMatchResultTy parseSDWADstUnused(OperandVector &Operands);
1031   void cvtSdwaVOP1(MCInst &Inst, const OperandVector &Operands);
1032   void cvtSdwaVOP2(MCInst &Inst, const OperandVector &Operands);
1033   void cvtSdwaVOPC(MCInst &Inst, const OperandVector &Operands);
1034   void cvtSDWA(MCInst &Inst, const OperandVector &Operands,
1035                uint64_t BasicInstType);
1036 };
1037 
1038 struct OptionalOperand {
1039   const char *Name;
1040   AMDGPUOperand::ImmTy Type;
1041   bool IsBit;
1042   bool (*ConvertResult)(int64_t&);
1043 };
1044 
1045 } // end anonymous namespace
1046 
1047 // May be called with integer type with equivalent bitwidth.
1048 static const fltSemantics *getFltSemantics(unsigned Size) {
1049   switch (Size) {
1050   case 4:
1051     return &APFloat::IEEEsingle();
1052   case 8:
1053     return &APFloat::IEEEdouble();
1054   case 2:
1055     return &APFloat::IEEEhalf();
1056   default:
1057     llvm_unreachable("unsupported fp type");
1058   }
1059 }
1060 
1061 static const fltSemantics *getFltSemantics(MVT VT) {
1062   return getFltSemantics(VT.getSizeInBits() / 8);
1063 }
1064 
1065 static const fltSemantics *getOpFltSemantics(uint8_t OperandType) {
1066   switch (OperandType) {
1067   case AMDGPU::OPERAND_REG_IMM_INT32:
1068   case AMDGPU::OPERAND_REG_IMM_FP32:
1069   case AMDGPU::OPERAND_REG_INLINE_C_INT32:
1070   case AMDGPU::OPERAND_REG_INLINE_C_FP32:
1071     return &APFloat::IEEEsingle();
1072   case AMDGPU::OPERAND_REG_IMM_INT64:
1073   case AMDGPU::OPERAND_REG_IMM_FP64:
1074   case AMDGPU::OPERAND_REG_INLINE_C_INT64:
1075   case AMDGPU::OPERAND_REG_INLINE_C_FP64:
1076     return &APFloat::IEEEdouble();
1077   case AMDGPU::OPERAND_REG_IMM_INT16:
1078   case AMDGPU::OPERAND_REG_IMM_FP16:
1079   case AMDGPU::OPERAND_REG_INLINE_C_INT16:
1080   case AMDGPU::OPERAND_REG_INLINE_C_FP16:
1081   case AMDGPU::OPERAND_REG_INLINE_C_V2INT16:
1082   case AMDGPU::OPERAND_REG_INLINE_C_V2FP16:
1083     return &APFloat::IEEEhalf();
1084   default:
1085     llvm_unreachable("unsupported fp type");
1086   }
1087 }
1088 
1089 //===----------------------------------------------------------------------===//
1090 // Operand
1091 //===----------------------------------------------------------------------===//
1092 
1093 static bool canLosslesslyConvertToFPType(APFloat &FPLiteral, MVT VT) {
1094   bool Lost;
1095 
1096   // Convert literal to single precision
1097   APFloat::opStatus Status = FPLiteral.convert(*getFltSemantics(VT),
1098                                                APFloat::rmNearestTiesToEven,
1099                                                &Lost);
1100   // We allow precision lost but not overflow or underflow
1101   if (Status != APFloat::opOK &&
1102       Lost &&
1103       ((Status & APFloat::opOverflow)  != 0 ||
1104        (Status & APFloat::opUnderflow) != 0)) {
1105     return false;
1106   }
1107 
1108   return true;
1109 }
1110 
1111 bool AMDGPUOperand::isInlinableImm(MVT type) const {
1112   if (!isImmTy(ImmTyNone)) {
1113     // Only plain immediates are inlinable (e.g. "clamp" attribute is not)
1114     return false;
1115   }
1116   // TODO: We should avoid using host float here. It would be better to
1117   // check the float bit values which is what a few other places do.
1118   // We've had bot failures before due to weird NaN support on mips hosts.
1119 
1120   APInt Literal(64, Imm.Val);
1121 
1122   if (Imm.IsFPImm) { // We got fp literal token
1123     if (type == MVT::f64 || type == MVT::i64) { // Expected 64-bit operand
1124       return AMDGPU::isInlinableLiteral64(Imm.Val,
1125                                           AsmParser->hasInv2PiInlineImm());
1126     }
1127 
1128     APFloat FPLiteral(APFloat::IEEEdouble(), APInt(64, Imm.Val));
1129     if (!canLosslesslyConvertToFPType(FPLiteral, type))
1130       return false;
1131 
1132     if (type.getScalarSizeInBits() == 16) {
1133       return AMDGPU::isInlinableLiteral16(
1134         static_cast<int16_t>(FPLiteral.bitcastToAPInt().getZExtValue()),
1135         AsmParser->hasInv2PiInlineImm());
1136     }
1137 
1138     // Check if single precision literal is inlinable
1139     return AMDGPU::isInlinableLiteral32(
1140       static_cast<int32_t>(FPLiteral.bitcastToAPInt().getZExtValue()),
1141       AsmParser->hasInv2PiInlineImm());
1142   }
1143 
1144   // We got int literal token.
1145   if (type == MVT::f64 || type == MVT::i64) { // Expected 64-bit operand
1146     return AMDGPU::isInlinableLiteral64(Imm.Val,
1147                                         AsmParser->hasInv2PiInlineImm());
1148   }
1149 
1150   if (type.getScalarSizeInBits() == 16) {
1151     return AMDGPU::isInlinableLiteral16(
1152       static_cast<int16_t>(Literal.getLoBits(16).getSExtValue()),
1153       AsmParser->hasInv2PiInlineImm());
1154   }
1155 
1156   return AMDGPU::isInlinableLiteral32(
1157     static_cast<int32_t>(Literal.getLoBits(32).getZExtValue()),
1158     AsmParser->hasInv2PiInlineImm());
1159 }
1160 
1161 bool AMDGPUOperand::isLiteralImm(MVT type) const {
1162   // Check that this imediate can be added as literal
1163   if (!isImmTy(ImmTyNone)) {
1164     return false;
1165   }
1166 
1167   if (!Imm.IsFPImm) {
1168     // We got int literal token.
1169 
1170     unsigned Size = type.getSizeInBits();
1171     if (Size == 64)
1172       Size = 32;
1173 
1174     // FIXME: 64-bit operands can zero extend, sign extend, or pad zeroes for FP
1175     // types.
1176     return isUIntN(Size, Imm.Val) || isIntN(Size, Imm.Val);
1177   }
1178 
1179   // We got fp literal token
1180   if (type == MVT::f64) { // Expected 64-bit fp operand
1181     // We would set low 64-bits of literal to zeroes but we accept this literals
1182     return true;
1183   }
1184 
1185   if (type == MVT::i64) { // Expected 64-bit int operand
1186     // We don't allow fp literals in 64-bit integer instructions. It is
1187     // unclear how we should encode them.
1188     return false;
1189   }
1190 
1191   APFloat FPLiteral(APFloat::IEEEdouble(), APInt(64, Imm.Val));
1192   return canLosslesslyConvertToFPType(FPLiteral, type);
1193 }
1194 
1195 bool AMDGPUOperand::isRegClass(unsigned RCID) const {
1196   return isRegKind() && AsmParser->getMRI()->getRegClass(RCID).contains(getReg());
1197 }
1198 
1199 void AMDGPUOperand::addImmOperands(MCInst &Inst, unsigned N, bool ApplyModifiers) const {
1200   int64_t Val = Imm.Val;
1201   if (isImmTy(ImmTyNone) && ApplyModifiers && Imm.Mods.hasFPModifiers() && Imm.Mods.Neg) {
1202     // Apply modifiers to immediate value. Only negate can get here
1203     if (Imm.IsFPImm) {
1204       APFloat F(BitsToDouble(Val));
1205       F.changeSign();
1206       Val = F.bitcastToAPInt().getZExtValue();
1207     } else {
1208       Val = -Val;
1209     }
1210   }
1211 
1212   if (AMDGPU::isSISrcOperand(AsmParser->getMII()->get(Inst.getOpcode()),
1213                              Inst.getNumOperands())) {
1214     addLiteralImmOperand(Inst, Val);
1215   } else {
1216     Inst.addOperand(MCOperand::createImm(Val));
1217   }
1218 }
1219 
1220 void AMDGPUOperand::addLiteralImmOperand(MCInst &Inst, int64_t Val) const {
1221   const auto& InstDesc = AsmParser->getMII()->get(Inst.getOpcode());
1222   auto OpNum = Inst.getNumOperands();
1223   // Check that this operand accepts literals
1224   assert(AMDGPU::isSISrcOperand(InstDesc, OpNum));
1225 
1226   APInt Literal(64, Val);
1227   uint8_t OpTy = InstDesc.OpInfo[OpNum].OperandType;
1228 
1229   if (Imm.IsFPImm) { // We got fp literal token
1230     switch (OpTy) {
1231     case AMDGPU::OPERAND_REG_IMM_INT64:
1232     case AMDGPU::OPERAND_REG_IMM_FP64:
1233     case AMDGPU::OPERAND_REG_INLINE_C_INT64:
1234     case AMDGPU::OPERAND_REG_INLINE_C_FP64: {
1235       if (AMDGPU::isInlinableLiteral64(Literal.getZExtValue(),
1236                                        AsmParser->hasInv2PiInlineImm())) {
1237         Inst.addOperand(MCOperand::createImm(Literal.getZExtValue()));
1238         return;
1239       }
1240 
1241       // Non-inlineable
1242       if (AMDGPU::isSISrcFPOperand(InstDesc, OpNum)) { // Expected 64-bit fp operand
1243         // For fp operands we check if low 32 bits are zeros
1244         if (Literal.getLoBits(32) != 0) {
1245           const_cast<AMDGPUAsmParser *>(AsmParser)->Warning(Inst.getLoc(),
1246           "Can't encode literal as exact 64-bit floating-point operand. "
1247           "Low 32-bits will be set to zero");
1248         }
1249 
1250         Inst.addOperand(MCOperand::createImm(Literal.lshr(32).getZExtValue()));
1251         return;
1252       }
1253 
1254       // We don't allow fp literals in 64-bit integer instructions. It is
1255       // unclear how we should encode them. This case should be checked earlier
1256       // in predicate methods (isLiteralImm())
1257       llvm_unreachable("fp literal in 64-bit integer instruction.");
1258     }
1259     case AMDGPU::OPERAND_REG_IMM_INT32:
1260     case AMDGPU::OPERAND_REG_IMM_FP32:
1261     case AMDGPU::OPERAND_REG_INLINE_C_INT32:
1262     case AMDGPU::OPERAND_REG_INLINE_C_FP32:
1263     case AMDGPU::OPERAND_REG_IMM_INT16:
1264     case AMDGPU::OPERAND_REG_IMM_FP16:
1265     case AMDGPU::OPERAND_REG_INLINE_C_INT16:
1266     case AMDGPU::OPERAND_REG_INLINE_C_FP16:
1267     case AMDGPU::OPERAND_REG_INLINE_C_V2INT16:
1268     case AMDGPU::OPERAND_REG_INLINE_C_V2FP16: {
1269       bool lost;
1270       APFloat FPLiteral(APFloat::IEEEdouble(), Literal);
1271       // Convert literal to single precision
1272       FPLiteral.convert(*getOpFltSemantics(OpTy),
1273                         APFloat::rmNearestTiesToEven, &lost);
1274       // We allow precision lost but not overflow or underflow. This should be
1275       // checked earlier in isLiteralImm()
1276 
1277       uint64_t ImmVal = FPLiteral.bitcastToAPInt().getZExtValue();
1278       if (OpTy == AMDGPU::OPERAND_REG_INLINE_C_V2INT16 ||
1279           OpTy == AMDGPU::OPERAND_REG_INLINE_C_V2FP16) {
1280         ImmVal |= (ImmVal << 16);
1281       }
1282 
1283       Inst.addOperand(MCOperand::createImm(ImmVal));
1284       return;
1285     }
1286     default:
1287       llvm_unreachable("invalid operand size");
1288     }
1289 
1290     return;
1291   }
1292 
1293    // We got int literal token.
1294   // Only sign extend inline immediates.
1295   // FIXME: No errors on truncation
1296   switch (OpTy) {
1297   case AMDGPU::OPERAND_REG_IMM_INT32:
1298   case AMDGPU::OPERAND_REG_IMM_FP32:
1299   case AMDGPU::OPERAND_REG_INLINE_C_INT32:
1300   case AMDGPU::OPERAND_REG_INLINE_C_FP32: {
1301     if (isInt<32>(Val) &&
1302         AMDGPU::isInlinableLiteral32(static_cast<int32_t>(Val),
1303                                      AsmParser->hasInv2PiInlineImm())) {
1304       Inst.addOperand(MCOperand::createImm(Val));
1305       return;
1306     }
1307 
1308     Inst.addOperand(MCOperand::createImm(Val & 0xffffffff));
1309     return;
1310   }
1311   case AMDGPU::OPERAND_REG_IMM_INT64:
1312   case AMDGPU::OPERAND_REG_IMM_FP64:
1313   case AMDGPU::OPERAND_REG_INLINE_C_INT64:
1314   case AMDGPU::OPERAND_REG_INLINE_C_FP64: {
1315     if (AMDGPU::isInlinableLiteral64(Val, AsmParser->hasInv2PiInlineImm())) {
1316       Inst.addOperand(MCOperand::createImm(Val));
1317       return;
1318     }
1319 
1320     Inst.addOperand(MCOperand::createImm(Lo_32(Val)));
1321     return;
1322   }
1323   case AMDGPU::OPERAND_REG_IMM_INT16:
1324   case AMDGPU::OPERAND_REG_IMM_FP16:
1325   case AMDGPU::OPERAND_REG_INLINE_C_INT16:
1326   case AMDGPU::OPERAND_REG_INLINE_C_FP16: {
1327     if (isInt<16>(Val) &&
1328         AMDGPU::isInlinableLiteral16(static_cast<int16_t>(Val),
1329                                      AsmParser->hasInv2PiInlineImm())) {
1330       Inst.addOperand(MCOperand::createImm(Val));
1331       return;
1332     }
1333 
1334     Inst.addOperand(MCOperand::createImm(Val & 0xffff));
1335     return;
1336   }
1337   case AMDGPU::OPERAND_REG_INLINE_C_V2INT16:
1338   case AMDGPU::OPERAND_REG_INLINE_C_V2FP16: {
1339     auto LiteralVal = static_cast<uint16_t>(Literal.getLoBits(16).getZExtValue());
1340     assert(AMDGPU::isInlinableLiteral16(LiteralVal,
1341                                         AsmParser->hasInv2PiInlineImm()));
1342 
1343     uint32_t ImmVal = static_cast<uint32_t>(LiteralVal) << 16 |
1344                       static_cast<uint32_t>(LiteralVal);
1345     Inst.addOperand(MCOperand::createImm(ImmVal));
1346     return;
1347   }
1348   default:
1349     llvm_unreachable("invalid operand size");
1350   }
1351 }
1352 
1353 template <unsigned Bitwidth>
1354 void AMDGPUOperand::addKImmFPOperands(MCInst &Inst, unsigned N) const {
1355   APInt Literal(64, Imm.Val);
1356 
1357   if (!Imm.IsFPImm) {
1358     // We got int literal token.
1359     Inst.addOperand(MCOperand::createImm(Literal.getLoBits(Bitwidth).getZExtValue()));
1360     return;
1361   }
1362 
1363   bool Lost;
1364   APFloat FPLiteral(APFloat::IEEEdouble(), Literal);
1365   FPLiteral.convert(*getFltSemantics(Bitwidth / 8),
1366                     APFloat::rmNearestTiesToEven, &Lost);
1367   Inst.addOperand(MCOperand::createImm(FPLiteral.bitcastToAPInt().getZExtValue()));
1368 }
1369 
1370 void AMDGPUOperand::addRegOperands(MCInst &Inst, unsigned N) const {
1371   Inst.addOperand(MCOperand::createReg(AMDGPU::getMCReg(getReg(), AsmParser->getSTI())));
1372 }
1373 
1374 //===----------------------------------------------------------------------===//
1375 // AsmParser
1376 //===----------------------------------------------------------------------===//
1377 
1378 static int getRegClass(RegisterKind Is, unsigned RegWidth) {
1379   if (Is == IS_VGPR) {
1380     switch (RegWidth) {
1381       default: return -1;
1382       case 1: return AMDGPU::VGPR_32RegClassID;
1383       case 2: return AMDGPU::VReg_64RegClassID;
1384       case 3: return AMDGPU::VReg_96RegClassID;
1385       case 4: return AMDGPU::VReg_128RegClassID;
1386       case 8: return AMDGPU::VReg_256RegClassID;
1387       case 16: return AMDGPU::VReg_512RegClassID;
1388     }
1389   } else if (Is == IS_TTMP) {
1390     switch (RegWidth) {
1391       default: return -1;
1392       case 1: return AMDGPU::TTMP_32RegClassID;
1393       case 2: return AMDGPU::TTMP_64RegClassID;
1394       case 4: return AMDGPU::TTMP_128RegClassID;
1395     }
1396   } else if (Is == IS_SGPR) {
1397     switch (RegWidth) {
1398       default: return -1;
1399       case 1: return AMDGPU::SGPR_32RegClassID;
1400       case 2: return AMDGPU::SGPR_64RegClassID;
1401       case 4: return AMDGPU::SGPR_128RegClassID;
1402       case 8: return AMDGPU::SReg_256RegClassID;
1403       case 16: return AMDGPU::SReg_512RegClassID;
1404     }
1405   }
1406   return -1;
1407 }
1408 
1409 static unsigned getSpecialRegForName(StringRef RegName) {
1410   return StringSwitch<unsigned>(RegName)
1411     .Case("exec", AMDGPU::EXEC)
1412     .Case("vcc", AMDGPU::VCC)
1413     .Case("flat_scratch", AMDGPU::FLAT_SCR)
1414     .Case("m0", AMDGPU::M0)
1415     .Case("scc", AMDGPU::SCC)
1416     .Case("tba", AMDGPU::TBA)
1417     .Case("tma", AMDGPU::TMA)
1418     .Case("flat_scratch_lo", AMDGPU::FLAT_SCR_LO)
1419     .Case("flat_scratch_hi", AMDGPU::FLAT_SCR_HI)
1420     .Case("vcc_lo", AMDGPU::VCC_LO)
1421     .Case("vcc_hi", AMDGPU::VCC_HI)
1422     .Case("exec_lo", AMDGPU::EXEC_LO)
1423     .Case("exec_hi", AMDGPU::EXEC_HI)
1424     .Case("tma_lo", AMDGPU::TMA_LO)
1425     .Case("tma_hi", AMDGPU::TMA_HI)
1426     .Case("tba_lo", AMDGPU::TBA_LO)
1427     .Case("tba_hi", AMDGPU::TBA_HI)
1428     .Default(0);
1429 }
1430 
1431 bool AMDGPUAsmParser::ParseRegister(unsigned &RegNo, SMLoc &StartLoc,
1432                                     SMLoc &EndLoc) {
1433   auto R = parseRegister();
1434   if (!R) return true;
1435   assert(R->isReg());
1436   RegNo = R->getReg();
1437   StartLoc = R->getStartLoc();
1438   EndLoc = R->getEndLoc();
1439   return false;
1440 }
1441 
1442 bool AMDGPUAsmParser::AddNextRegisterToList(unsigned &Reg, unsigned &RegWidth,
1443                                             RegisterKind RegKind, unsigned Reg1,
1444                                             unsigned RegNum) {
1445   switch (RegKind) {
1446   case IS_SPECIAL:
1447     if (Reg == AMDGPU::EXEC_LO && Reg1 == AMDGPU::EXEC_HI) {
1448       Reg = AMDGPU::EXEC;
1449       RegWidth = 2;
1450       return true;
1451     }
1452     if (Reg == AMDGPU::FLAT_SCR_LO && Reg1 == AMDGPU::FLAT_SCR_HI) {
1453       Reg = AMDGPU::FLAT_SCR;
1454       RegWidth = 2;
1455       return true;
1456     }
1457     if (Reg == AMDGPU::VCC_LO && Reg1 == AMDGPU::VCC_HI) {
1458       Reg = AMDGPU::VCC;
1459       RegWidth = 2;
1460       return true;
1461     }
1462     if (Reg == AMDGPU::TBA_LO && Reg1 == AMDGPU::TBA_HI) {
1463       Reg = AMDGPU::TBA;
1464       RegWidth = 2;
1465       return true;
1466     }
1467     if (Reg == AMDGPU::TMA_LO && Reg1 == AMDGPU::TMA_HI) {
1468       Reg = AMDGPU::TMA;
1469       RegWidth = 2;
1470       return true;
1471     }
1472     return false;
1473   case IS_VGPR:
1474   case IS_SGPR:
1475   case IS_TTMP:
1476     if (Reg1 != Reg + RegWidth) {
1477       return false;
1478     }
1479     RegWidth++;
1480     return true;
1481   default:
1482     llvm_unreachable("unexpected register kind");
1483   }
1484 }
1485 
1486 bool AMDGPUAsmParser::ParseAMDGPURegister(RegisterKind &RegKind, unsigned &Reg,
1487                                           unsigned &RegNum, unsigned &RegWidth,
1488                                           unsigned *DwordRegIndex) {
1489   if (DwordRegIndex) { *DwordRegIndex = 0; }
1490   const MCRegisterInfo *TRI = getContext().getRegisterInfo();
1491   if (getLexer().is(AsmToken::Identifier)) {
1492     StringRef RegName = Parser.getTok().getString();
1493     if ((Reg = getSpecialRegForName(RegName))) {
1494       Parser.Lex();
1495       RegKind = IS_SPECIAL;
1496     } else {
1497       unsigned RegNumIndex = 0;
1498       if (RegName[0] == 'v') {
1499         RegNumIndex = 1;
1500         RegKind = IS_VGPR;
1501       } else if (RegName[0] == 's') {
1502         RegNumIndex = 1;
1503         RegKind = IS_SGPR;
1504       } else if (RegName.startswith("ttmp")) {
1505         RegNumIndex = strlen("ttmp");
1506         RegKind = IS_TTMP;
1507       } else {
1508         return false;
1509       }
1510       if (RegName.size() > RegNumIndex) {
1511         // Single 32-bit register: vXX.
1512         if (RegName.substr(RegNumIndex).getAsInteger(10, RegNum))
1513           return false;
1514         Parser.Lex();
1515         RegWidth = 1;
1516       } else {
1517         // Range of registers: v[XX:YY]. ":YY" is optional.
1518         Parser.Lex();
1519         int64_t RegLo, RegHi;
1520         if (getLexer().isNot(AsmToken::LBrac))
1521           return false;
1522         Parser.Lex();
1523 
1524         if (getParser().parseAbsoluteExpression(RegLo))
1525           return false;
1526 
1527         const bool isRBrace = getLexer().is(AsmToken::RBrac);
1528         if (!isRBrace && getLexer().isNot(AsmToken::Colon))
1529           return false;
1530         Parser.Lex();
1531 
1532         if (isRBrace) {
1533           RegHi = RegLo;
1534         } else {
1535           if (getParser().parseAbsoluteExpression(RegHi))
1536             return false;
1537 
1538           if (getLexer().isNot(AsmToken::RBrac))
1539             return false;
1540           Parser.Lex();
1541         }
1542         RegNum = (unsigned) RegLo;
1543         RegWidth = (RegHi - RegLo) + 1;
1544       }
1545     }
1546   } else if (getLexer().is(AsmToken::LBrac)) {
1547     // List of consecutive registers: [s0,s1,s2,s3]
1548     Parser.Lex();
1549     if (!ParseAMDGPURegister(RegKind, Reg, RegNum, RegWidth, nullptr))
1550       return false;
1551     if (RegWidth != 1)
1552       return false;
1553     RegisterKind RegKind1;
1554     unsigned Reg1, RegNum1, RegWidth1;
1555     do {
1556       if (getLexer().is(AsmToken::Comma)) {
1557         Parser.Lex();
1558       } else if (getLexer().is(AsmToken::RBrac)) {
1559         Parser.Lex();
1560         break;
1561       } else if (ParseAMDGPURegister(RegKind1, Reg1, RegNum1, RegWidth1, nullptr)) {
1562         if (RegWidth1 != 1) {
1563           return false;
1564         }
1565         if (RegKind1 != RegKind) {
1566           return false;
1567         }
1568         if (!AddNextRegisterToList(Reg, RegWidth, RegKind1, Reg1, RegNum1)) {
1569           return false;
1570         }
1571       } else {
1572         return false;
1573       }
1574     } while (true);
1575   } else {
1576     return false;
1577   }
1578   switch (RegKind) {
1579   case IS_SPECIAL:
1580     RegNum = 0;
1581     RegWidth = 1;
1582     break;
1583   case IS_VGPR:
1584   case IS_SGPR:
1585   case IS_TTMP:
1586   {
1587     unsigned Size = 1;
1588     if (RegKind == IS_SGPR || RegKind == IS_TTMP) {
1589       // SGPR and TTMP registers must be aligned. Max required alignment is 4 dwords.
1590       Size = std::min(RegWidth, 4u);
1591     }
1592     if (RegNum % Size != 0)
1593       return false;
1594     if (DwordRegIndex) { *DwordRegIndex = RegNum; }
1595     RegNum = RegNum / Size;
1596     int RCID = getRegClass(RegKind, RegWidth);
1597     if (RCID == -1)
1598       return false;
1599     const MCRegisterClass RC = TRI->getRegClass(RCID);
1600     if (RegNum >= RC.getNumRegs())
1601       return false;
1602     Reg = RC.getRegister(RegNum);
1603     break;
1604   }
1605 
1606   default:
1607     llvm_unreachable("unexpected register kind");
1608   }
1609 
1610   if (!subtargetHasRegister(*TRI, Reg))
1611     return false;
1612   return true;
1613 }
1614 
1615 std::unique_ptr<AMDGPUOperand> AMDGPUAsmParser::parseRegister() {
1616   const auto &Tok = Parser.getTok();
1617   SMLoc StartLoc = Tok.getLoc();
1618   SMLoc EndLoc = Tok.getEndLoc();
1619   RegisterKind RegKind;
1620   unsigned Reg, RegNum, RegWidth, DwordRegIndex;
1621 
1622   if (!ParseAMDGPURegister(RegKind, Reg, RegNum, RegWidth, &DwordRegIndex)) {
1623     return nullptr;
1624   }
1625   KernelScope.usesRegister(RegKind, DwordRegIndex, RegWidth);
1626   return AMDGPUOperand::CreateReg(this, Reg, StartLoc, EndLoc, false);
1627 }
1628 
1629 OperandMatchResultTy
1630 AMDGPUAsmParser::parseImm(OperandVector &Operands) {
1631   // TODO: add syntactic sugar for 1/(2*PI)
1632   bool Minus = false;
1633   if (getLexer().getKind() == AsmToken::Minus) {
1634     Minus = true;
1635     Parser.Lex();
1636   }
1637 
1638   SMLoc S = Parser.getTok().getLoc();
1639   switch(getLexer().getKind()) {
1640   case AsmToken::Integer: {
1641     int64_t IntVal;
1642     if (getParser().parseAbsoluteExpression(IntVal))
1643       return MatchOperand_ParseFail;
1644     if (Minus)
1645       IntVal *= -1;
1646     Operands.push_back(AMDGPUOperand::CreateImm(this, IntVal, S));
1647     return MatchOperand_Success;
1648   }
1649   case AsmToken::Real: {
1650     int64_t IntVal;
1651     if (getParser().parseAbsoluteExpression(IntVal))
1652       return MatchOperand_ParseFail;
1653 
1654     APFloat F(BitsToDouble(IntVal));
1655     if (Minus)
1656       F.changeSign();
1657     Operands.push_back(
1658         AMDGPUOperand::CreateImm(this, F.bitcastToAPInt().getZExtValue(), S,
1659                                  AMDGPUOperand::ImmTyNone, true));
1660     return MatchOperand_Success;
1661   }
1662   default:
1663     return Minus ? MatchOperand_ParseFail : MatchOperand_NoMatch;
1664   }
1665 }
1666 
1667 OperandMatchResultTy
1668 AMDGPUAsmParser::parseReg(OperandVector &Operands) {
1669   if (auto R = parseRegister()) {
1670     assert(R->isReg());
1671     R->Reg.IsForcedVOP3 = isForcedVOP3();
1672     Operands.push_back(std::move(R));
1673     return MatchOperand_Success;
1674   }
1675   return MatchOperand_NoMatch;
1676 }
1677 
1678 OperandMatchResultTy
1679 AMDGPUAsmParser::parseRegOrImm(OperandVector &Operands) {
1680   auto res = parseImm(Operands);
1681   if (res != MatchOperand_NoMatch) {
1682     return res;
1683   }
1684 
1685   return parseReg(Operands);
1686 }
1687 
1688 OperandMatchResultTy
1689 AMDGPUAsmParser::parseRegOrImmWithFPInputMods(OperandVector &Operands,
1690                                               bool AllowImm) {
1691   // XXX: During parsing we can't determine if minus sign means
1692   // negate-modifier or negative immediate value.
1693   // By default we suppose it is modifier.
1694   bool Negate = false, Abs = false, Abs2 = false;
1695 
1696   if (getLexer().getKind()== AsmToken::Minus) {
1697     Parser.Lex();
1698     Negate = true;
1699   }
1700 
1701   if (getLexer().getKind() == AsmToken::Identifier &&
1702       Parser.getTok().getString() == "abs") {
1703     Parser.Lex();
1704     Abs2 = true;
1705     if (getLexer().isNot(AsmToken::LParen)) {
1706       Error(Parser.getTok().getLoc(), "expected left paren after abs");
1707       return MatchOperand_ParseFail;
1708     }
1709     Parser.Lex();
1710   }
1711 
1712   if (getLexer().getKind() == AsmToken::Pipe) {
1713     if (Abs2) {
1714       Error(Parser.getTok().getLoc(), "expected register or immediate");
1715       return MatchOperand_ParseFail;
1716     }
1717     Parser.Lex();
1718     Abs = true;
1719   }
1720 
1721   OperandMatchResultTy Res;
1722   if (AllowImm) {
1723     Res = parseRegOrImm(Operands);
1724   } else {
1725     Res = parseReg(Operands);
1726   }
1727   if (Res != MatchOperand_Success) {
1728     return Res;
1729   }
1730 
1731   AMDGPUOperand::Modifiers Mods;
1732   if (Negate) {
1733     Mods.Neg = true;
1734   }
1735   if (Abs) {
1736     if (getLexer().getKind() != AsmToken::Pipe) {
1737       Error(Parser.getTok().getLoc(), "expected vertical bar");
1738       return MatchOperand_ParseFail;
1739     }
1740     Parser.Lex();
1741     Mods.Abs = true;
1742   }
1743   if (Abs2) {
1744     if (getLexer().isNot(AsmToken::RParen)) {
1745       Error(Parser.getTok().getLoc(), "expected closing parentheses");
1746       return MatchOperand_ParseFail;
1747     }
1748     Parser.Lex();
1749     Mods.Abs = true;
1750   }
1751 
1752   if (Mods.hasFPModifiers()) {
1753     AMDGPUOperand &Op = static_cast<AMDGPUOperand &>(*Operands.back());
1754     Op.setModifiers(Mods);
1755   }
1756   return MatchOperand_Success;
1757 }
1758 
1759 OperandMatchResultTy
1760 AMDGPUAsmParser::parseRegOrImmWithIntInputMods(OperandVector &Operands,
1761                                                bool AllowImm) {
1762   bool Sext = false;
1763 
1764   if (getLexer().getKind() == AsmToken::Identifier &&
1765       Parser.getTok().getString() == "sext") {
1766     Parser.Lex();
1767     Sext = true;
1768     if (getLexer().isNot(AsmToken::LParen)) {
1769       Error(Parser.getTok().getLoc(), "expected left paren after sext");
1770       return MatchOperand_ParseFail;
1771     }
1772     Parser.Lex();
1773   }
1774 
1775   OperandMatchResultTy Res;
1776   if (AllowImm) {
1777     Res = parseRegOrImm(Operands);
1778   } else {
1779     Res = parseReg(Operands);
1780   }
1781   if (Res != MatchOperand_Success) {
1782     return Res;
1783   }
1784 
1785   AMDGPUOperand::Modifiers Mods;
1786   if (Sext) {
1787     if (getLexer().isNot(AsmToken::RParen)) {
1788       Error(Parser.getTok().getLoc(), "expected closing parentheses");
1789       return MatchOperand_ParseFail;
1790     }
1791     Parser.Lex();
1792     Mods.Sext = true;
1793   }
1794 
1795   if (Mods.hasIntModifiers()) {
1796     AMDGPUOperand &Op = static_cast<AMDGPUOperand &>(*Operands.back());
1797     Op.setModifiers(Mods);
1798   }
1799 
1800   return MatchOperand_Success;
1801 }
1802 
1803 OperandMatchResultTy
1804 AMDGPUAsmParser::parseRegWithFPInputMods(OperandVector &Operands) {
1805   return parseRegOrImmWithFPInputMods(Operands, false);
1806 }
1807 
1808 OperandMatchResultTy
1809 AMDGPUAsmParser::parseRegWithIntInputMods(OperandVector &Operands) {
1810   return parseRegOrImmWithIntInputMods(Operands, false);
1811 }
1812 
1813 OperandMatchResultTy AMDGPUAsmParser::parseVReg32OrOff(OperandVector &Operands) {
1814   std::unique_ptr<AMDGPUOperand> Reg = parseRegister();
1815   if (Reg) {
1816     Operands.push_back(std::move(Reg));
1817     return MatchOperand_Success;
1818   }
1819 
1820   const AsmToken &Tok = Parser.getTok();
1821   if (Tok.getString() == "off") {
1822     Operands.push_back(AMDGPUOperand::CreateImm(this, 0, Tok.getLoc(),
1823                                                 AMDGPUOperand::ImmTyOff, false));
1824     Parser.Lex();
1825     return MatchOperand_Success;
1826   }
1827 
1828   return MatchOperand_NoMatch;
1829 }
1830 
1831 unsigned AMDGPUAsmParser::checkTargetMatchPredicate(MCInst &Inst) {
1832   uint64_t TSFlags = MII.get(Inst.getOpcode()).TSFlags;
1833 
1834   if ((getForcedEncodingSize() == 32 && (TSFlags & SIInstrFlags::VOP3)) ||
1835       (getForcedEncodingSize() == 64 && !(TSFlags & SIInstrFlags::VOP3)) ||
1836       (isForcedDPP() && !(TSFlags & SIInstrFlags::DPP)) ||
1837       (isForcedSDWA() && !(TSFlags & SIInstrFlags::SDWA)) )
1838     return Match_InvalidOperand;
1839 
1840   if ((TSFlags & SIInstrFlags::VOP3) &&
1841       (TSFlags & SIInstrFlags::VOPAsmPrefer32Bit) &&
1842       getForcedEncodingSize() != 64)
1843     return Match_PreferE32;
1844 
1845   if (Inst.getOpcode() == AMDGPU::V_MAC_F32_sdwa_vi ||
1846       Inst.getOpcode() == AMDGPU::V_MAC_F16_sdwa_vi) {
1847     // v_mac_f32/16 allow only dst_sel == DWORD;
1848     auto OpNum =
1849         AMDGPU::getNamedOperandIdx(Inst.getOpcode(), AMDGPU::OpName::dst_sel);
1850     const auto &Op = Inst.getOperand(OpNum);
1851     if (!Op.isImm() || Op.getImm() != AMDGPU::SDWA::SdwaSel::DWORD) {
1852       return Match_InvalidOperand;
1853     }
1854   }
1855 
1856   return Match_Success;
1857 }
1858 
1859 // What asm variants we should check
1860 ArrayRef<unsigned> AMDGPUAsmParser::getMatchedVariants() const {
1861   if (getForcedEncodingSize() == 32) {
1862     static const unsigned Variants[] = {AMDGPUAsmVariants::DEFAULT};
1863     return makeArrayRef(Variants);
1864   }
1865 
1866   if (isForcedVOP3()) {
1867     static const unsigned Variants[] = {AMDGPUAsmVariants::VOP3};
1868     return makeArrayRef(Variants);
1869   }
1870 
1871   if (isForcedSDWA()) {
1872     static const unsigned Variants[] = {AMDGPUAsmVariants::SDWA};
1873     return makeArrayRef(Variants);
1874   }
1875 
1876   if (isForcedDPP()) {
1877     static const unsigned Variants[] = {AMDGPUAsmVariants::DPP};
1878     return makeArrayRef(Variants);
1879   }
1880 
1881   static const unsigned Variants[] = {
1882     AMDGPUAsmVariants::DEFAULT, AMDGPUAsmVariants::VOP3,
1883     AMDGPUAsmVariants::SDWA, AMDGPUAsmVariants::DPP
1884   };
1885 
1886   return makeArrayRef(Variants);
1887 }
1888 
1889 bool AMDGPUAsmParser::MatchAndEmitInstruction(SMLoc IDLoc, unsigned &Opcode,
1890                                               OperandVector &Operands,
1891                                               MCStreamer &Out,
1892                                               uint64_t &ErrorInfo,
1893                                               bool MatchingInlineAsm) {
1894   MCInst Inst;
1895   unsigned Result = Match_Success;
1896   for (auto Variant : getMatchedVariants()) {
1897     uint64_t EI;
1898     auto R = MatchInstructionImpl(Operands, Inst, EI, MatchingInlineAsm,
1899                                   Variant);
1900     // We order match statuses from least to most specific. We use most specific
1901     // status as resulting
1902     // Match_MnemonicFail < Match_InvalidOperand < Match_MissingFeature < Match_PreferE32
1903     if ((R == Match_Success) ||
1904         (R == Match_PreferE32) ||
1905         (R == Match_MissingFeature && Result != Match_PreferE32) ||
1906         (R == Match_InvalidOperand && Result != Match_MissingFeature
1907                                    && Result != Match_PreferE32) ||
1908         (R == Match_MnemonicFail   && Result != Match_InvalidOperand
1909                                    && Result != Match_MissingFeature
1910                                    && Result != Match_PreferE32)) {
1911       Result = R;
1912       ErrorInfo = EI;
1913     }
1914     if (R == Match_Success)
1915       break;
1916   }
1917 
1918   switch (Result) {
1919   default: break;
1920   case Match_Success:
1921     Inst.setLoc(IDLoc);
1922     Out.EmitInstruction(Inst, getSTI());
1923     return false;
1924 
1925   case Match_MissingFeature:
1926     return Error(IDLoc, "instruction not supported on this GPU");
1927 
1928   case Match_MnemonicFail:
1929     return Error(IDLoc, "unrecognized instruction mnemonic");
1930 
1931   case Match_InvalidOperand: {
1932     SMLoc ErrorLoc = IDLoc;
1933     if (ErrorInfo != ~0ULL) {
1934       if (ErrorInfo >= Operands.size()) {
1935         return Error(IDLoc, "too few operands for instruction");
1936       }
1937       ErrorLoc = ((AMDGPUOperand &)*Operands[ErrorInfo]).getStartLoc();
1938       if (ErrorLoc == SMLoc())
1939         ErrorLoc = IDLoc;
1940     }
1941     return Error(ErrorLoc, "invalid operand for instruction");
1942   }
1943 
1944   case Match_PreferE32:
1945     return Error(IDLoc, "internal error: instruction without _e64 suffix "
1946                         "should be encoded as e32");
1947   }
1948   llvm_unreachable("Implement any new match types added!");
1949 }
1950 
1951 bool AMDGPUAsmParser::ParseAsAbsoluteExpression(uint32_t &Ret) {
1952   int64_t Tmp = -1;
1953   if (getLexer().isNot(AsmToken::Integer) && getLexer().isNot(AsmToken::Identifier)) {
1954     return true;
1955   }
1956   if (getParser().parseAbsoluteExpression(Tmp)) {
1957     return true;
1958   }
1959   Ret = static_cast<uint32_t>(Tmp);
1960   return false;
1961 }
1962 
1963 bool AMDGPUAsmParser::ParseDirectiveMajorMinor(uint32_t &Major,
1964                                                uint32_t &Minor) {
1965   if (ParseAsAbsoluteExpression(Major))
1966     return TokError("invalid major version");
1967 
1968   if (getLexer().isNot(AsmToken::Comma))
1969     return TokError("minor version number required, comma expected");
1970   Lex();
1971 
1972   if (ParseAsAbsoluteExpression(Minor))
1973     return TokError("invalid minor version");
1974 
1975   return false;
1976 }
1977 
1978 bool AMDGPUAsmParser::ParseDirectiveHSACodeObjectVersion() {
1979   uint32_t Major;
1980   uint32_t Minor;
1981 
1982   if (ParseDirectiveMajorMinor(Major, Minor))
1983     return true;
1984 
1985   getTargetStreamer().EmitDirectiveHSACodeObjectVersion(Major, Minor);
1986   return false;
1987 }
1988 
1989 bool AMDGPUAsmParser::ParseDirectiveHSACodeObjectISA() {
1990   uint32_t Major;
1991   uint32_t Minor;
1992   uint32_t Stepping;
1993   StringRef VendorName;
1994   StringRef ArchName;
1995 
1996   // If this directive has no arguments, then use the ISA version for the
1997   // targeted GPU.
1998   if (getLexer().is(AsmToken::EndOfStatement)) {
1999     AMDGPU::IsaInfo::IsaVersion ISA =
2000         AMDGPU::IsaInfo::getIsaVersion(getFeatureBits());
2001     getTargetStreamer().EmitDirectiveHSACodeObjectISA(ISA.Major, ISA.Minor,
2002                                                       ISA.Stepping,
2003                                                       "AMD", "AMDGPU");
2004     return false;
2005   }
2006 
2007   if (ParseDirectiveMajorMinor(Major, Minor))
2008     return true;
2009 
2010   if (getLexer().isNot(AsmToken::Comma))
2011     return TokError("stepping version number required, comma expected");
2012   Lex();
2013 
2014   if (ParseAsAbsoluteExpression(Stepping))
2015     return TokError("invalid stepping version");
2016 
2017   if (getLexer().isNot(AsmToken::Comma))
2018     return TokError("vendor name required, comma expected");
2019   Lex();
2020 
2021   if (getLexer().isNot(AsmToken::String))
2022     return TokError("invalid vendor name");
2023 
2024   VendorName = getLexer().getTok().getStringContents();
2025   Lex();
2026 
2027   if (getLexer().isNot(AsmToken::Comma))
2028     return TokError("arch name required, comma expected");
2029   Lex();
2030 
2031   if (getLexer().isNot(AsmToken::String))
2032     return TokError("invalid arch name");
2033 
2034   ArchName = getLexer().getTok().getStringContents();
2035   Lex();
2036 
2037   getTargetStreamer().EmitDirectiveHSACodeObjectISA(Major, Minor, Stepping,
2038                                                     VendorName, ArchName);
2039   return false;
2040 }
2041 
2042 bool AMDGPUAsmParser::ParseDirectiveRuntimeMetadata() {
2043   std::string Metadata;
2044   raw_string_ostream MS(Metadata);
2045 
2046   getLexer().setSkipSpace(false);
2047 
2048   bool FoundEnd = false;
2049   while (!getLexer().is(AsmToken::Eof)) {
2050     while (getLexer().is(AsmToken::Space)) {
2051       MS << ' ';
2052       Lex();
2053     }
2054 
2055     if (getLexer().is(AsmToken::Identifier)) {
2056       StringRef ID = getLexer().getTok().getIdentifier();
2057       if (ID == ".end_amdgpu_runtime_metadata") {
2058         Lex();
2059         FoundEnd = true;
2060         break;
2061       }
2062     }
2063 
2064     MS << Parser.parseStringToEndOfStatement()
2065        << getContext().getAsmInfo()->getSeparatorString();
2066 
2067     Parser.eatToEndOfStatement();
2068   }
2069 
2070   getLexer().setSkipSpace(true);
2071 
2072   if (getLexer().is(AsmToken::Eof) && !FoundEnd)
2073     return TokError("expected directive .end_amdgpu_runtime_metadata not found");
2074 
2075   MS.flush();
2076 
2077   if (getTargetStreamer().EmitRuntimeMetadata(getFeatureBits(), Metadata))
2078     return Error(getParser().getTok().getLoc(), "invalid runtime metadata");
2079 
2080   return false;
2081 }
2082 
2083 bool AMDGPUAsmParser::ParseAMDKernelCodeTValue(StringRef ID,
2084                                                amd_kernel_code_t &Header) {
2085   SmallString<40> ErrStr;
2086   raw_svector_ostream Err(ErrStr);
2087   if (!parseAmdKernelCodeField(ID, getParser(), Header, Err)) {
2088     return TokError(Err.str());
2089   }
2090   Lex();
2091   return false;
2092 }
2093 
2094 bool AMDGPUAsmParser::ParseDirectiveAMDKernelCodeT() {
2095   amd_kernel_code_t Header;
2096   AMDGPU::initDefaultAMDKernelCodeT(Header, getFeatureBits());
2097 
2098   while (true) {
2099     // Lex EndOfStatement.  This is in a while loop, because lexing a comment
2100     // will set the current token to EndOfStatement.
2101     while(getLexer().is(AsmToken::EndOfStatement))
2102       Lex();
2103 
2104     if (getLexer().isNot(AsmToken::Identifier))
2105       return TokError("expected value identifier or .end_amd_kernel_code_t");
2106 
2107     StringRef ID = getLexer().getTok().getIdentifier();
2108     Lex();
2109 
2110     if (ID == ".end_amd_kernel_code_t")
2111       break;
2112 
2113     if (ParseAMDKernelCodeTValue(ID, Header))
2114       return true;
2115   }
2116 
2117   getTargetStreamer().EmitAMDKernelCodeT(Header);
2118 
2119   return false;
2120 }
2121 
2122 bool AMDGPUAsmParser::ParseSectionDirectiveHSAText() {
2123   getParser().getStreamer().SwitchSection(
2124       AMDGPU::getHSATextSection(getContext()));
2125   return false;
2126 }
2127 
2128 bool AMDGPUAsmParser::ParseDirectiveAMDGPUHsaKernel() {
2129   if (getLexer().isNot(AsmToken::Identifier))
2130     return TokError("expected symbol name");
2131 
2132   StringRef KernelName = Parser.getTok().getString();
2133 
2134   getTargetStreamer().EmitAMDGPUSymbolType(KernelName,
2135                                            ELF::STT_AMDGPU_HSA_KERNEL);
2136   Lex();
2137   KernelScope.initialize(getContext());
2138   return false;
2139 }
2140 
2141 bool AMDGPUAsmParser::ParseDirectiveAMDGPUHsaModuleGlobal() {
2142   if (getLexer().isNot(AsmToken::Identifier))
2143     return TokError("expected symbol name");
2144 
2145   StringRef GlobalName = Parser.getTok().getIdentifier();
2146 
2147   getTargetStreamer().EmitAMDGPUHsaModuleScopeGlobal(GlobalName);
2148   Lex();
2149   return false;
2150 }
2151 
2152 bool AMDGPUAsmParser::ParseDirectiveAMDGPUHsaProgramGlobal() {
2153   if (getLexer().isNot(AsmToken::Identifier))
2154     return TokError("expected symbol name");
2155 
2156   StringRef GlobalName = Parser.getTok().getIdentifier();
2157 
2158   getTargetStreamer().EmitAMDGPUHsaProgramScopeGlobal(GlobalName);
2159   Lex();
2160   return false;
2161 }
2162 
2163 bool AMDGPUAsmParser::ParseSectionDirectiveHSADataGlobalAgent() {
2164   getParser().getStreamer().SwitchSection(
2165       AMDGPU::getHSADataGlobalAgentSection(getContext()));
2166   return false;
2167 }
2168 
2169 bool AMDGPUAsmParser::ParseSectionDirectiveHSADataGlobalProgram() {
2170   getParser().getStreamer().SwitchSection(
2171       AMDGPU::getHSADataGlobalProgramSection(getContext()));
2172   return false;
2173 }
2174 
2175 bool AMDGPUAsmParser::ParseSectionDirectiveHSARodataReadonlyAgent() {
2176   getParser().getStreamer().SwitchSection(
2177       AMDGPU::getHSARodataReadonlyAgentSection(getContext()));
2178   return false;
2179 }
2180 
2181 bool AMDGPUAsmParser::ParseDirective(AsmToken DirectiveID) {
2182   StringRef IDVal = DirectiveID.getString();
2183 
2184   if (IDVal == ".hsa_code_object_version")
2185     return ParseDirectiveHSACodeObjectVersion();
2186 
2187   if (IDVal == ".hsa_code_object_isa")
2188     return ParseDirectiveHSACodeObjectISA();
2189 
2190   if (IDVal == ".amdgpu_runtime_metadata")
2191     return ParseDirectiveRuntimeMetadata();
2192 
2193   if (IDVal == ".amd_kernel_code_t")
2194     return ParseDirectiveAMDKernelCodeT();
2195 
2196   if (IDVal == ".hsatext")
2197     return ParseSectionDirectiveHSAText();
2198 
2199   if (IDVal == ".amdgpu_hsa_kernel")
2200     return ParseDirectiveAMDGPUHsaKernel();
2201 
2202   if (IDVal == ".amdgpu_hsa_module_global")
2203     return ParseDirectiveAMDGPUHsaModuleGlobal();
2204 
2205   if (IDVal == ".amdgpu_hsa_program_global")
2206     return ParseDirectiveAMDGPUHsaProgramGlobal();
2207 
2208   if (IDVal == ".hsadata_global_agent")
2209     return ParseSectionDirectiveHSADataGlobalAgent();
2210 
2211   if (IDVal == ".hsadata_global_program")
2212     return ParseSectionDirectiveHSADataGlobalProgram();
2213 
2214   if (IDVal == ".hsarodata_readonly_agent")
2215     return ParseSectionDirectiveHSARodataReadonlyAgent();
2216 
2217   return true;
2218 }
2219 
2220 bool AMDGPUAsmParser::subtargetHasRegister(const MCRegisterInfo &MRI,
2221                                            unsigned RegNo) const {
2222   if (isCI())
2223     return true;
2224 
2225   if (isSI()) {
2226     // No flat_scr
2227     switch (RegNo) {
2228     case AMDGPU::FLAT_SCR:
2229     case AMDGPU::FLAT_SCR_LO:
2230     case AMDGPU::FLAT_SCR_HI:
2231       return false;
2232     default:
2233       return true;
2234     }
2235   }
2236 
2237   // VI only has 102 SGPRs, so make sure we aren't trying to use the 2 more that
2238   // SI/CI have.
2239   for (MCRegAliasIterator R(AMDGPU::SGPR102_SGPR103, &MRI, true);
2240        R.isValid(); ++R) {
2241     if (*R == RegNo)
2242       return false;
2243   }
2244 
2245   return true;
2246 }
2247 
2248 OperandMatchResultTy
2249 AMDGPUAsmParser::parseOperand(OperandVector &Operands, StringRef Mnemonic) {
2250   // Try to parse with a custom parser
2251   OperandMatchResultTy ResTy = MatchOperandParserImpl(Operands, Mnemonic);
2252 
2253   // If we successfully parsed the operand or if there as an error parsing,
2254   // we are done.
2255   //
2256   // If we are parsing after we reach EndOfStatement then this means we
2257   // are appending default values to the Operands list.  This is only done
2258   // by custom parser, so we shouldn't continue on to the generic parsing.
2259   if (ResTy == MatchOperand_Success || ResTy == MatchOperand_ParseFail ||
2260       getLexer().is(AsmToken::EndOfStatement))
2261     return ResTy;
2262 
2263   ResTy = parseRegOrImm(Operands);
2264 
2265   if (ResTy == MatchOperand_Success)
2266     return ResTy;
2267 
2268   if (getLexer().getKind() == AsmToken::Identifier) {
2269     // If this identifier is a symbol, we want to create an expression for it.
2270     // It is a little difficult to distinguish between a symbol name, and
2271     // an instruction flag like 'gds'.  In order to do this, we parse
2272     // all tokens as expressions and then treate the symbol name as the token
2273     // string when we want to interpret the operand as a token.
2274     const auto &Tok = Parser.getTok();
2275     SMLoc S = Tok.getLoc();
2276     const MCExpr *Expr = nullptr;
2277     if (!Parser.parseExpression(Expr)) {
2278       Operands.push_back(AMDGPUOperand::CreateExpr(this, Expr, S));
2279       return MatchOperand_Success;
2280     }
2281 
2282     Operands.push_back(AMDGPUOperand::CreateToken(this, Tok.getString(), Tok.getLoc()));
2283     Parser.Lex();
2284     return MatchOperand_Success;
2285   }
2286   return MatchOperand_NoMatch;
2287 }
2288 
2289 StringRef AMDGPUAsmParser::parseMnemonicSuffix(StringRef Name) {
2290   // Clear any forced encodings from the previous instruction.
2291   setForcedEncodingSize(0);
2292   setForcedDPP(false);
2293   setForcedSDWA(false);
2294 
2295   if (Name.endswith("_e64")) {
2296     setForcedEncodingSize(64);
2297     return Name.substr(0, Name.size() - 4);
2298   } else if (Name.endswith("_e32")) {
2299     setForcedEncodingSize(32);
2300     return Name.substr(0, Name.size() - 4);
2301   } else if (Name.endswith("_dpp")) {
2302     setForcedDPP(true);
2303     return Name.substr(0, Name.size() - 4);
2304   } else if (Name.endswith("_sdwa")) {
2305     setForcedSDWA(true);
2306     return Name.substr(0, Name.size() - 5);
2307   }
2308   return Name;
2309 }
2310 
2311 bool AMDGPUAsmParser::ParseInstruction(ParseInstructionInfo &Info,
2312                                        StringRef Name,
2313                                        SMLoc NameLoc, OperandVector &Operands) {
2314   // Add the instruction mnemonic
2315   Name = parseMnemonicSuffix(Name);
2316   Operands.push_back(AMDGPUOperand::CreateToken(this, Name, NameLoc));
2317 
2318   while (!getLexer().is(AsmToken::EndOfStatement)) {
2319     OperandMatchResultTy Res = parseOperand(Operands, Name);
2320 
2321     // Eat the comma or space if there is one.
2322     if (getLexer().is(AsmToken::Comma))
2323       Parser.Lex();
2324 
2325     switch (Res) {
2326       case MatchOperand_Success: break;
2327       case MatchOperand_ParseFail:
2328         Error(getLexer().getLoc(), "failed parsing operand.");
2329         while (!getLexer().is(AsmToken::EndOfStatement)) {
2330           Parser.Lex();
2331         }
2332         return true;
2333       case MatchOperand_NoMatch:
2334         Error(getLexer().getLoc(), "not a valid operand.");
2335         while (!getLexer().is(AsmToken::EndOfStatement)) {
2336           Parser.Lex();
2337         }
2338         return true;
2339     }
2340   }
2341 
2342   return false;
2343 }
2344 
2345 //===----------------------------------------------------------------------===//
2346 // Utility functions
2347 //===----------------------------------------------------------------------===//
2348 
2349 OperandMatchResultTy
2350 AMDGPUAsmParser::parseIntWithPrefix(const char *Prefix, int64_t &Int) {
2351   switch(getLexer().getKind()) {
2352     default: return MatchOperand_NoMatch;
2353     case AsmToken::Identifier: {
2354       StringRef Name = Parser.getTok().getString();
2355       if (!Name.equals(Prefix)) {
2356         return MatchOperand_NoMatch;
2357       }
2358 
2359       Parser.Lex();
2360       if (getLexer().isNot(AsmToken::Colon))
2361         return MatchOperand_ParseFail;
2362 
2363       Parser.Lex();
2364       if (getLexer().isNot(AsmToken::Integer))
2365         return MatchOperand_ParseFail;
2366 
2367       if (getParser().parseAbsoluteExpression(Int))
2368         return MatchOperand_ParseFail;
2369       break;
2370     }
2371   }
2372   return MatchOperand_Success;
2373 }
2374 
2375 OperandMatchResultTy
2376 AMDGPUAsmParser::parseIntWithPrefix(const char *Prefix, OperandVector &Operands,
2377                                     AMDGPUOperand::ImmTy ImmTy,
2378                                     bool (*ConvertResult)(int64_t&)) {
2379   SMLoc S = Parser.getTok().getLoc();
2380   int64_t Value = 0;
2381 
2382   OperandMatchResultTy Res = parseIntWithPrefix(Prefix, Value);
2383   if (Res != MatchOperand_Success)
2384     return Res;
2385 
2386   if (ConvertResult && !ConvertResult(Value)) {
2387     return MatchOperand_ParseFail;
2388   }
2389 
2390   Operands.push_back(AMDGPUOperand::CreateImm(this, Value, S, ImmTy));
2391   return MatchOperand_Success;
2392 }
2393 
2394 OperandMatchResultTy AMDGPUAsmParser::parseOperandArrayWithPrefix(
2395   const char *Prefix,
2396   OperandVector &Operands,
2397   AMDGPUOperand::ImmTy ImmTy,
2398   bool (*ConvertResult)(int64_t&)) {
2399   StringRef Name = Parser.getTok().getString();
2400   if (!Name.equals(Prefix))
2401     return MatchOperand_NoMatch;
2402 
2403   Parser.Lex();
2404   if (getLexer().isNot(AsmToken::Colon))
2405     return MatchOperand_ParseFail;
2406 
2407   Parser.Lex();
2408   if (getLexer().isNot(AsmToken::LBrac))
2409     return MatchOperand_ParseFail;
2410   Parser.Lex();
2411 
2412   unsigned Val = 0;
2413   SMLoc S = Parser.getTok().getLoc();
2414 
2415   // FIXME: How to verify the number of elements matches the number of src
2416   // operands?
2417   for (int I = 0; I < 3; ++I) {
2418     if (I != 0) {
2419       if (getLexer().is(AsmToken::RBrac))
2420         break;
2421 
2422       if (getLexer().isNot(AsmToken::Comma))
2423         return MatchOperand_ParseFail;
2424       Parser.Lex();
2425     }
2426 
2427     if (getLexer().isNot(AsmToken::Integer))
2428       return MatchOperand_ParseFail;
2429 
2430     int64_t Op;
2431     if (getParser().parseAbsoluteExpression(Op))
2432       return MatchOperand_ParseFail;
2433 
2434     if (Op != 0 && Op != 1)
2435       return MatchOperand_ParseFail;
2436     Val |= (Op << I);
2437   }
2438 
2439   Parser.Lex();
2440   Operands.push_back(AMDGPUOperand::CreateImm(this, Val, S, ImmTy));
2441   return MatchOperand_Success;
2442 }
2443 
2444 OperandMatchResultTy
2445 AMDGPUAsmParser::parseNamedBit(const char *Name, OperandVector &Operands,
2446                                AMDGPUOperand::ImmTy ImmTy) {
2447   int64_t Bit = 0;
2448   SMLoc S = Parser.getTok().getLoc();
2449 
2450   // We are at the end of the statement, and this is a default argument, so
2451   // use a default value.
2452   if (getLexer().isNot(AsmToken::EndOfStatement)) {
2453     switch(getLexer().getKind()) {
2454       case AsmToken::Identifier: {
2455         StringRef Tok = Parser.getTok().getString();
2456         if (Tok == Name) {
2457           Bit = 1;
2458           Parser.Lex();
2459         } else if (Tok.startswith("no") && Tok.endswith(Name)) {
2460           Bit = 0;
2461           Parser.Lex();
2462         } else {
2463           return MatchOperand_NoMatch;
2464         }
2465         break;
2466       }
2467       default:
2468         return MatchOperand_NoMatch;
2469     }
2470   }
2471 
2472   Operands.push_back(AMDGPUOperand::CreateImm(this, Bit, S, ImmTy));
2473   return MatchOperand_Success;
2474 }
2475 
2476 static void addOptionalImmOperand(
2477   MCInst& Inst, const OperandVector& Operands,
2478   AMDGPUAsmParser::OptionalImmIndexMap& OptionalIdx,
2479   AMDGPUOperand::ImmTy ImmT,
2480   int64_t Default = 0) {
2481   auto i = OptionalIdx.find(ImmT);
2482   if (i != OptionalIdx.end()) {
2483     unsigned Idx = i->second;
2484     ((AMDGPUOperand &)*Operands[Idx]).addImmOperands(Inst, 1);
2485   } else {
2486     Inst.addOperand(MCOperand::createImm(Default));
2487   }
2488 }
2489 
2490 OperandMatchResultTy
2491 AMDGPUAsmParser::parseStringWithPrefix(StringRef Prefix, StringRef &Value) {
2492   if (getLexer().isNot(AsmToken::Identifier)) {
2493     return MatchOperand_NoMatch;
2494   }
2495   StringRef Tok = Parser.getTok().getString();
2496   if (Tok != Prefix) {
2497     return MatchOperand_NoMatch;
2498   }
2499 
2500   Parser.Lex();
2501   if (getLexer().isNot(AsmToken::Colon)) {
2502     return MatchOperand_ParseFail;
2503   }
2504 
2505   Parser.Lex();
2506   if (getLexer().isNot(AsmToken::Identifier)) {
2507     return MatchOperand_ParseFail;
2508   }
2509 
2510   Value = Parser.getTok().getString();
2511   return MatchOperand_Success;
2512 }
2513 
2514 //===----------------------------------------------------------------------===//
2515 // ds
2516 //===----------------------------------------------------------------------===//
2517 
2518 void AMDGPUAsmParser::cvtDSOffset01(MCInst &Inst,
2519                                     const OperandVector &Operands) {
2520   OptionalImmIndexMap OptionalIdx;
2521 
2522   for (unsigned i = 1, e = Operands.size(); i != e; ++i) {
2523     AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[i]);
2524 
2525     // Add the register arguments
2526     if (Op.isReg()) {
2527       Op.addRegOperands(Inst, 1);
2528       continue;
2529     }
2530 
2531     // Handle optional arguments
2532     OptionalIdx[Op.getImmTy()] = i;
2533   }
2534 
2535   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyOffset0);
2536   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyOffset1);
2537   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyGDS);
2538 
2539   Inst.addOperand(MCOperand::createReg(AMDGPU::M0)); // m0
2540 }
2541 
2542 void AMDGPUAsmParser::cvtDSImpl(MCInst &Inst, const OperandVector &Operands,
2543                                 bool IsGdsHardcoded) {
2544   OptionalImmIndexMap OptionalIdx;
2545 
2546   for (unsigned i = 1, e = Operands.size(); i != e; ++i) {
2547     AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[i]);
2548 
2549     // Add the register arguments
2550     if (Op.isReg()) {
2551       Op.addRegOperands(Inst, 1);
2552       continue;
2553     }
2554 
2555     if (Op.isToken() && Op.getToken() == "gds") {
2556       IsGdsHardcoded = true;
2557       continue;
2558     }
2559 
2560     // Handle optional arguments
2561     OptionalIdx[Op.getImmTy()] = i;
2562   }
2563 
2564   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyOffset);
2565   if (!IsGdsHardcoded) {
2566     addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyGDS);
2567   }
2568   Inst.addOperand(MCOperand::createReg(AMDGPU::M0)); // m0
2569 }
2570 
2571 void AMDGPUAsmParser::cvtExp(MCInst &Inst, const OperandVector &Operands) {
2572   OptionalImmIndexMap OptionalIdx;
2573 
2574   unsigned EnMask = 0;
2575   int SrcIdx = 0;
2576 
2577   for (unsigned i = 1, e = Operands.size(); i != e; ++i) {
2578     AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[i]);
2579 
2580     // Add the register arguments
2581     if (Op.isReg()) {
2582       EnMask |= (1 << SrcIdx);
2583       Op.addRegOperands(Inst, 1);
2584       ++SrcIdx;
2585       continue;
2586     }
2587 
2588     if (Op.isOff()) {
2589       ++SrcIdx;
2590       Inst.addOperand(MCOperand::createReg(AMDGPU::NoRegister));
2591       continue;
2592     }
2593 
2594     if (Op.isImm() && Op.getImmTy() == AMDGPUOperand::ImmTyExpTgt) {
2595       Op.addImmOperands(Inst, 1);
2596       continue;
2597     }
2598 
2599     if (Op.isToken() && Op.getToken() == "done")
2600       continue;
2601 
2602     // Handle optional arguments
2603     OptionalIdx[Op.getImmTy()] = i;
2604   }
2605 
2606   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyExpVM);
2607   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyExpCompr);
2608 
2609   Inst.addOperand(MCOperand::createImm(EnMask));
2610 }
2611 
2612 //===----------------------------------------------------------------------===//
2613 // s_waitcnt
2614 //===----------------------------------------------------------------------===//
2615 
2616 bool AMDGPUAsmParser::parseCnt(int64_t &IntVal) {
2617   StringRef CntName = Parser.getTok().getString();
2618   int64_t CntVal;
2619 
2620   Parser.Lex();
2621   if (getLexer().isNot(AsmToken::LParen))
2622     return true;
2623 
2624   Parser.Lex();
2625   if (getLexer().isNot(AsmToken::Integer))
2626     return true;
2627 
2628   if (getParser().parseAbsoluteExpression(CntVal))
2629     return true;
2630 
2631   if (getLexer().isNot(AsmToken::RParen))
2632     return true;
2633 
2634   Parser.Lex();
2635   if (getLexer().is(AsmToken::Amp) || getLexer().is(AsmToken::Comma))
2636     Parser.Lex();
2637 
2638   AMDGPU::IsaInfo::IsaVersion ISA =
2639       AMDGPU::IsaInfo::getIsaVersion(getFeatureBits());
2640   if (CntName == "vmcnt")
2641     IntVal = encodeVmcnt(ISA, IntVal, CntVal);
2642   else if (CntName == "expcnt")
2643     IntVal = encodeExpcnt(ISA, IntVal, CntVal);
2644   else if (CntName == "lgkmcnt")
2645     IntVal = encodeLgkmcnt(ISA, IntVal, CntVal);
2646   else
2647     return true;
2648 
2649   return false;
2650 }
2651 
2652 OperandMatchResultTy
2653 AMDGPUAsmParser::parseSWaitCntOps(OperandVector &Operands) {
2654   AMDGPU::IsaInfo::IsaVersion ISA =
2655       AMDGPU::IsaInfo::getIsaVersion(getFeatureBits());
2656   int64_t Waitcnt = getWaitcntBitMask(ISA);
2657   SMLoc S = Parser.getTok().getLoc();
2658 
2659   switch(getLexer().getKind()) {
2660     default: return MatchOperand_ParseFail;
2661     case AsmToken::Integer:
2662       // The operand can be an integer value.
2663       if (getParser().parseAbsoluteExpression(Waitcnt))
2664         return MatchOperand_ParseFail;
2665       break;
2666 
2667     case AsmToken::Identifier:
2668       do {
2669         if (parseCnt(Waitcnt))
2670           return MatchOperand_ParseFail;
2671       } while(getLexer().isNot(AsmToken::EndOfStatement));
2672       break;
2673   }
2674   Operands.push_back(AMDGPUOperand::CreateImm(this, Waitcnt, S));
2675   return MatchOperand_Success;
2676 }
2677 
2678 bool AMDGPUAsmParser::parseHwregConstruct(OperandInfoTy &HwReg, int64_t &Offset,
2679                                           int64_t &Width) {
2680   using namespace llvm::AMDGPU::Hwreg;
2681 
2682   if (Parser.getTok().getString() != "hwreg")
2683     return true;
2684   Parser.Lex();
2685 
2686   if (getLexer().isNot(AsmToken::LParen))
2687     return true;
2688   Parser.Lex();
2689 
2690   if (getLexer().is(AsmToken::Identifier)) {
2691     HwReg.IsSymbolic = true;
2692     HwReg.Id = ID_UNKNOWN_;
2693     const StringRef tok = Parser.getTok().getString();
2694     for (int i = ID_SYMBOLIC_FIRST_; i < ID_SYMBOLIC_LAST_; ++i) {
2695       if (tok == IdSymbolic[i]) {
2696         HwReg.Id = i;
2697         break;
2698       }
2699     }
2700     Parser.Lex();
2701   } else {
2702     HwReg.IsSymbolic = false;
2703     if (getLexer().isNot(AsmToken::Integer))
2704       return true;
2705     if (getParser().parseAbsoluteExpression(HwReg.Id))
2706       return true;
2707   }
2708 
2709   if (getLexer().is(AsmToken::RParen)) {
2710     Parser.Lex();
2711     return false;
2712   }
2713 
2714   // optional params
2715   if (getLexer().isNot(AsmToken::Comma))
2716     return true;
2717   Parser.Lex();
2718 
2719   if (getLexer().isNot(AsmToken::Integer))
2720     return true;
2721   if (getParser().parseAbsoluteExpression(Offset))
2722     return true;
2723 
2724   if (getLexer().isNot(AsmToken::Comma))
2725     return true;
2726   Parser.Lex();
2727 
2728   if (getLexer().isNot(AsmToken::Integer))
2729     return true;
2730   if (getParser().parseAbsoluteExpression(Width))
2731     return true;
2732 
2733   if (getLexer().isNot(AsmToken::RParen))
2734     return true;
2735   Parser.Lex();
2736 
2737   return false;
2738 }
2739 
2740 OperandMatchResultTy AMDGPUAsmParser::parseHwreg(OperandVector &Operands) {
2741   using namespace llvm::AMDGPU::Hwreg;
2742 
2743   int64_t Imm16Val = 0;
2744   SMLoc S = Parser.getTok().getLoc();
2745 
2746   switch(getLexer().getKind()) {
2747     default: return MatchOperand_NoMatch;
2748     case AsmToken::Integer:
2749       // The operand can be an integer value.
2750       if (getParser().parseAbsoluteExpression(Imm16Val))
2751         return MatchOperand_NoMatch;
2752       if (Imm16Val < 0 || !isUInt<16>(Imm16Val)) {
2753         Error(S, "invalid immediate: only 16-bit values are legal");
2754         // Do not return error code, but create an imm operand anyway and proceed
2755         // to the next operand, if any. That avoids unneccessary error messages.
2756       }
2757       break;
2758 
2759     case AsmToken::Identifier: {
2760         OperandInfoTy HwReg(ID_UNKNOWN_);
2761         int64_t Offset = OFFSET_DEFAULT_;
2762         int64_t Width = WIDTH_M1_DEFAULT_ + 1;
2763         if (parseHwregConstruct(HwReg, Offset, Width))
2764           return MatchOperand_ParseFail;
2765         if (HwReg.Id < 0 || !isUInt<ID_WIDTH_>(HwReg.Id)) {
2766           if (HwReg.IsSymbolic)
2767             Error(S, "invalid symbolic name of hardware register");
2768           else
2769             Error(S, "invalid code of hardware register: only 6-bit values are legal");
2770         }
2771         if (Offset < 0 || !isUInt<OFFSET_WIDTH_>(Offset))
2772           Error(S, "invalid bit offset: only 5-bit values are legal");
2773         if ((Width-1) < 0 || !isUInt<WIDTH_M1_WIDTH_>(Width-1))
2774           Error(S, "invalid bitfield width: only values from 1 to 32 are legal");
2775         Imm16Val = (HwReg.Id << ID_SHIFT_) | (Offset << OFFSET_SHIFT_) | ((Width-1) << WIDTH_M1_SHIFT_);
2776       }
2777       break;
2778   }
2779   Operands.push_back(AMDGPUOperand::CreateImm(this, Imm16Val, S, AMDGPUOperand::ImmTyHwreg));
2780   return MatchOperand_Success;
2781 }
2782 
2783 bool AMDGPUOperand::isSWaitCnt() const {
2784   return isImm();
2785 }
2786 
2787 bool AMDGPUOperand::isHwreg() const {
2788   return isImmTy(ImmTyHwreg);
2789 }
2790 
2791 bool AMDGPUAsmParser::parseSendMsgConstruct(OperandInfoTy &Msg, OperandInfoTy &Operation, int64_t &StreamId) {
2792   using namespace llvm::AMDGPU::SendMsg;
2793 
2794   if (Parser.getTok().getString() != "sendmsg")
2795     return true;
2796   Parser.Lex();
2797 
2798   if (getLexer().isNot(AsmToken::LParen))
2799     return true;
2800   Parser.Lex();
2801 
2802   if (getLexer().is(AsmToken::Identifier)) {
2803     Msg.IsSymbolic = true;
2804     Msg.Id = ID_UNKNOWN_;
2805     const std::string tok = Parser.getTok().getString();
2806     for (int i = ID_GAPS_FIRST_; i < ID_GAPS_LAST_; ++i) {
2807       switch(i) {
2808         default: continue; // Omit gaps.
2809         case ID_INTERRUPT: case ID_GS: case ID_GS_DONE:  case ID_SYSMSG: break;
2810       }
2811       if (tok == IdSymbolic[i]) {
2812         Msg.Id = i;
2813         break;
2814       }
2815     }
2816     Parser.Lex();
2817   } else {
2818     Msg.IsSymbolic = false;
2819     if (getLexer().isNot(AsmToken::Integer))
2820       return true;
2821     if (getParser().parseAbsoluteExpression(Msg.Id))
2822       return true;
2823     if (getLexer().is(AsmToken::Integer))
2824       if (getParser().parseAbsoluteExpression(Msg.Id))
2825         Msg.Id = ID_UNKNOWN_;
2826   }
2827   if (Msg.Id == ID_UNKNOWN_) // Don't know how to parse the rest.
2828     return false;
2829 
2830   if (!(Msg.Id == ID_GS || Msg.Id == ID_GS_DONE || Msg.Id == ID_SYSMSG)) {
2831     if (getLexer().isNot(AsmToken::RParen))
2832       return true;
2833     Parser.Lex();
2834     return false;
2835   }
2836 
2837   if (getLexer().isNot(AsmToken::Comma))
2838     return true;
2839   Parser.Lex();
2840 
2841   assert(Msg.Id == ID_GS || Msg.Id == ID_GS_DONE || Msg.Id == ID_SYSMSG);
2842   Operation.Id = ID_UNKNOWN_;
2843   if (getLexer().is(AsmToken::Identifier)) {
2844     Operation.IsSymbolic = true;
2845     const char* const *S = (Msg.Id == ID_SYSMSG) ? OpSysSymbolic : OpGsSymbolic;
2846     const int F = (Msg.Id == ID_SYSMSG) ? OP_SYS_FIRST_ : OP_GS_FIRST_;
2847     const int L = (Msg.Id == ID_SYSMSG) ? OP_SYS_LAST_ : OP_GS_LAST_;
2848     const StringRef Tok = Parser.getTok().getString();
2849     for (int i = F; i < L; ++i) {
2850       if (Tok == S[i]) {
2851         Operation.Id = i;
2852         break;
2853       }
2854     }
2855     Parser.Lex();
2856   } else {
2857     Operation.IsSymbolic = false;
2858     if (getLexer().isNot(AsmToken::Integer))
2859       return true;
2860     if (getParser().parseAbsoluteExpression(Operation.Id))
2861       return true;
2862   }
2863 
2864   if ((Msg.Id == ID_GS || Msg.Id == ID_GS_DONE) && Operation.Id != OP_GS_NOP) {
2865     // Stream id is optional.
2866     if (getLexer().is(AsmToken::RParen)) {
2867       Parser.Lex();
2868       return false;
2869     }
2870 
2871     if (getLexer().isNot(AsmToken::Comma))
2872       return true;
2873     Parser.Lex();
2874 
2875     if (getLexer().isNot(AsmToken::Integer))
2876       return true;
2877     if (getParser().parseAbsoluteExpression(StreamId))
2878       return true;
2879   }
2880 
2881   if (getLexer().isNot(AsmToken::RParen))
2882     return true;
2883   Parser.Lex();
2884   return false;
2885 }
2886 
2887 OperandMatchResultTy AMDGPUAsmParser::parseInterpSlot(OperandVector &Operands) {
2888   if (getLexer().getKind() != AsmToken::Identifier)
2889     return MatchOperand_NoMatch;
2890 
2891   StringRef Str = Parser.getTok().getString();
2892   int Slot = StringSwitch<int>(Str)
2893     .Case("p10", 0)
2894     .Case("p20", 1)
2895     .Case("p0", 2)
2896     .Default(-1);
2897 
2898   SMLoc S = Parser.getTok().getLoc();
2899   if (Slot == -1)
2900     return MatchOperand_ParseFail;
2901 
2902   Parser.Lex();
2903   Operands.push_back(AMDGPUOperand::CreateImm(this, Slot, S,
2904                                               AMDGPUOperand::ImmTyInterpSlot));
2905   return MatchOperand_Success;
2906 }
2907 
2908 OperandMatchResultTy AMDGPUAsmParser::parseInterpAttr(OperandVector &Operands) {
2909   if (getLexer().getKind() != AsmToken::Identifier)
2910     return MatchOperand_NoMatch;
2911 
2912   StringRef Str = Parser.getTok().getString();
2913   if (!Str.startswith("attr"))
2914     return MatchOperand_NoMatch;
2915 
2916   StringRef Chan = Str.take_back(2);
2917   int AttrChan = StringSwitch<int>(Chan)
2918     .Case(".x", 0)
2919     .Case(".y", 1)
2920     .Case(".z", 2)
2921     .Case(".w", 3)
2922     .Default(-1);
2923   if (AttrChan == -1)
2924     return MatchOperand_ParseFail;
2925 
2926   Str = Str.drop_back(2).drop_front(4);
2927 
2928   uint8_t Attr;
2929   if (Str.getAsInteger(10, Attr))
2930     return MatchOperand_ParseFail;
2931 
2932   SMLoc S = Parser.getTok().getLoc();
2933   Parser.Lex();
2934   if (Attr > 63) {
2935     Error(S, "out of bounds attr");
2936     return MatchOperand_Success;
2937   }
2938 
2939   SMLoc SChan = SMLoc::getFromPointer(Chan.data());
2940 
2941   Operands.push_back(AMDGPUOperand::CreateImm(this, Attr, S,
2942                                               AMDGPUOperand::ImmTyInterpAttr));
2943   Operands.push_back(AMDGPUOperand::CreateImm(this, AttrChan, SChan,
2944                                               AMDGPUOperand::ImmTyAttrChan));
2945   return MatchOperand_Success;
2946 }
2947 
2948 void AMDGPUAsmParser::errorExpTgt() {
2949   Error(Parser.getTok().getLoc(), "invalid exp target");
2950 }
2951 
2952 OperandMatchResultTy AMDGPUAsmParser::parseExpTgtImpl(StringRef Str,
2953                                                       uint8_t &Val) {
2954   if (Str == "null") {
2955     Val = 9;
2956     return MatchOperand_Success;
2957   }
2958 
2959   if (Str.startswith("mrt")) {
2960     Str = Str.drop_front(3);
2961     if (Str == "z") { // == mrtz
2962       Val = 8;
2963       return MatchOperand_Success;
2964     }
2965 
2966     if (Str.getAsInteger(10, Val))
2967       return MatchOperand_ParseFail;
2968 
2969     if (Val > 7)
2970       errorExpTgt();
2971 
2972     return MatchOperand_Success;
2973   }
2974 
2975   if (Str.startswith("pos")) {
2976     Str = Str.drop_front(3);
2977     if (Str.getAsInteger(10, Val))
2978       return MatchOperand_ParseFail;
2979 
2980     if (Val > 3)
2981       errorExpTgt();
2982 
2983     Val += 12;
2984     return MatchOperand_Success;
2985   }
2986 
2987   if (Str.startswith("param")) {
2988     Str = Str.drop_front(5);
2989     if (Str.getAsInteger(10, Val))
2990       return MatchOperand_ParseFail;
2991 
2992     if (Val >= 32)
2993       errorExpTgt();
2994 
2995     Val += 32;
2996     return MatchOperand_Success;
2997   }
2998 
2999   if (Str.startswith("invalid_target_")) {
3000     Str = Str.drop_front(15);
3001     if (Str.getAsInteger(10, Val))
3002       return MatchOperand_ParseFail;
3003 
3004     errorExpTgt();
3005     return MatchOperand_Success;
3006   }
3007 
3008   return MatchOperand_NoMatch;
3009 }
3010 
3011 OperandMatchResultTy AMDGPUAsmParser::parseExpTgt(OperandVector &Operands) {
3012   uint8_t Val;
3013   StringRef Str = Parser.getTok().getString();
3014 
3015   auto Res = parseExpTgtImpl(Str, Val);
3016   if (Res != MatchOperand_Success)
3017     return Res;
3018 
3019   SMLoc S = Parser.getTok().getLoc();
3020   Parser.Lex();
3021 
3022   Operands.push_back(AMDGPUOperand::CreateImm(this, Val, S,
3023                                               AMDGPUOperand::ImmTyExpTgt));
3024   return MatchOperand_Success;
3025 }
3026 
3027 OperandMatchResultTy
3028 AMDGPUAsmParser::parseSendMsgOp(OperandVector &Operands) {
3029   using namespace llvm::AMDGPU::SendMsg;
3030 
3031   int64_t Imm16Val = 0;
3032   SMLoc S = Parser.getTok().getLoc();
3033 
3034   switch(getLexer().getKind()) {
3035   default:
3036     return MatchOperand_NoMatch;
3037   case AsmToken::Integer:
3038     // The operand can be an integer value.
3039     if (getParser().parseAbsoluteExpression(Imm16Val))
3040       return MatchOperand_NoMatch;
3041     if (Imm16Val < 0 || !isUInt<16>(Imm16Val)) {
3042       Error(S, "invalid immediate: only 16-bit values are legal");
3043       // Do not return error code, but create an imm operand anyway and proceed
3044       // to the next operand, if any. That avoids unneccessary error messages.
3045     }
3046     break;
3047   case AsmToken::Identifier: {
3048       OperandInfoTy Msg(ID_UNKNOWN_);
3049       OperandInfoTy Operation(OP_UNKNOWN_);
3050       int64_t StreamId = STREAM_ID_DEFAULT_;
3051       if (parseSendMsgConstruct(Msg, Operation, StreamId))
3052         return MatchOperand_ParseFail;
3053       do {
3054         // Validate and encode message ID.
3055         if (! ((ID_INTERRUPT <= Msg.Id && Msg.Id <= ID_GS_DONE)
3056                 || Msg.Id == ID_SYSMSG)) {
3057           if (Msg.IsSymbolic)
3058             Error(S, "invalid/unsupported symbolic name of message");
3059           else
3060             Error(S, "invalid/unsupported code of message");
3061           break;
3062         }
3063         Imm16Val = (Msg.Id << ID_SHIFT_);
3064         // Validate and encode operation ID.
3065         if (Msg.Id == ID_GS || Msg.Id == ID_GS_DONE) {
3066           if (! (OP_GS_FIRST_ <= Operation.Id && Operation.Id < OP_GS_LAST_)) {
3067             if (Operation.IsSymbolic)
3068               Error(S, "invalid symbolic name of GS_OP");
3069             else
3070               Error(S, "invalid code of GS_OP: only 2-bit values are legal");
3071             break;
3072           }
3073           if (Operation.Id == OP_GS_NOP
3074               && Msg.Id != ID_GS_DONE) {
3075             Error(S, "invalid GS_OP: NOP is for GS_DONE only");
3076             break;
3077           }
3078           Imm16Val |= (Operation.Id << OP_SHIFT_);
3079         }
3080         if (Msg.Id == ID_SYSMSG) {
3081           if (! (OP_SYS_FIRST_ <= Operation.Id && Operation.Id < OP_SYS_LAST_)) {
3082             if (Operation.IsSymbolic)
3083               Error(S, "invalid/unsupported symbolic name of SYSMSG_OP");
3084             else
3085               Error(S, "invalid/unsupported code of SYSMSG_OP");
3086             break;
3087           }
3088           Imm16Val |= (Operation.Id << OP_SHIFT_);
3089         }
3090         // Validate and encode stream ID.
3091         if ((Msg.Id == ID_GS || Msg.Id == ID_GS_DONE) && Operation.Id != OP_GS_NOP) {
3092           if (! (STREAM_ID_FIRST_ <= StreamId && StreamId < STREAM_ID_LAST_)) {
3093             Error(S, "invalid stream id: only 2-bit values are legal");
3094             break;
3095           }
3096           Imm16Val |= (StreamId << STREAM_ID_SHIFT_);
3097         }
3098       } while (false);
3099     }
3100     break;
3101   }
3102   Operands.push_back(AMDGPUOperand::CreateImm(this, Imm16Val, S, AMDGPUOperand::ImmTySendMsg));
3103   return MatchOperand_Success;
3104 }
3105 
3106 bool AMDGPUOperand::isSendMsg() const {
3107   return isImmTy(ImmTySendMsg);
3108 }
3109 
3110 //===----------------------------------------------------------------------===//
3111 // sopp branch targets
3112 //===----------------------------------------------------------------------===//
3113 
3114 OperandMatchResultTy
3115 AMDGPUAsmParser::parseSOppBrTarget(OperandVector &Operands) {
3116   SMLoc S = Parser.getTok().getLoc();
3117 
3118   switch (getLexer().getKind()) {
3119     default: return MatchOperand_ParseFail;
3120     case AsmToken::Integer: {
3121       int64_t Imm;
3122       if (getParser().parseAbsoluteExpression(Imm))
3123         return MatchOperand_ParseFail;
3124       Operands.push_back(AMDGPUOperand::CreateImm(this, Imm, S));
3125       return MatchOperand_Success;
3126     }
3127 
3128     case AsmToken::Identifier:
3129       Operands.push_back(AMDGPUOperand::CreateExpr(this,
3130           MCSymbolRefExpr::create(getContext().getOrCreateSymbol(
3131                                   Parser.getTok().getString()), getContext()), S));
3132       Parser.Lex();
3133       return MatchOperand_Success;
3134   }
3135 }
3136 
3137 //===----------------------------------------------------------------------===//
3138 // mubuf
3139 //===----------------------------------------------------------------------===//
3140 
3141 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultGLC() const {
3142   return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTyGLC);
3143 }
3144 
3145 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultSLC() const {
3146   return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTySLC);
3147 }
3148 
3149 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultTFE() const {
3150   return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTyTFE);
3151 }
3152 
3153 void AMDGPUAsmParser::cvtMubufImpl(MCInst &Inst,
3154                                const OperandVector &Operands,
3155                                bool IsAtomic, bool IsAtomicReturn) {
3156   OptionalImmIndexMap OptionalIdx;
3157   assert(IsAtomicReturn ? IsAtomic : true);
3158 
3159   for (unsigned i = 1, e = Operands.size(); i != e; ++i) {
3160     AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[i]);
3161 
3162     // Add the register arguments
3163     if (Op.isReg()) {
3164       Op.addRegOperands(Inst, 1);
3165       continue;
3166     }
3167 
3168     // Handle the case where soffset is an immediate
3169     if (Op.isImm() && Op.getImmTy() == AMDGPUOperand::ImmTyNone) {
3170       Op.addImmOperands(Inst, 1);
3171       continue;
3172     }
3173 
3174     // Handle tokens like 'offen' which are sometimes hard-coded into the
3175     // asm string.  There are no MCInst operands for these.
3176     if (Op.isToken()) {
3177       continue;
3178     }
3179     assert(Op.isImm());
3180 
3181     // Handle optional arguments
3182     OptionalIdx[Op.getImmTy()] = i;
3183   }
3184 
3185   // Copy $vdata_in operand and insert as $vdata for MUBUF_Atomic RTN insns.
3186   if (IsAtomicReturn) {
3187     MCInst::iterator I = Inst.begin(); // $vdata_in is always at the beginning.
3188     Inst.insert(I, *I);
3189   }
3190 
3191   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyOffset);
3192   if (!IsAtomic) { // glc is hard-coded.
3193     addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyGLC);
3194   }
3195   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySLC);
3196   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyTFE);
3197 }
3198 
3199 //===----------------------------------------------------------------------===//
3200 // mimg
3201 //===----------------------------------------------------------------------===//
3202 
3203 void AMDGPUAsmParser::cvtMIMG(MCInst &Inst, const OperandVector &Operands) {
3204   unsigned I = 1;
3205   const MCInstrDesc &Desc = MII.get(Inst.getOpcode());
3206   for (unsigned J = 0; J < Desc.getNumDefs(); ++J) {
3207     ((AMDGPUOperand &)*Operands[I++]).addRegOperands(Inst, 1);
3208   }
3209 
3210   OptionalImmIndexMap OptionalIdx;
3211 
3212   for (unsigned E = Operands.size(); I != E; ++I) {
3213     AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[I]);
3214 
3215     // Add the register arguments
3216     if (Op.isRegOrImm()) {
3217       Op.addRegOrImmOperands(Inst, 1);
3218       continue;
3219     } else if (Op.isImmModifier()) {
3220       OptionalIdx[Op.getImmTy()] = I;
3221     } else {
3222       llvm_unreachable("unexpected operand type");
3223     }
3224   }
3225 
3226   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyDMask);
3227   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyUNorm);
3228   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyGLC);
3229   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyDA);
3230   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyR128);
3231   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyTFE);
3232   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyLWE);
3233   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySLC);
3234 }
3235 
3236 void AMDGPUAsmParser::cvtMIMGAtomic(MCInst &Inst, const OperandVector &Operands) {
3237   unsigned I = 1;
3238   const MCInstrDesc &Desc = MII.get(Inst.getOpcode());
3239   for (unsigned J = 0; J < Desc.getNumDefs(); ++J) {
3240     ((AMDGPUOperand &)*Operands[I++]).addRegOperands(Inst, 1);
3241   }
3242 
3243   // Add src, same as dst
3244   ((AMDGPUOperand &)*Operands[I]).addRegOperands(Inst, 1);
3245 
3246   OptionalImmIndexMap OptionalIdx;
3247 
3248   for (unsigned E = Operands.size(); I != E; ++I) {
3249     AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[I]);
3250 
3251     // Add the register arguments
3252     if (Op.isRegOrImm()) {
3253       Op.addRegOrImmOperands(Inst, 1);
3254       continue;
3255     } else if (Op.isImmModifier()) {
3256       OptionalIdx[Op.getImmTy()] = I;
3257     } else {
3258       llvm_unreachable("unexpected operand type");
3259     }
3260   }
3261 
3262   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyDMask);
3263   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyUNorm);
3264   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyGLC);
3265   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyDA);
3266   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyR128);
3267   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyTFE);
3268   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyLWE);
3269   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySLC);
3270 }
3271 
3272 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultDMask() const {
3273   return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTyDMask);
3274 }
3275 
3276 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultUNorm() const {
3277   return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTyUNorm);
3278 }
3279 
3280 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultDA() const {
3281   return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTyDA);
3282 }
3283 
3284 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultR128() const {
3285   return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTyR128);
3286 }
3287 
3288 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultLWE() const {
3289   return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTyLWE);
3290 }
3291 
3292 //===----------------------------------------------------------------------===//
3293 // smrd
3294 //===----------------------------------------------------------------------===//
3295 
3296 bool AMDGPUOperand::isSMRDOffset8() const {
3297   return isImm() && isUInt<8>(getImm());
3298 }
3299 
3300 bool AMDGPUOperand::isSMRDOffset20() const {
3301   return isImm() && isUInt<20>(getImm());
3302 }
3303 
3304 bool AMDGPUOperand::isSMRDLiteralOffset() const {
3305   // 32-bit literals are only supported on CI and we only want to use them
3306   // when the offset is > 8-bits.
3307   return isImm() && !isUInt<8>(getImm()) && isUInt<32>(getImm());
3308 }
3309 
3310 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultSMRDOffset8() const {
3311   return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTyOffset);
3312 }
3313 
3314 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultSMRDOffset20() const {
3315   return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTyOffset);
3316 }
3317 
3318 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultSMRDLiteralOffset() const {
3319   return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTyOffset);
3320 }
3321 
3322 //===----------------------------------------------------------------------===//
3323 // vop3
3324 //===----------------------------------------------------------------------===//
3325 
3326 static bool ConvertOmodMul(int64_t &Mul) {
3327   if (Mul != 1 && Mul != 2 && Mul != 4)
3328     return false;
3329 
3330   Mul >>= 1;
3331   return true;
3332 }
3333 
3334 static bool ConvertOmodDiv(int64_t &Div) {
3335   if (Div == 1) {
3336     Div = 0;
3337     return true;
3338   }
3339 
3340   if (Div == 2) {
3341     Div = 3;
3342     return true;
3343   }
3344 
3345   return false;
3346 }
3347 
3348 static bool ConvertBoundCtrl(int64_t &BoundCtrl) {
3349   if (BoundCtrl == 0) {
3350     BoundCtrl = 1;
3351     return true;
3352   }
3353 
3354   if (BoundCtrl == -1) {
3355     BoundCtrl = 0;
3356     return true;
3357   }
3358 
3359   return false;
3360 }
3361 
3362 // Note: the order in this table matches the order of operands in AsmString.
3363 static const OptionalOperand AMDGPUOptionalOperandTable[] = {
3364   {"offen",   AMDGPUOperand::ImmTyOffen, true, nullptr},
3365   {"idxen",   AMDGPUOperand::ImmTyIdxen, true, nullptr},
3366   {"addr64",  AMDGPUOperand::ImmTyAddr64, true, nullptr},
3367   {"offset0", AMDGPUOperand::ImmTyOffset0, false, nullptr},
3368   {"offset1", AMDGPUOperand::ImmTyOffset1, false, nullptr},
3369   {"gds",     AMDGPUOperand::ImmTyGDS, true, nullptr},
3370   {"offset",  AMDGPUOperand::ImmTyOffset, false, nullptr},
3371   {"glc",     AMDGPUOperand::ImmTyGLC, true, nullptr},
3372   {"slc",     AMDGPUOperand::ImmTySLC, true, nullptr},
3373   {"tfe",     AMDGPUOperand::ImmTyTFE, true, nullptr},
3374   {"clamp",   AMDGPUOperand::ImmTyClampSI, true, nullptr},
3375   {"omod",    AMDGPUOperand::ImmTyOModSI, false, ConvertOmodMul},
3376   {"unorm",   AMDGPUOperand::ImmTyUNorm, true, nullptr},
3377   {"da",      AMDGPUOperand::ImmTyDA,    true, nullptr},
3378   {"r128",    AMDGPUOperand::ImmTyR128,  true, nullptr},
3379   {"lwe",     AMDGPUOperand::ImmTyLWE,   true, nullptr},
3380   {"dmask",   AMDGPUOperand::ImmTyDMask, false, nullptr},
3381   {"row_mask",   AMDGPUOperand::ImmTyDppRowMask, false, nullptr},
3382   {"bank_mask",  AMDGPUOperand::ImmTyDppBankMask, false, nullptr},
3383   {"bound_ctrl", AMDGPUOperand::ImmTyDppBoundCtrl, false, ConvertBoundCtrl},
3384   {"dst_sel",    AMDGPUOperand::ImmTySdwaDstSel, false, nullptr},
3385   {"src0_sel",   AMDGPUOperand::ImmTySdwaSrc0Sel, false, nullptr},
3386   {"src1_sel",   AMDGPUOperand::ImmTySdwaSrc1Sel, false, nullptr},
3387   {"dst_unused", AMDGPUOperand::ImmTySdwaDstUnused, false, nullptr},
3388   {"vm", AMDGPUOperand::ImmTyExpVM, true, nullptr},
3389   {"op_sel", AMDGPUOperand::ImmTyOpSel, false, nullptr},
3390   {"op_sel_hi", AMDGPUOperand::ImmTyOpSelHi, false, nullptr},
3391   {"neg_lo", AMDGPUOperand::ImmTyNegLo, false, nullptr},
3392   {"neg_hi", AMDGPUOperand::ImmTyNegHi, false, nullptr}
3393 };
3394 
3395 OperandMatchResultTy AMDGPUAsmParser::parseOptionalOperand(OperandVector &Operands) {
3396   OperandMatchResultTy res;
3397   for (const OptionalOperand &Op : AMDGPUOptionalOperandTable) {
3398     // try to parse any optional operand here
3399     if (Op.IsBit) {
3400       res = parseNamedBit(Op.Name, Operands, Op.Type);
3401     } else if (Op.Type == AMDGPUOperand::ImmTyOModSI) {
3402       res = parseOModOperand(Operands);
3403     } else if (Op.Type == AMDGPUOperand::ImmTySdwaDstSel ||
3404                Op.Type == AMDGPUOperand::ImmTySdwaSrc0Sel ||
3405                Op.Type == AMDGPUOperand::ImmTySdwaSrc1Sel) {
3406       res = parseSDWASel(Operands, Op.Name, Op.Type);
3407     } else if (Op.Type == AMDGPUOperand::ImmTySdwaDstUnused) {
3408       res = parseSDWADstUnused(Operands);
3409     } else if (Op.Type == AMDGPUOperand::ImmTyOpSel ||
3410                Op.Type == AMDGPUOperand::ImmTyOpSelHi ||
3411                Op.Type == AMDGPUOperand::ImmTyNegLo ||
3412                Op.Type == AMDGPUOperand::ImmTyNegHi) {
3413       res = parseOperandArrayWithPrefix(Op.Name, Operands, Op.Type,
3414                                         Op.ConvertResult);
3415     } else {
3416       res = parseIntWithPrefix(Op.Name, Operands, Op.Type, Op.ConvertResult);
3417     }
3418     if (res != MatchOperand_NoMatch) {
3419       return res;
3420     }
3421   }
3422   return MatchOperand_NoMatch;
3423 }
3424 
3425 OperandMatchResultTy AMDGPUAsmParser::parseOModOperand(OperandVector &Operands) {
3426   StringRef Name = Parser.getTok().getString();
3427   if (Name == "mul") {
3428     return parseIntWithPrefix("mul", Operands,
3429                               AMDGPUOperand::ImmTyOModSI, ConvertOmodMul);
3430   }
3431 
3432   if (Name == "div") {
3433     return parseIntWithPrefix("div", Operands,
3434                               AMDGPUOperand::ImmTyOModSI, ConvertOmodDiv);
3435   }
3436 
3437   return MatchOperand_NoMatch;
3438 }
3439 
3440 void AMDGPUAsmParser::cvtId(MCInst &Inst, const OperandVector &Operands) {
3441   unsigned I = 1;
3442   const MCInstrDesc &Desc = MII.get(Inst.getOpcode());
3443   for (unsigned J = 0; J < Desc.getNumDefs(); ++J) {
3444     ((AMDGPUOperand &)*Operands[I++]).addRegOperands(Inst, 1);
3445   }
3446   for (unsigned E = Operands.size(); I != E; ++I)
3447     ((AMDGPUOperand &)*Operands[I]).addRegOrImmOperands(Inst, 1);
3448 }
3449 
3450 void AMDGPUAsmParser::cvtVOP3_2_mod(MCInst &Inst, const OperandVector &Operands) {
3451   uint64_t TSFlags = MII.get(Inst.getOpcode()).TSFlags;
3452   if (TSFlags & SIInstrFlags::VOP3) {
3453     cvtVOP3(Inst, Operands);
3454   } else {
3455     cvtId(Inst, Operands);
3456   }
3457 }
3458 
3459 static bool isRegOrImmWithInputMods(const MCInstrDesc &Desc, unsigned OpNum) {
3460       // 1. This operand is input modifiers
3461   return Desc.OpInfo[OpNum].OperandType == AMDGPU::OPERAND_INPUT_MODS
3462       // 2. This is not last operand
3463       && Desc.NumOperands > (OpNum + 1)
3464       // 3. Next operand is register class
3465       && Desc.OpInfo[OpNum + 1].RegClass != -1
3466       // 4. Next register is not tied to any other operand
3467       && Desc.getOperandConstraint(OpNum + 1, MCOI::OperandConstraint::TIED_TO) == -1;
3468 }
3469 
3470 void AMDGPUAsmParser::cvtVOP3Impl(MCInst &Inst, const OperandVector &Operands,
3471                                   OptionalImmIndexMap &OptionalIdx) {
3472   unsigned I = 1;
3473   const MCInstrDesc &Desc = MII.get(Inst.getOpcode());
3474   for (unsigned J = 0; J < Desc.getNumDefs(); ++J) {
3475     ((AMDGPUOperand &)*Operands[I++]).addRegOperands(Inst, 1);
3476   }
3477 
3478   for (unsigned E = Operands.size(); I != E; ++I) {
3479     AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[I]);
3480     if (isRegOrImmWithInputMods(Desc, Inst.getNumOperands())) {
3481       Op.addRegOrImmWithFPInputModsOperands(Inst, 2);
3482     } else if (Op.isImm()) {
3483       OptionalIdx[Op.getImmTy()] = I;
3484     } else {
3485       llvm_unreachable("unhandled operand type");
3486     }
3487   }
3488 }
3489 
3490 void AMDGPUAsmParser::cvtVOP3(MCInst &Inst, const OperandVector &Operands) {
3491   OptionalImmIndexMap OptionalIdx;
3492 
3493   cvtVOP3Impl(Inst, Operands, OptionalIdx);
3494 
3495   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyClampSI);
3496   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyOModSI);
3497 
3498   // special case v_mac_{f16, f32}:
3499   // it has src2 register operand that is tied to dst operand
3500   // we don't allow modifiers for this operand in assembler so src2_modifiers
3501   // should be 0
3502   if (Inst.getOpcode() == AMDGPU::V_MAC_F32_e64_si ||
3503       Inst.getOpcode() == AMDGPU::V_MAC_F32_e64_vi ||
3504       Inst.getOpcode() == AMDGPU::V_MAC_F16_e64_vi) {
3505     auto it = Inst.begin();
3506     std::advance(
3507       it,
3508       AMDGPU::getNamedOperandIdx(Inst.getOpcode() == AMDGPU::V_MAC_F16_e64_vi ?
3509                                      AMDGPU::V_MAC_F16_e64 :
3510                                      AMDGPU::V_MAC_F32_e64,
3511                                  AMDGPU::OpName::src2_modifiers));
3512     it = Inst.insert(it, MCOperand::createImm(0)); // no modifiers for src2
3513     ++it;
3514     Inst.insert(it, Inst.getOperand(0)); // src2 = dst
3515   }
3516 }
3517 
3518 void AMDGPUAsmParser::cvtVOP3P(MCInst &Inst, const OperandVector &Operands) {
3519   OptionalImmIndexMap OptIdx;
3520 
3521   cvtVOP3Impl(Inst, Operands, OptIdx);
3522 
3523   // FIXME: This is messy. Parse the modifiers as if it was a normal VOP3
3524   // instruction, and then figure out where to actually put the modifiers
3525   int Opc = Inst.getOpcode();
3526 
3527   if (AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::clamp) != -1) {
3528     addOptionalImmOperand(Inst, Operands, OptIdx, AMDGPUOperand::ImmTyClampSI);
3529   }
3530 
3531   addOptionalImmOperand(Inst, Operands, OptIdx, AMDGPUOperand::ImmTyOpSel);
3532   addOptionalImmOperand(Inst, Operands, OptIdx, AMDGPUOperand::ImmTyOpSelHi, -1);
3533 
3534   int NegLoIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::neg_lo);
3535   if (NegLoIdx != -1) {
3536     addOptionalImmOperand(Inst, Operands, OptIdx, AMDGPUOperand::ImmTyNegLo);
3537     addOptionalImmOperand(Inst, Operands, OptIdx, AMDGPUOperand::ImmTyNegHi);
3538   }
3539 
3540   const int Ops[] = { AMDGPU::OpName::src0,
3541                       AMDGPU::OpName::src1,
3542                       AMDGPU::OpName::src2 };
3543   const int ModOps[] = { AMDGPU::OpName::src0_modifiers,
3544                          AMDGPU::OpName::src1_modifiers,
3545                          AMDGPU::OpName::src2_modifiers };
3546 
3547   int OpSelIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::op_sel);
3548   int OpSelHiIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::op_sel_hi);
3549 
3550   unsigned OpSel = Inst.getOperand(OpSelIdx).getImm();
3551   unsigned OpSelHi = Inst.getOperand(OpSelHiIdx).getImm();
3552   unsigned NegLo = 0;
3553   unsigned NegHi = 0;
3554 
3555   if (NegLoIdx != -1) {
3556     int NegHiIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::neg_hi);
3557     NegLo = Inst.getOperand(NegLoIdx).getImm();
3558     NegHi = Inst.getOperand(NegHiIdx).getImm();
3559   }
3560 
3561   for (int J = 0; J < 3; ++J) {
3562     int OpIdx = AMDGPU::getNamedOperandIdx(Opc, Ops[J]);
3563     if (OpIdx == -1)
3564       break;
3565 
3566     uint32_t ModVal = 0;
3567 
3568     if ((OpSel & (1 << J)) != 0)
3569       ModVal |= SISrcMods::OP_SEL_0;
3570 
3571     if ((OpSelHi & (1 << J)) != 0)
3572       ModVal |= SISrcMods::OP_SEL_1;
3573 
3574     if ((NegLo & (1 << J)) != 0)
3575       ModVal |= SISrcMods::NEG;
3576 
3577     if ((NegHi & (1 << J)) != 0)
3578       ModVal |= SISrcMods::NEG_HI;
3579 
3580     int ModIdx = AMDGPU::getNamedOperandIdx(Opc, ModOps[J]);
3581 
3582     Inst.getOperand(ModIdx).setImm(ModVal);
3583   }
3584 }
3585 
3586 //===----------------------------------------------------------------------===//
3587 // dpp
3588 //===----------------------------------------------------------------------===//
3589 
3590 bool AMDGPUOperand::isDPPCtrl() const {
3591   bool result = isImm() && getImmTy() == ImmTyDppCtrl && isUInt<9>(getImm());
3592   if (result) {
3593     int64_t Imm = getImm();
3594     return ((Imm >= 0x000) && (Imm <= 0x0ff)) ||
3595            ((Imm >= 0x101) && (Imm <= 0x10f)) ||
3596            ((Imm >= 0x111) && (Imm <= 0x11f)) ||
3597            ((Imm >= 0x121) && (Imm <= 0x12f)) ||
3598            (Imm == 0x130) ||
3599            (Imm == 0x134) ||
3600            (Imm == 0x138) ||
3601            (Imm == 0x13c) ||
3602            (Imm == 0x140) ||
3603            (Imm == 0x141) ||
3604            (Imm == 0x142) ||
3605            (Imm == 0x143);
3606   }
3607   return false;
3608 }
3609 
3610 bool AMDGPUOperand::isGPRIdxMode() const {
3611   return isImm() && isUInt<4>(getImm());
3612 }
3613 
3614 OperandMatchResultTy
3615 AMDGPUAsmParser::parseDPPCtrl(OperandVector &Operands) {
3616   SMLoc S = Parser.getTok().getLoc();
3617   StringRef Prefix;
3618   int64_t Int;
3619 
3620   if (getLexer().getKind() == AsmToken::Identifier) {
3621     Prefix = Parser.getTok().getString();
3622   } else {
3623     return MatchOperand_NoMatch;
3624   }
3625 
3626   if (Prefix == "row_mirror") {
3627     Int = 0x140;
3628     Parser.Lex();
3629   } else if (Prefix == "row_half_mirror") {
3630     Int = 0x141;
3631     Parser.Lex();
3632   } else {
3633     // Check to prevent parseDPPCtrlOps from eating invalid tokens
3634     if (Prefix != "quad_perm"
3635         && Prefix != "row_shl"
3636         && Prefix != "row_shr"
3637         && Prefix != "row_ror"
3638         && Prefix != "wave_shl"
3639         && Prefix != "wave_rol"
3640         && Prefix != "wave_shr"
3641         && Prefix != "wave_ror"
3642         && Prefix != "row_bcast") {
3643       return MatchOperand_NoMatch;
3644     }
3645 
3646     Parser.Lex();
3647     if (getLexer().isNot(AsmToken::Colon))
3648       return MatchOperand_ParseFail;
3649 
3650     if (Prefix == "quad_perm") {
3651       // quad_perm:[%d,%d,%d,%d]
3652       Parser.Lex();
3653       if (getLexer().isNot(AsmToken::LBrac))
3654         return MatchOperand_ParseFail;
3655       Parser.Lex();
3656 
3657       if (getParser().parseAbsoluteExpression(Int) || !(0 <= Int && Int <=3))
3658         return MatchOperand_ParseFail;
3659 
3660       for (int i = 0; i < 3; ++i) {
3661         if (getLexer().isNot(AsmToken::Comma))
3662           return MatchOperand_ParseFail;
3663         Parser.Lex();
3664 
3665         int64_t Temp;
3666         if (getParser().parseAbsoluteExpression(Temp) || !(0 <= Temp && Temp <=3))
3667           return MatchOperand_ParseFail;
3668         const int shift = i*2 + 2;
3669         Int += (Temp << shift);
3670       }
3671 
3672       if (getLexer().isNot(AsmToken::RBrac))
3673         return MatchOperand_ParseFail;
3674       Parser.Lex();
3675 
3676     } else {
3677       // sel:%d
3678       Parser.Lex();
3679       if (getParser().parseAbsoluteExpression(Int))
3680         return MatchOperand_ParseFail;
3681 
3682       if (Prefix == "row_shl" && 1 <= Int && Int <= 15) {
3683         Int |= 0x100;
3684       } else if (Prefix == "row_shr" && 1 <= Int && Int <= 15) {
3685         Int |= 0x110;
3686       } else if (Prefix == "row_ror" && 1 <= Int && Int <= 15) {
3687         Int |= 0x120;
3688       } else if (Prefix == "wave_shl" && 1 == Int) {
3689         Int = 0x130;
3690       } else if (Prefix == "wave_rol" && 1 == Int) {
3691         Int = 0x134;
3692       } else if (Prefix == "wave_shr" && 1 == Int) {
3693         Int = 0x138;
3694       } else if (Prefix == "wave_ror" && 1 == Int) {
3695         Int = 0x13C;
3696       } else if (Prefix == "row_bcast") {
3697         if (Int == 15) {
3698           Int = 0x142;
3699         } else if (Int == 31) {
3700           Int = 0x143;
3701         } else {
3702           return MatchOperand_ParseFail;
3703         }
3704       } else {
3705         return MatchOperand_ParseFail;
3706       }
3707     }
3708   }
3709 
3710   Operands.push_back(AMDGPUOperand::CreateImm(this, Int, S, AMDGPUOperand::ImmTyDppCtrl));
3711   return MatchOperand_Success;
3712 }
3713 
3714 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultRowMask() const {
3715   return AMDGPUOperand::CreateImm(this, 0xf, SMLoc(), AMDGPUOperand::ImmTyDppRowMask);
3716 }
3717 
3718 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultBankMask() const {
3719   return AMDGPUOperand::CreateImm(this, 0xf, SMLoc(), AMDGPUOperand::ImmTyDppBankMask);
3720 }
3721 
3722 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultBoundCtrl() const {
3723   return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTyDppBoundCtrl);
3724 }
3725 
3726 void AMDGPUAsmParser::cvtDPP(MCInst &Inst, const OperandVector &Operands) {
3727   OptionalImmIndexMap OptionalIdx;
3728 
3729   unsigned I = 1;
3730   const MCInstrDesc &Desc = MII.get(Inst.getOpcode());
3731   for (unsigned J = 0; J < Desc.getNumDefs(); ++J) {
3732     ((AMDGPUOperand &)*Operands[I++]).addRegOperands(Inst, 1);
3733   }
3734 
3735   for (unsigned E = Operands.size(); I != E; ++I) {
3736     AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[I]);
3737     // Add the register arguments
3738     if (Op.isReg() && Op.Reg.RegNo == AMDGPU::VCC) {
3739       // VOP2b (v_add_u32, v_sub_u32 ...) dpp use "vcc" token.
3740       // Skip it.
3741       continue;
3742     } if (isRegOrImmWithInputMods(Desc, Inst.getNumOperands())) {
3743       Op.addRegWithFPInputModsOperands(Inst, 2);
3744     } else if (Op.isDPPCtrl()) {
3745       Op.addImmOperands(Inst, 1);
3746     } else if (Op.isImm()) {
3747       // Handle optional arguments
3748       OptionalIdx[Op.getImmTy()] = I;
3749     } else {
3750       llvm_unreachable("Invalid operand type");
3751     }
3752   }
3753 
3754   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyDppRowMask, 0xf);
3755   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyDppBankMask, 0xf);
3756   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyDppBoundCtrl);
3757 
3758   // special case v_mac_{f16, f32}:
3759   // it has src2 register operand that is tied to dst operand
3760   if (Inst.getOpcode() == AMDGPU::V_MAC_F32_dpp ||
3761       Inst.getOpcode() == AMDGPU::V_MAC_F16_dpp) {
3762     auto it = Inst.begin();
3763     std::advance(
3764         it, AMDGPU::getNamedOperandIdx(Inst.getOpcode(), AMDGPU::OpName::src2));
3765     Inst.insert(it, Inst.getOperand(0)); // src2 = dst
3766   }
3767 }
3768 
3769 //===----------------------------------------------------------------------===//
3770 // sdwa
3771 //===----------------------------------------------------------------------===//
3772 
3773 OperandMatchResultTy
3774 AMDGPUAsmParser::parseSDWASel(OperandVector &Operands, StringRef Prefix,
3775                               AMDGPUOperand::ImmTy Type) {
3776   using namespace llvm::AMDGPU::SDWA;
3777 
3778   SMLoc S = Parser.getTok().getLoc();
3779   StringRef Value;
3780   OperandMatchResultTy res;
3781 
3782   res = parseStringWithPrefix(Prefix, Value);
3783   if (res != MatchOperand_Success) {
3784     return res;
3785   }
3786 
3787   int64_t Int;
3788   Int = StringSwitch<int64_t>(Value)
3789         .Case("BYTE_0", SdwaSel::BYTE_0)
3790         .Case("BYTE_1", SdwaSel::BYTE_1)
3791         .Case("BYTE_2", SdwaSel::BYTE_2)
3792         .Case("BYTE_3", SdwaSel::BYTE_3)
3793         .Case("WORD_0", SdwaSel::WORD_0)
3794         .Case("WORD_1", SdwaSel::WORD_1)
3795         .Case("DWORD", SdwaSel::DWORD)
3796         .Default(0xffffffff);
3797   Parser.Lex(); // eat last token
3798 
3799   if (Int == 0xffffffff) {
3800     return MatchOperand_ParseFail;
3801   }
3802 
3803   Operands.push_back(AMDGPUOperand::CreateImm(this, Int, S, Type));
3804   return MatchOperand_Success;
3805 }
3806 
3807 OperandMatchResultTy
3808 AMDGPUAsmParser::parseSDWADstUnused(OperandVector &Operands) {
3809   using namespace llvm::AMDGPU::SDWA;
3810 
3811   SMLoc S = Parser.getTok().getLoc();
3812   StringRef Value;
3813   OperandMatchResultTy res;
3814 
3815   res = parseStringWithPrefix("dst_unused", Value);
3816   if (res != MatchOperand_Success) {
3817     return res;
3818   }
3819 
3820   int64_t Int;
3821   Int = StringSwitch<int64_t>(Value)
3822         .Case("UNUSED_PAD", DstUnused::UNUSED_PAD)
3823         .Case("UNUSED_SEXT", DstUnused::UNUSED_SEXT)
3824         .Case("UNUSED_PRESERVE", DstUnused::UNUSED_PRESERVE)
3825         .Default(0xffffffff);
3826   Parser.Lex(); // eat last token
3827 
3828   if (Int == 0xffffffff) {
3829     return MatchOperand_ParseFail;
3830   }
3831 
3832   Operands.push_back(AMDGPUOperand::CreateImm(this, Int, S, AMDGPUOperand::ImmTySdwaDstUnused));
3833   return MatchOperand_Success;
3834 }
3835 
3836 void AMDGPUAsmParser::cvtSdwaVOP1(MCInst &Inst, const OperandVector &Operands) {
3837   cvtSDWA(Inst, Operands, SIInstrFlags::VOP1);
3838 }
3839 
3840 void AMDGPUAsmParser::cvtSdwaVOP2(MCInst &Inst, const OperandVector &Operands) {
3841   cvtSDWA(Inst, Operands, SIInstrFlags::VOP2);
3842 }
3843 
3844 void AMDGPUAsmParser::cvtSdwaVOPC(MCInst &Inst, const OperandVector &Operands) {
3845   cvtSDWA(Inst, Operands, SIInstrFlags::VOPC);
3846 }
3847 
3848 void AMDGPUAsmParser::cvtSDWA(MCInst &Inst, const OperandVector &Operands,
3849                               uint64_t BasicInstType) {
3850   using namespace llvm::AMDGPU::SDWA;
3851   OptionalImmIndexMap OptionalIdx;
3852 
3853   unsigned I = 1;
3854   const MCInstrDesc &Desc = MII.get(Inst.getOpcode());
3855   for (unsigned J = 0; J < Desc.getNumDefs(); ++J) {
3856     ((AMDGPUOperand &)*Operands[I++]).addRegOperands(Inst, 1);
3857   }
3858 
3859   for (unsigned E = Operands.size(); I != E; ++I) {
3860     AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[I]);
3861     // Add the register arguments
3862     if ((BasicInstType == SIInstrFlags::VOPC ||
3863          BasicInstType == SIInstrFlags::VOP2)&&
3864         Op.isReg() &&
3865         Op.Reg.RegNo == AMDGPU::VCC) {
3866       // VOPC and VOP2b (v_add_u32, v_sub_u32 ...) sdwa use "vcc" token as dst.
3867       // Skip it.
3868       continue;
3869     } else if (isRegOrImmWithInputMods(Desc, Inst.getNumOperands())) {
3870       Op.addRegWithInputModsOperands(Inst, 2);
3871     } else if (Op.isImm()) {
3872       // Handle optional arguments
3873       OptionalIdx[Op.getImmTy()] = I;
3874     } else {
3875       llvm_unreachable("Invalid operand type");
3876     }
3877   }
3878 
3879   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyClampSI, 0);
3880 
3881   if (Inst.getOpcode() != AMDGPU::V_NOP_sdwa_vi) {
3882     // V_NOP_sdwa_vi has no optional sdwa arguments
3883     switch (BasicInstType) {
3884     case SIInstrFlags::VOP1:
3885       addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySdwaDstSel, SdwaSel::DWORD);
3886       addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySdwaDstUnused, DstUnused::UNUSED_PRESERVE);
3887       addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySdwaSrc0Sel, SdwaSel::DWORD);
3888       break;
3889 
3890     case SIInstrFlags::VOP2:
3891       addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySdwaDstSel, SdwaSel::DWORD);
3892       addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySdwaDstUnused, DstUnused::UNUSED_PRESERVE);
3893       addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySdwaSrc0Sel, SdwaSel::DWORD);
3894       addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySdwaSrc1Sel, SdwaSel::DWORD);
3895       break;
3896 
3897     case SIInstrFlags::VOPC:
3898       addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySdwaSrc0Sel, SdwaSel::DWORD);
3899       addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySdwaSrc1Sel, SdwaSel::DWORD);
3900       break;
3901 
3902     default:
3903       llvm_unreachable("Invalid instruction type. Only VOP1, VOP2 and VOPC allowed");
3904     }
3905   }
3906 
3907   // special case v_mac_{f16, f32}:
3908   // it has src2 register operand that is tied to dst operand
3909   if (Inst.getOpcode() == AMDGPU::V_MAC_F32_sdwa_vi ||
3910       Inst.getOpcode() == AMDGPU::V_MAC_F16_sdwa_vi)  {
3911     auto it = Inst.begin();
3912     std::advance(
3913         it, AMDGPU::getNamedOperandIdx(Inst.getOpcode(), AMDGPU::OpName::src2));
3914     Inst.insert(it, Inst.getOperand(0)); // src2 = dst
3915   }
3916 
3917 }
3918 
3919 /// Force static initialization.
3920 extern "C" void LLVMInitializeAMDGPUAsmParser() {
3921   RegisterMCAsmParser<AMDGPUAsmParser> A(getTheAMDGPUTarget());
3922   RegisterMCAsmParser<AMDGPUAsmParser> B(getTheGCNTarget());
3923 }
3924 
3925 #define GET_REGISTER_MATCHER
3926 #define GET_MATCHER_IMPLEMENTATION
3927 #include "AMDGPUGenAsmMatcher.inc"
3928 
3929 // This fuction should be defined after auto-generated include so that we have
3930 // MatchClassKind enum defined
3931 unsigned AMDGPUAsmParser::validateTargetOperandClass(MCParsedAsmOperand &Op,
3932                                                      unsigned Kind) {
3933   // Tokens like "glc" would be parsed as immediate operands in ParseOperand().
3934   // But MatchInstructionImpl() expects to meet token and fails to validate
3935   // operand. This method checks if we are given immediate operand but expect to
3936   // get corresponding token.
3937   AMDGPUOperand &Operand = (AMDGPUOperand&)Op;
3938   switch (Kind) {
3939   case MCK_addr64:
3940     return Operand.isAddr64() ? Match_Success : Match_InvalidOperand;
3941   case MCK_gds:
3942     return Operand.isGDS() ? Match_Success : Match_InvalidOperand;
3943   case MCK_glc:
3944     return Operand.isGLC() ? Match_Success : Match_InvalidOperand;
3945   case MCK_idxen:
3946     return Operand.isIdxen() ? Match_Success : Match_InvalidOperand;
3947   case MCK_offen:
3948     return Operand.isOffen() ? Match_Success : Match_InvalidOperand;
3949   case MCK_SSrcB32:
3950     // When operands have expression values, they will return true for isToken,
3951     // because it is not possible to distinguish between a token and an
3952     // expression at parse time. MatchInstructionImpl() will always try to
3953     // match an operand as a token, when isToken returns true, and when the
3954     // name of the expression is not a valid token, the match will fail,
3955     // so we need to handle it here.
3956     return Operand.isSSrcB32() ? Match_Success : Match_InvalidOperand;
3957   case MCK_SSrcF32:
3958     return Operand.isSSrcF32() ? Match_Success : Match_InvalidOperand;
3959   case MCK_SoppBrTarget:
3960     return Operand.isSoppBrTarget() ? Match_Success : Match_InvalidOperand;
3961   case MCK_VReg32OrOff:
3962     return Operand.isVReg32OrOff() ? Match_Success : Match_InvalidOperand;
3963   case MCK_InterpSlot:
3964     return Operand.isInterpSlot() ? Match_Success : Match_InvalidOperand;
3965   case MCK_Attr:
3966     return Operand.isInterpAttr() ? Match_Success : Match_InvalidOperand;
3967   case MCK_AttrChan:
3968     return Operand.isAttrChan() ? Match_Success : Match_InvalidOperand;
3969   default:
3970     return Match_InvalidOperand;
3971   }
3972 }
3973