1 //===- AMDGPUAsmParser.cpp - Parse SI asm to MCInst instructions ----------===//
2 //
3 //                     The LLVM Compiler Infrastructure
4 //
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
7 //
8 //===----------------------------------------------------------------------===//
9 
10 #include "AMDGPU.h"
11 #include "AMDKernelCodeT.h"
12 #include "MCTargetDesc/AMDGPUMCTargetDesc.h"
13 #include "MCTargetDesc/AMDGPUTargetStreamer.h"
14 #include "SIDefines.h"
15 #include "Utils/AMDGPUAsmUtils.h"
16 #include "Utils/AMDGPUBaseInfo.h"
17 #include "Utils/AMDKernelCodeTUtils.h"
18 #include "llvm/ADT/APFloat.h"
19 #include "llvm/ADT/APInt.h"
20 #include "llvm/ADT/ArrayRef.h"
21 #include "llvm/ADT/STLExtras.h"
22 #include "llvm/ADT/SmallBitVector.h"
23 #include "llvm/ADT/SmallString.h"
24 #include "llvm/ADT/StringRef.h"
25 #include "llvm/ADT/StringSwitch.h"
26 #include "llvm/ADT/Twine.h"
27 #include "llvm/BinaryFormat/ELF.h"
28 #include "llvm/CodeGen/MachineValueType.h"
29 #include "llvm/MC/MCAsmInfo.h"
30 #include "llvm/MC/MCContext.h"
31 #include "llvm/MC/MCExpr.h"
32 #include "llvm/MC/MCInst.h"
33 #include "llvm/MC/MCInstrDesc.h"
34 #include "llvm/MC/MCInstrInfo.h"
35 #include "llvm/MC/MCParser/MCAsmLexer.h"
36 #include "llvm/MC/MCParser/MCAsmParser.h"
37 #include "llvm/MC/MCParser/MCAsmParserExtension.h"
38 #include "llvm/MC/MCParser/MCParsedAsmOperand.h"
39 #include "llvm/MC/MCParser/MCTargetAsmParser.h"
40 #include "llvm/MC/MCRegisterInfo.h"
41 #include "llvm/MC/MCStreamer.h"
42 #include "llvm/MC/MCSubtargetInfo.h"
43 #include "llvm/MC/MCSymbol.h"
44 #include "llvm/Support/AMDGPUMetadata.h"
45 #include "llvm/Support/Casting.h"
46 #include "llvm/Support/Compiler.h"
47 #include "llvm/Support/ErrorHandling.h"
48 #include "llvm/Support/MathExtras.h"
49 #include "llvm/Support/SMLoc.h"
50 #include "llvm/Support/TargetRegistry.h"
51 #include "llvm/Support/raw_ostream.h"
52 #include <algorithm>
53 #include <cassert>
54 #include <cstdint>
55 #include <cstring>
56 #include <iterator>
57 #include <map>
58 #include <memory>
59 #include <string>
60 
61 using namespace llvm;
62 using namespace llvm::AMDGPU;
63 
64 namespace {
65 
66 class AMDGPUAsmParser;
67 
68 enum RegisterKind { IS_UNKNOWN, IS_VGPR, IS_SGPR, IS_TTMP, IS_SPECIAL };
69 
70 //===----------------------------------------------------------------------===//
71 // Operand
72 //===----------------------------------------------------------------------===//
73 
74 class AMDGPUOperand : public MCParsedAsmOperand {
75   enum KindTy {
76     Token,
77     Immediate,
78     Register,
79     Expression
80   } Kind;
81 
82   SMLoc StartLoc, EndLoc;
83   const AMDGPUAsmParser *AsmParser;
84 
85 public:
86   AMDGPUOperand(KindTy Kind_, const AMDGPUAsmParser *AsmParser_)
87     : MCParsedAsmOperand(), Kind(Kind_), AsmParser(AsmParser_) {}
88 
89   using Ptr = std::unique_ptr<AMDGPUOperand>;
90 
91   struct Modifiers {
92     bool Abs = false;
93     bool Neg = false;
94     bool Sext = false;
95 
96     bool hasFPModifiers() const { return Abs || Neg; }
97     bool hasIntModifiers() const { return Sext; }
98     bool hasModifiers() const { return hasFPModifiers() || hasIntModifiers(); }
99 
100     int64_t getFPModifiersOperand() const {
101       int64_t Operand = 0;
102       Operand |= Abs ? SISrcMods::ABS : 0;
103       Operand |= Neg ? SISrcMods::NEG : 0;
104       return Operand;
105     }
106 
107     int64_t getIntModifiersOperand() const {
108       int64_t Operand = 0;
109       Operand |= Sext ? SISrcMods::SEXT : 0;
110       return Operand;
111     }
112 
113     int64_t getModifiersOperand() const {
114       assert(!(hasFPModifiers() && hasIntModifiers())
115            && "fp and int modifiers should not be used simultaneously");
116       if (hasFPModifiers()) {
117         return getFPModifiersOperand();
118       } else if (hasIntModifiers()) {
119         return getIntModifiersOperand();
120       } else {
121         return 0;
122       }
123     }
124 
125     friend raw_ostream &operator <<(raw_ostream &OS, AMDGPUOperand::Modifiers Mods);
126   };
127 
128   enum ImmTy {
129     ImmTyNone,
130     ImmTyGDS,
131     ImmTyOffen,
132     ImmTyIdxen,
133     ImmTyAddr64,
134     ImmTyOffset,
135     ImmTyInstOffset,
136     ImmTyOffset0,
137     ImmTyOffset1,
138     ImmTyGLC,
139     ImmTySLC,
140     ImmTyTFE,
141     ImmTyClampSI,
142     ImmTyOModSI,
143     ImmTyDppCtrl,
144     ImmTyDppRowMask,
145     ImmTyDppBankMask,
146     ImmTyDppBoundCtrl,
147     ImmTySdwaDstSel,
148     ImmTySdwaSrc0Sel,
149     ImmTySdwaSrc1Sel,
150     ImmTySdwaDstUnused,
151     ImmTyDMask,
152     ImmTyUNorm,
153     ImmTyDA,
154     ImmTyR128,
155     ImmTyLWE,
156     ImmTyExpTgt,
157     ImmTyExpCompr,
158     ImmTyExpVM,
159     ImmTyDFMT,
160     ImmTyNFMT,
161     ImmTyHwreg,
162     ImmTyOff,
163     ImmTySendMsg,
164     ImmTyInterpSlot,
165     ImmTyInterpAttr,
166     ImmTyAttrChan,
167     ImmTyOpSel,
168     ImmTyOpSelHi,
169     ImmTyNegLo,
170     ImmTyNegHi,
171     ImmTySwizzle,
172     ImmTyHigh
173   };
174 
175   struct TokOp {
176     const char *Data;
177     unsigned Length;
178   };
179 
180   struct ImmOp {
181     int64_t Val;
182     ImmTy Type;
183     bool IsFPImm;
184     Modifiers Mods;
185   };
186 
187   struct RegOp {
188     unsigned RegNo;
189     bool IsForcedVOP3;
190     Modifiers Mods;
191   };
192 
193   union {
194     TokOp Tok;
195     ImmOp Imm;
196     RegOp Reg;
197     const MCExpr *Expr;
198   };
199 
200   bool isToken() const override {
201     if (Kind == Token)
202       return true;
203 
204     if (Kind != Expression || !Expr)
205       return false;
206 
207     // When parsing operands, we can't always tell if something was meant to be
208     // a token, like 'gds', or an expression that references a global variable.
209     // In this case, we assume the string is an expression, and if we need to
210     // interpret is a token, then we treat the symbol name as the token.
211     return isa<MCSymbolRefExpr>(Expr);
212   }
213 
214   bool isImm() const override {
215     return Kind == Immediate;
216   }
217 
218   bool isInlinableImm(MVT type) const;
219   bool isLiteralImm(MVT type) const;
220 
221   bool isRegKind() const {
222     return Kind == Register;
223   }
224 
225   bool isReg() const override {
226     return isRegKind() && !hasModifiers();
227   }
228 
229   bool isRegOrImmWithInputMods(MVT type) const {
230     return isRegKind() || isInlinableImm(type);
231   }
232 
233   bool isRegOrImmWithInt16InputMods() const {
234     return isRegOrImmWithInputMods(MVT::i16);
235   }
236 
237   bool isRegOrImmWithInt32InputMods() const {
238     return isRegOrImmWithInputMods(MVT::i32);
239   }
240 
241   bool isRegOrImmWithInt64InputMods() const {
242     return isRegOrImmWithInputMods(MVT::i64);
243   }
244 
245   bool isRegOrImmWithFP16InputMods() const {
246     return isRegOrImmWithInputMods(MVT::f16);
247   }
248 
249   bool isRegOrImmWithFP32InputMods() const {
250     return isRegOrImmWithInputMods(MVT::f32);
251   }
252 
253   bool isRegOrImmWithFP64InputMods() const {
254     return isRegOrImmWithInputMods(MVT::f64);
255   }
256 
257   bool isVReg() const {
258     return isRegClass(AMDGPU::VGPR_32RegClassID) ||
259            isRegClass(AMDGPU::VReg_64RegClassID) ||
260            isRegClass(AMDGPU::VReg_96RegClassID) ||
261            isRegClass(AMDGPU::VReg_128RegClassID) ||
262            isRegClass(AMDGPU::VReg_256RegClassID) ||
263            isRegClass(AMDGPU::VReg_512RegClassID);
264   }
265 
266   bool isVReg32OrOff() const {
267     return isOff() || isRegClass(AMDGPU::VGPR_32RegClassID);
268   }
269 
270   bool isSDWAOperand(MVT type) const;
271   bool isSDWAFP16Operand() const;
272   bool isSDWAFP32Operand() const;
273   bool isSDWAInt16Operand() const;
274   bool isSDWAInt32Operand() const;
275 
276   bool isImmTy(ImmTy ImmT) const {
277     return isImm() && Imm.Type == ImmT;
278   }
279 
280   bool isImmModifier() const {
281     return isImm() && Imm.Type != ImmTyNone;
282   }
283 
284   bool isClampSI() const { return isImmTy(ImmTyClampSI); }
285   bool isOModSI() const { return isImmTy(ImmTyOModSI); }
286   bool isDMask() const { return isImmTy(ImmTyDMask); }
287   bool isUNorm() const { return isImmTy(ImmTyUNorm); }
288   bool isDA() const { return isImmTy(ImmTyDA); }
289   bool isR128() const { return isImmTy(ImmTyUNorm); }
290   bool isLWE() const { return isImmTy(ImmTyLWE); }
291   bool isOff() const { return isImmTy(ImmTyOff); }
292   bool isExpTgt() const { return isImmTy(ImmTyExpTgt); }
293   bool isExpVM() const { return isImmTy(ImmTyExpVM); }
294   bool isExpCompr() const { return isImmTy(ImmTyExpCompr); }
295   bool isOffen() const { return isImmTy(ImmTyOffen); }
296   bool isIdxen() const { return isImmTy(ImmTyIdxen); }
297   bool isAddr64() const { return isImmTy(ImmTyAddr64); }
298   bool isOffset() const { return isImmTy(ImmTyOffset) && isUInt<16>(getImm()); }
299   bool isOffset0() const { return isImmTy(ImmTyOffset0) && isUInt<16>(getImm()); }
300   bool isOffset1() const { return isImmTy(ImmTyOffset1) && isUInt<8>(getImm()); }
301 
302   bool isOffsetU12() const { return (isImmTy(ImmTyOffset) || isImmTy(ImmTyInstOffset)) && isUInt<12>(getImm()); }
303   bool isOffsetS13() const { return (isImmTy(ImmTyOffset) || isImmTy(ImmTyInstOffset)) && isInt<13>(getImm()); }
304   bool isGDS() const { return isImmTy(ImmTyGDS); }
305   bool isGLC() const { return isImmTy(ImmTyGLC); }
306   bool isSLC() const { return isImmTy(ImmTySLC); }
307   bool isTFE() const { return isImmTy(ImmTyTFE); }
308   bool isDFMT() const { return isImmTy(ImmTyDFMT) && isUInt<8>(getImm()); }
309   bool isNFMT() const { return isImmTy(ImmTyNFMT) && isUInt<8>(getImm()); }
310   bool isBankMask() const { return isImmTy(ImmTyDppBankMask); }
311   bool isRowMask() const { return isImmTy(ImmTyDppRowMask); }
312   bool isBoundCtrl() const { return isImmTy(ImmTyDppBoundCtrl); }
313   bool isSDWADstSel() const { return isImmTy(ImmTySdwaDstSel); }
314   bool isSDWASrc0Sel() const { return isImmTy(ImmTySdwaSrc0Sel); }
315   bool isSDWASrc1Sel() const { return isImmTy(ImmTySdwaSrc1Sel); }
316   bool isSDWADstUnused() const { return isImmTy(ImmTySdwaDstUnused); }
317   bool isInterpSlot() const { return isImmTy(ImmTyInterpSlot); }
318   bool isInterpAttr() const { return isImmTy(ImmTyInterpAttr); }
319   bool isAttrChan() const { return isImmTy(ImmTyAttrChan); }
320   bool isOpSel() const { return isImmTy(ImmTyOpSel); }
321   bool isOpSelHi() const { return isImmTy(ImmTyOpSelHi); }
322   bool isNegLo() const { return isImmTy(ImmTyNegLo); }
323   bool isNegHi() const { return isImmTy(ImmTyNegHi); }
324   bool isHigh() const { return isImmTy(ImmTyHigh); }
325 
326   bool isMod() const {
327     return isClampSI() || isOModSI();
328   }
329 
330   bool isRegOrImm() const {
331     return isReg() || isImm();
332   }
333 
334   bool isRegClass(unsigned RCID) const;
335 
336   bool isRegOrInlineNoMods(unsigned RCID, MVT type) const {
337     return (isRegClass(RCID) || isInlinableImm(type)) && !hasModifiers();
338   }
339 
340   bool isSCSrcB16() const {
341     return isRegOrInlineNoMods(AMDGPU::SReg_32RegClassID, MVT::i16);
342   }
343 
344   bool isSCSrcV2B16() const {
345     return isSCSrcB16();
346   }
347 
348   bool isSCSrcB32() const {
349     return isRegOrInlineNoMods(AMDGPU::SReg_32RegClassID, MVT::i32);
350   }
351 
352   bool isSCSrcB64() const {
353     return isRegOrInlineNoMods(AMDGPU::SReg_64RegClassID, MVT::i64);
354   }
355 
356   bool isSCSrcF16() const {
357     return isRegOrInlineNoMods(AMDGPU::SReg_32RegClassID, MVT::f16);
358   }
359 
360   bool isSCSrcV2F16() const {
361     return isSCSrcF16();
362   }
363 
364   bool isSCSrcF32() const {
365     return isRegOrInlineNoMods(AMDGPU::SReg_32RegClassID, MVT::f32);
366   }
367 
368   bool isSCSrcF64() const {
369     return isRegOrInlineNoMods(AMDGPU::SReg_64RegClassID, MVT::f64);
370   }
371 
372   bool isSSrcB32() const {
373     return isSCSrcB32() || isLiteralImm(MVT::i32) || isExpr();
374   }
375 
376   bool isSSrcB16() const {
377     return isSCSrcB16() || isLiteralImm(MVT::i16);
378   }
379 
380   bool isSSrcV2B16() const {
381     llvm_unreachable("cannot happen");
382     return isSSrcB16();
383   }
384 
385   bool isSSrcB64() const {
386     // TODO: Find out how SALU supports extension of 32-bit literals to 64 bits.
387     // See isVSrc64().
388     return isSCSrcB64() || isLiteralImm(MVT::i64);
389   }
390 
391   bool isSSrcF32() const {
392     return isSCSrcB32() || isLiteralImm(MVT::f32) || isExpr();
393   }
394 
395   bool isSSrcF64() const {
396     return isSCSrcB64() || isLiteralImm(MVT::f64);
397   }
398 
399   bool isSSrcF16() const {
400     return isSCSrcB16() || isLiteralImm(MVT::f16);
401   }
402 
403   bool isSSrcV2F16() const {
404     llvm_unreachable("cannot happen");
405     return isSSrcF16();
406   }
407 
408   bool isVCSrcB32() const {
409     return isRegOrInlineNoMods(AMDGPU::VS_32RegClassID, MVT::i32);
410   }
411 
412   bool isVCSrcB64() const {
413     return isRegOrInlineNoMods(AMDGPU::VS_64RegClassID, MVT::i64);
414   }
415 
416   bool isVCSrcB16() const {
417     return isRegOrInlineNoMods(AMDGPU::VS_32RegClassID, MVT::i16);
418   }
419 
420   bool isVCSrcV2B16() const {
421     return isVCSrcB16();
422   }
423 
424   bool isVCSrcF32() const {
425     return isRegOrInlineNoMods(AMDGPU::VS_32RegClassID, MVT::f32);
426   }
427 
428   bool isVCSrcF64() const {
429     return isRegOrInlineNoMods(AMDGPU::VS_64RegClassID, MVT::f64);
430   }
431 
432   bool isVCSrcF16() const {
433     return isRegOrInlineNoMods(AMDGPU::VS_32RegClassID, MVT::f16);
434   }
435 
436   bool isVCSrcV2F16() const {
437     return isVCSrcF16();
438   }
439 
440   bool isVSrcB32() const {
441     return isVCSrcF32() || isLiteralImm(MVT::i32);
442   }
443 
444   bool isVSrcB64() const {
445     return isVCSrcF64() || isLiteralImm(MVT::i64);
446   }
447 
448   bool isVSrcB16() const {
449     return isVCSrcF16() || isLiteralImm(MVT::i16);
450   }
451 
452   bool isVSrcV2B16() const {
453     llvm_unreachable("cannot happen");
454     return isVSrcB16();
455   }
456 
457   bool isVSrcF32() const {
458     return isVCSrcF32() || isLiteralImm(MVT::f32);
459   }
460 
461   bool isVSrcF64() const {
462     return isVCSrcF64() || isLiteralImm(MVT::f64);
463   }
464 
465   bool isVSrcF16() const {
466     return isVCSrcF16() || isLiteralImm(MVT::f16);
467   }
468 
469   bool isVSrcV2F16() const {
470     llvm_unreachable("cannot happen");
471     return isVSrcF16();
472   }
473 
474   bool isKImmFP32() const {
475     return isLiteralImm(MVT::f32);
476   }
477 
478   bool isKImmFP16() const {
479     return isLiteralImm(MVT::f16);
480   }
481 
482   bool isMem() const override {
483     return false;
484   }
485 
486   bool isExpr() const {
487     return Kind == Expression;
488   }
489 
490   bool isSoppBrTarget() const {
491     return isExpr() || isImm();
492   }
493 
494   bool isSWaitCnt() const;
495   bool isHwreg() const;
496   bool isSendMsg() const;
497   bool isSwizzle() const;
498   bool isSMRDOffset8() const;
499   bool isSMRDOffset20() const;
500   bool isSMRDLiteralOffset() const;
501   bool isDPPCtrl() const;
502   bool isGPRIdxMode() const;
503   bool isS16Imm() const;
504   bool isU16Imm() const;
505 
506   StringRef getExpressionAsToken() const {
507     assert(isExpr());
508     const MCSymbolRefExpr *S = cast<MCSymbolRefExpr>(Expr);
509     return S->getSymbol().getName();
510   }
511 
512   StringRef getToken() const {
513     assert(isToken());
514 
515     if (Kind == Expression)
516       return getExpressionAsToken();
517 
518     return StringRef(Tok.Data, Tok.Length);
519   }
520 
521   int64_t getImm() const {
522     assert(isImm());
523     return Imm.Val;
524   }
525 
526   ImmTy getImmTy() const {
527     assert(isImm());
528     return Imm.Type;
529   }
530 
531   unsigned getReg() const override {
532     return Reg.RegNo;
533   }
534 
535   SMLoc getStartLoc() const override {
536     return StartLoc;
537   }
538 
539   SMLoc getEndLoc() const override {
540     return EndLoc;
541   }
542 
543   SMRange getLocRange() const {
544     return SMRange(StartLoc, EndLoc);
545   }
546 
547   Modifiers getModifiers() const {
548     assert(isRegKind() || isImmTy(ImmTyNone));
549     return isRegKind() ? Reg.Mods : Imm.Mods;
550   }
551 
552   void setModifiers(Modifiers Mods) {
553     assert(isRegKind() || isImmTy(ImmTyNone));
554     if (isRegKind())
555       Reg.Mods = Mods;
556     else
557       Imm.Mods = Mods;
558   }
559 
560   bool hasModifiers() const {
561     return getModifiers().hasModifiers();
562   }
563 
564   bool hasFPModifiers() const {
565     return getModifiers().hasFPModifiers();
566   }
567 
568   bool hasIntModifiers() const {
569     return getModifiers().hasIntModifiers();
570   }
571 
572   uint64_t applyInputFPModifiers(uint64_t Val, unsigned Size) const;
573 
574   void addImmOperands(MCInst &Inst, unsigned N, bool ApplyModifiers = true) const;
575 
576   void addLiteralImmOperand(MCInst &Inst, int64_t Val, bool ApplyModifiers) const;
577 
578   template <unsigned Bitwidth>
579   void addKImmFPOperands(MCInst &Inst, unsigned N) const;
580 
581   void addKImmFP16Operands(MCInst &Inst, unsigned N) const {
582     addKImmFPOperands<16>(Inst, N);
583   }
584 
585   void addKImmFP32Operands(MCInst &Inst, unsigned N) const {
586     addKImmFPOperands<32>(Inst, N);
587   }
588 
589   void addRegOperands(MCInst &Inst, unsigned N) const;
590 
591   void addRegOrImmOperands(MCInst &Inst, unsigned N) const {
592     if (isRegKind())
593       addRegOperands(Inst, N);
594     else if (isExpr())
595       Inst.addOperand(MCOperand::createExpr(Expr));
596     else
597       addImmOperands(Inst, N);
598   }
599 
600   void addRegOrImmWithInputModsOperands(MCInst &Inst, unsigned N) const {
601     Modifiers Mods = getModifiers();
602     Inst.addOperand(MCOperand::createImm(Mods.getModifiersOperand()));
603     if (isRegKind()) {
604       addRegOperands(Inst, N);
605     } else {
606       addImmOperands(Inst, N, false);
607     }
608   }
609 
610   void addRegOrImmWithFPInputModsOperands(MCInst &Inst, unsigned N) const {
611     assert(!hasIntModifiers());
612     addRegOrImmWithInputModsOperands(Inst, N);
613   }
614 
615   void addRegOrImmWithIntInputModsOperands(MCInst &Inst, unsigned N) const {
616     assert(!hasFPModifiers());
617     addRegOrImmWithInputModsOperands(Inst, N);
618   }
619 
620   void addRegWithInputModsOperands(MCInst &Inst, unsigned N) const {
621     Modifiers Mods = getModifiers();
622     Inst.addOperand(MCOperand::createImm(Mods.getModifiersOperand()));
623     assert(isRegKind());
624     addRegOperands(Inst, N);
625   }
626 
627   void addRegWithFPInputModsOperands(MCInst &Inst, unsigned N) const {
628     assert(!hasIntModifiers());
629     addRegWithInputModsOperands(Inst, N);
630   }
631 
632   void addRegWithIntInputModsOperands(MCInst &Inst, unsigned N) const {
633     assert(!hasFPModifiers());
634     addRegWithInputModsOperands(Inst, N);
635   }
636 
637   void addSoppBrTargetOperands(MCInst &Inst, unsigned N) const {
638     if (isImm())
639       addImmOperands(Inst, N);
640     else {
641       assert(isExpr());
642       Inst.addOperand(MCOperand::createExpr(Expr));
643     }
644   }
645 
646   static void printImmTy(raw_ostream& OS, ImmTy Type) {
647     switch (Type) {
648     case ImmTyNone: OS << "None"; break;
649     case ImmTyGDS: OS << "GDS"; break;
650     case ImmTyOffen: OS << "Offen"; break;
651     case ImmTyIdxen: OS << "Idxen"; break;
652     case ImmTyAddr64: OS << "Addr64"; break;
653     case ImmTyOffset: OS << "Offset"; break;
654     case ImmTyInstOffset: OS << "InstOffset"; break;
655     case ImmTyOffset0: OS << "Offset0"; break;
656     case ImmTyOffset1: OS << "Offset1"; break;
657     case ImmTyGLC: OS << "GLC"; break;
658     case ImmTySLC: OS << "SLC"; break;
659     case ImmTyTFE: OS << "TFE"; break;
660     case ImmTyDFMT: OS << "DFMT"; break;
661     case ImmTyNFMT: OS << "NFMT"; break;
662     case ImmTyClampSI: OS << "ClampSI"; break;
663     case ImmTyOModSI: OS << "OModSI"; break;
664     case ImmTyDppCtrl: OS << "DppCtrl"; break;
665     case ImmTyDppRowMask: OS << "DppRowMask"; break;
666     case ImmTyDppBankMask: OS << "DppBankMask"; break;
667     case ImmTyDppBoundCtrl: OS << "DppBoundCtrl"; break;
668     case ImmTySdwaDstSel: OS << "SdwaDstSel"; break;
669     case ImmTySdwaSrc0Sel: OS << "SdwaSrc0Sel"; break;
670     case ImmTySdwaSrc1Sel: OS << "SdwaSrc1Sel"; break;
671     case ImmTySdwaDstUnused: OS << "SdwaDstUnused"; break;
672     case ImmTyDMask: OS << "DMask"; break;
673     case ImmTyUNorm: OS << "UNorm"; break;
674     case ImmTyDA: OS << "DA"; break;
675     case ImmTyR128: OS << "R128"; break;
676     case ImmTyLWE: OS << "LWE"; break;
677     case ImmTyOff: OS << "Off"; break;
678     case ImmTyExpTgt: OS << "ExpTgt"; break;
679     case ImmTyExpCompr: OS << "ExpCompr"; break;
680     case ImmTyExpVM: OS << "ExpVM"; break;
681     case ImmTyHwreg: OS << "Hwreg"; break;
682     case ImmTySendMsg: OS << "SendMsg"; break;
683     case ImmTyInterpSlot: OS << "InterpSlot"; break;
684     case ImmTyInterpAttr: OS << "InterpAttr"; break;
685     case ImmTyAttrChan: OS << "AttrChan"; break;
686     case ImmTyOpSel: OS << "OpSel"; break;
687     case ImmTyOpSelHi: OS << "OpSelHi"; break;
688     case ImmTyNegLo: OS << "NegLo"; break;
689     case ImmTyNegHi: OS << "NegHi"; break;
690     case ImmTySwizzle: OS << "Swizzle"; break;
691     case ImmTyHigh: OS << "High"; break;
692     }
693   }
694 
695   void print(raw_ostream &OS) const override {
696     switch (Kind) {
697     case Register:
698       OS << "<register " << getReg() << " mods: " << Reg.Mods << '>';
699       break;
700     case Immediate:
701       OS << '<' << getImm();
702       if (getImmTy() != ImmTyNone) {
703         OS << " type: "; printImmTy(OS, getImmTy());
704       }
705       OS << " mods: " << Imm.Mods << '>';
706       break;
707     case Token:
708       OS << '\'' << getToken() << '\'';
709       break;
710     case Expression:
711       OS << "<expr " << *Expr << '>';
712       break;
713     }
714   }
715 
716   static AMDGPUOperand::Ptr CreateImm(const AMDGPUAsmParser *AsmParser,
717                                       int64_t Val, SMLoc Loc,
718                                       ImmTy Type = ImmTyNone,
719                                       bool IsFPImm = false) {
720     auto Op = llvm::make_unique<AMDGPUOperand>(Immediate, AsmParser);
721     Op->Imm.Val = Val;
722     Op->Imm.IsFPImm = IsFPImm;
723     Op->Imm.Type = Type;
724     Op->Imm.Mods = Modifiers();
725     Op->StartLoc = Loc;
726     Op->EndLoc = Loc;
727     return Op;
728   }
729 
730   static AMDGPUOperand::Ptr CreateToken(const AMDGPUAsmParser *AsmParser,
731                                         StringRef Str, SMLoc Loc,
732                                         bool HasExplicitEncodingSize = true) {
733     auto Res = llvm::make_unique<AMDGPUOperand>(Token, AsmParser);
734     Res->Tok.Data = Str.data();
735     Res->Tok.Length = Str.size();
736     Res->StartLoc = Loc;
737     Res->EndLoc = Loc;
738     return Res;
739   }
740 
741   static AMDGPUOperand::Ptr CreateReg(const AMDGPUAsmParser *AsmParser,
742                                       unsigned RegNo, SMLoc S,
743                                       SMLoc E,
744                                       bool ForceVOP3) {
745     auto Op = llvm::make_unique<AMDGPUOperand>(Register, AsmParser);
746     Op->Reg.RegNo = RegNo;
747     Op->Reg.Mods = Modifiers();
748     Op->Reg.IsForcedVOP3 = ForceVOP3;
749     Op->StartLoc = S;
750     Op->EndLoc = E;
751     return Op;
752   }
753 
754   static AMDGPUOperand::Ptr CreateExpr(const AMDGPUAsmParser *AsmParser,
755                                        const class MCExpr *Expr, SMLoc S) {
756     auto Op = llvm::make_unique<AMDGPUOperand>(Expression, AsmParser);
757     Op->Expr = Expr;
758     Op->StartLoc = S;
759     Op->EndLoc = S;
760     return Op;
761   }
762 };
763 
764 raw_ostream &operator <<(raw_ostream &OS, AMDGPUOperand::Modifiers Mods) {
765   OS << "abs:" << Mods.Abs << " neg: " << Mods.Neg << " sext:" << Mods.Sext;
766   return OS;
767 }
768 
769 //===----------------------------------------------------------------------===//
770 // AsmParser
771 //===----------------------------------------------------------------------===//
772 
773 // Holds info related to the current kernel, e.g. count of SGPRs used.
774 // Kernel scope begins at .amdgpu_hsa_kernel directive, ends at next
775 // .amdgpu_hsa_kernel or at EOF.
776 class KernelScopeInfo {
777   int SgprIndexUnusedMin = -1;
778   int VgprIndexUnusedMin = -1;
779   MCContext *Ctx = nullptr;
780 
781   void usesSgprAt(int i) {
782     if (i >= SgprIndexUnusedMin) {
783       SgprIndexUnusedMin = ++i;
784       if (Ctx) {
785         MCSymbol * const Sym = Ctx->getOrCreateSymbol(Twine(".kernel.sgpr_count"));
786         Sym->setVariableValue(MCConstantExpr::create(SgprIndexUnusedMin, *Ctx));
787       }
788     }
789   }
790 
791   void usesVgprAt(int i) {
792     if (i >= VgprIndexUnusedMin) {
793       VgprIndexUnusedMin = ++i;
794       if (Ctx) {
795         MCSymbol * const Sym = Ctx->getOrCreateSymbol(Twine(".kernel.vgpr_count"));
796         Sym->setVariableValue(MCConstantExpr::create(VgprIndexUnusedMin, *Ctx));
797       }
798     }
799   }
800 
801 public:
802   KernelScopeInfo() = default;
803 
804   void initialize(MCContext &Context) {
805     Ctx = &Context;
806     usesSgprAt(SgprIndexUnusedMin = -1);
807     usesVgprAt(VgprIndexUnusedMin = -1);
808   }
809 
810   void usesRegister(RegisterKind RegKind, unsigned DwordRegIndex, unsigned RegWidth) {
811     switch (RegKind) {
812       case IS_SGPR: usesSgprAt(DwordRegIndex + RegWidth - 1); break;
813       case IS_VGPR: usesVgprAt(DwordRegIndex + RegWidth - 1); break;
814       default: break;
815     }
816   }
817 };
818 
819 class AMDGPUAsmParser : public MCTargetAsmParser {
820   MCAsmParser &Parser;
821 
822   // Number of extra operands parsed after the first optional operand.
823   // This may be necessary to skip hardcoded mandatory operands.
824   static const unsigned MAX_OPR_LOOKAHEAD = 1;
825 
826   unsigned ForcedEncodingSize = 0;
827   bool ForcedDPP = false;
828   bool ForcedSDWA = false;
829   KernelScopeInfo KernelScope;
830 
831   /// @name Auto-generated Match Functions
832   /// {
833 
834 #define GET_ASSEMBLER_HEADER
835 #include "AMDGPUGenAsmMatcher.inc"
836 
837   /// }
838 
839 private:
840   bool ParseAsAbsoluteExpression(uint32_t &Ret);
841   bool ParseDirectiveMajorMinor(uint32_t &Major, uint32_t &Minor);
842   bool ParseDirectiveHSACodeObjectVersion();
843   bool ParseDirectiveHSACodeObjectISA();
844   bool ParseAMDKernelCodeTValue(StringRef ID, amd_kernel_code_t &Header);
845   bool ParseDirectiveAMDKernelCodeT();
846   bool subtargetHasRegister(const MCRegisterInfo &MRI, unsigned RegNo) const;
847   bool ParseDirectiveAMDGPUHsaKernel();
848 
849   bool ParseDirectiveISAVersion();
850   bool ParseDirectiveHSAMetadata();
851   bool ParseDirectivePALMetadata();
852 
853   bool AddNextRegisterToList(unsigned& Reg, unsigned& RegWidth,
854                              RegisterKind RegKind, unsigned Reg1,
855                              unsigned RegNum);
856   bool ParseAMDGPURegister(RegisterKind& RegKind, unsigned& Reg,
857                            unsigned& RegNum, unsigned& RegWidth,
858                            unsigned *DwordRegIndex);
859   void cvtMubufImpl(MCInst &Inst, const OperandVector &Operands,
860                     bool IsAtomic, bool IsAtomicReturn);
861   void cvtDSImpl(MCInst &Inst, const OperandVector &Operands,
862                  bool IsGdsHardcoded);
863 
864 public:
865   enum AMDGPUMatchResultTy {
866     Match_PreferE32 = FIRST_TARGET_MATCH_RESULT_TY
867   };
868 
869   using OptionalImmIndexMap = std::map<AMDGPUOperand::ImmTy, unsigned>;
870 
871   AMDGPUAsmParser(const MCSubtargetInfo &STI, MCAsmParser &_Parser,
872                const MCInstrInfo &MII,
873                const MCTargetOptions &Options)
874       : MCTargetAsmParser(Options, STI, MII), Parser(_Parser) {
875     MCAsmParserExtension::Initialize(Parser);
876 
877     if (getFeatureBits().none()) {
878       // Set default features.
879       copySTI().ToggleFeature("SOUTHERN_ISLANDS");
880     }
881 
882     setAvailableFeatures(ComputeAvailableFeatures(getFeatureBits()));
883 
884     {
885       // TODO: make those pre-defined variables read-only.
886       // Currently there is none suitable machinery in the core llvm-mc for this.
887       // MCSymbol::isRedefinable is intended for another purpose, and
888       // AsmParser::parseDirectiveSet() cannot be specialized for specific target.
889       AMDGPU::IsaInfo::IsaVersion ISA =
890           AMDGPU::IsaInfo::getIsaVersion(getFeatureBits());
891       MCContext &Ctx = getContext();
892       MCSymbol *Sym =
893           Ctx.getOrCreateSymbol(Twine(".option.machine_version_major"));
894       Sym->setVariableValue(MCConstantExpr::create(ISA.Major, Ctx));
895       Sym = Ctx.getOrCreateSymbol(Twine(".option.machine_version_minor"));
896       Sym->setVariableValue(MCConstantExpr::create(ISA.Minor, Ctx));
897       Sym = Ctx.getOrCreateSymbol(Twine(".option.machine_version_stepping"));
898       Sym->setVariableValue(MCConstantExpr::create(ISA.Stepping, Ctx));
899     }
900     KernelScope.initialize(getContext());
901   }
902 
903   bool hasXNACK() const {
904     return AMDGPU::hasXNACK(getSTI());
905   }
906 
907   bool isSI() const {
908     return AMDGPU::isSI(getSTI());
909   }
910 
911   bool isCI() const {
912     return AMDGPU::isCI(getSTI());
913   }
914 
915   bool isVI() const {
916     return AMDGPU::isVI(getSTI());
917   }
918 
919   bool isGFX9() const {
920     return AMDGPU::isGFX9(getSTI());
921   }
922 
923   bool hasInv2PiInlineImm() const {
924     return getFeatureBits()[AMDGPU::FeatureInv2PiInlineImm];
925   }
926 
927   bool hasFlatOffsets() const {
928     return getFeatureBits()[AMDGPU::FeatureFlatInstOffsets];
929   }
930 
931   bool hasSGPR102_SGPR103() const {
932     return !isVI();
933   }
934 
935   bool hasIntClamp() const {
936     return getFeatureBits()[AMDGPU::FeatureIntClamp];
937   }
938 
939   AMDGPUTargetStreamer &getTargetStreamer() {
940     MCTargetStreamer &TS = *getParser().getStreamer().getTargetStreamer();
941     return static_cast<AMDGPUTargetStreamer &>(TS);
942   }
943 
944   const MCRegisterInfo *getMRI() const {
945     // We need this const_cast because for some reason getContext() is not const
946     // in MCAsmParser.
947     return const_cast<AMDGPUAsmParser*>(this)->getContext().getRegisterInfo();
948   }
949 
950   const MCInstrInfo *getMII() const {
951     return &MII;
952   }
953 
954   const FeatureBitset &getFeatureBits() const {
955     return getSTI().getFeatureBits();
956   }
957 
958   void setForcedEncodingSize(unsigned Size) { ForcedEncodingSize = Size; }
959   void setForcedDPP(bool ForceDPP_) { ForcedDPP = ForceDPP_; }
960   void setForcedSDWA(bool ForceSDWA_) { ForcedSDWA = ForceSDWA_; }
961 
962   unsigned getForcedEncodingSize() const { return ForcedEncodingSize; }
963   bool isForcedVOP3() const { return ForcedEncodingSize == 64; }
964   bool isForcedDPP() const { return ForcedDPP; }
965   bool isForcedSDWA() const { return ForcedSDWA; }
966   ArrayRef<unsigned> getMatchedVariants() const;
967 
968   std::unique_ptr<AMDGPUOperand> parseRegister();
969   bool ParseRegister(unsigned &RegNo, SMLoc &StartLoc, SMLoc &EndLoc) override;
970   unsigned checkTargetMatchPredicate(MCInst &Inst) override;
971   unsigned validateTargetOperandClass(MCParsedAsmOperand &Op,
972                                       unsigned Kind) override;
973   bool MatchAndEmitInstruction(SMLoc IDLoc, unsigned &Opcode,
974                                OperandVector &Operands, MCStreamer &Out,
975                                uint64_t &ErrorInfo,
976                                bool MatchingInlineAsm) override;
977   bool ParseDirective(AsmToken DirectiveID) override;
978   OperandMatchResultTy parseOperand(OperandVector &Operands, StringRef Mnemonic);
979   StringRef parseMnemonicSuffix(StringRef Name);
980   bool ParseInstruction(ParseInstructionInfo &Info, StringRef Name,
981                         SMLoc NameLoc, OperandVector &Operands) override;
982   //bool ProcessInstruction(MCInst &Inst);
983 
984   OperandMatchResultTy parseIntWithPrefix(const char *Prefix, int64_t &Int);
985 
986   OperandMatchResultTy
987   parseIntWithPrefix(const char *Prefix, OperandVector &Operands,
988                      AMDGPUOperand::ImmTy ImmTy = AMDGPUOperand::ImmTyNone,
989                      bool (*ConvertResult)(int64_t &) = nullptr);
990 
991   OperandMatchResultTy parseOperandArrayWithPrefix(
992     const char *Prefix,
993     OperandVector &Operands,
994     AMDGPUOperand::ImmTy ImmTy = AMDGPUOperand::ImmTyNone,
995     bool (*ConvertResult)(int64_t&) = nullptr);
996 
997   OperandMatchResultTy
998   parseNamedBit(const char *Name, OperandVector &Operands,
999                 AMDGPUOperand::ImmTy ImmTy = AMDGPUOperand::ImmTyNone);
1000   OperandMatchResultTy parseStringWithPrefix(StringRef Prefix,
1001                                              StringRef &Value);
1002 
1003   bool parseAbsoluteExpr(int64_t &Val, bool AbsMod = false);
1004   OperandMatchResultTy parseImm(OperandVector &Operands, bool AbsMod = false);
1005   OperandMatchResultTy parseReg(OperandVector &Operands);
1006   OperandMatchResultTy parseRegOrImm(OperandVector &Operands, bool AbsMod = false);
1007   OperandMatchResultTy parseRegOrImmWithFPInputMods(OperandVector &Operands, bool AllowImm = true);
1008   OperandMatchResultTy parseRegOrImmWithIntInputMods(OperandVector &Operands, bool AllowImm = true);
1009   OperandMatchResultTy parseRegWithFPInputMods(OperandVector &Operands);
1010   OperandMatchResultTy parseRegWithIntInputMods(OperandVector &Operands);
1011   OperandMatchResultTy parseVReg32OrOff(OperandVector &Operands);
1012 
1013   void cvtDSOffset01(MCInst &Inst, const OperandVector &Operands);
1014   void cvtDS(MCInst &Inst, const OperandVector &Operands) { cvtDSImpl(Inst, Operands, false); }
1015   void cvtDSGds(MCInst &Inst, const OperandVector &Operands) { cvtDSImpl(Inst, Operands, true); }
1016   void cvtExp(MCInst &Inst, const OperandVector &Operands);
1017 
1018   bool parseCnt(int64_t &IntVal);
1019   OperandMatchResultTy parseSWaitCntOps(OperandVector &Operands);
1020   OperandMatchResultTy parseHwreg(OperandVector &Operands);
1021 
1022 private:
1023   struct OperandInfoTy {
1024     int64_t Id;
1025     bool IsSymbolic = false;
1026 
1027     OperandInfoTy(int64_t Id_) : Id(Id_) {}
1028   };
1029 
1030   bool parseSendMsgConstruct(OperandInfoTy &Msg, OperandInfoTy &Operation, int64_t &StreamId);
1031   bool parseHwregConstruct(OperandInfoTy &HwReg, int64_t &Offset, int64_t &Width);
1032 
1033   void errorExpTgt();
1034   OperandMatchResultTy parseExpTgtImpl(StringRef Str, uint8_t &Val);
1035 
1036   bool validateInstruction(const MCInst &Inst, const SMLoc &IDLoc);
1037   bool validateConstantBusLimitations(const MCInst &Inst);
1038   bool validateEarlyClobberLimitations(const MCInst &Inst);
1039   bool validateIntClampSupported(const MCInst &Inst);
1040   bool validateMIMGAtomicDMask(const MCInst &Inst);
1041   bool validateMIMGDataSize(const MCInst &Inst);
1042   bool usesConstantBus(const MCInst &Inst, unsigned OpIdx);
1043   bool isInlineConstant(const MCInst &Inst, unsigned OpIdx) const;
1044   unsigned findImplicitSGPRReadInVOP(const MCInst &Inst) const;
1045 
1046   bool trySkipId(const StringRef Id);
1047   bool trySkipToken(const AsmToken::TokenKind Kind);
1048   bool skipToken(const AsmToken::TokenKind Kind, const StringRef ErrMsg);
1049   bool parseString(StringRef &Val, const StringRef ErrMsg = "expected a string");
1050   bool parseExpr(int64_t &Imm);
1051 
1052 public:
1053   OperandMatchResultTy parseOptionalOperand(OperandVector &Operands);
1054   OperandMatchResultTy parseOptionalOpr(OperandVector &Operands);
1055 
1056   OperandMatchResultTy parseExpTgt(OperandVector &Operands);
1057   OperandMatchResultTy parseSendMsgOp(OperandVector &Operands);
1058   OperandMatchResultTy parseInterpSlot(OperandVector &Operands);
1059   OperandMatchResultTy parseInterpAttr(OperandVector &Operands);
1060   OperandMatchResultTy parseSOppBrTarget(OperandVector &Operands);
1061 
1062   bool parseSwizzleOperands(const unsigned OpNum, int64_t* Op,
1063                             const unsigned MinVal,
1064                             const unsigned MaxVal,
1065                             const StringRef ErrMsg);
1066   OperandMatchResultTy parseSwizzleOp(OperandVector &Operands);
1067   bool parseSwizzleOffset(int64_t &Imm);
1068   bool parseSwizzleMacro(int64_t &Imm);
1069   bool parseSwizzleQuadPerm(int64_t &Imm);
1070   bool parseSwizzleBitmaskPerm(int64_t &Imm);
1071   bool parseSwizzleBroadcast(int64_t &Imm);
1072   bool parseSwizzleSwap(int64_t &Imm);
1073   bool parseSwizzleReverse(int64_t &Imm);
1074 
1075   void cvtMubuf(MCInst &Inst, const OperandVector &Operands) { cvtMubufImpl(Inst, Operands, false, false); }
1076   void cvtMubufAtomic(MCInst &Inst, const OperandVector &Operands) { cvtMubufImpl(Inst, Operands, true, false); }
1077   void cvtMubufAtomicReturn(MCInst &Inst, const OperandVector &Operands) { cvtMubufImpl(Inst, Operands, true, true); }
1078   void cvtMtbuf(MCInst &Inst, const OperandVector &Operands);
1079 
1080   AMDGPUOperand::Ptr defaultGLC() const;
1081   AMDGPUOperand::Ptr defaultSLC() const;
1082   AMDGPUOperand::Ptr defaultTFE() const;
1083 
1084   AMDGPUOperand::Ptr defaultDMask() const;
1085   AMDGPUOperand::Ptr defaultUNorm() const;
1086   AMDGPUOperand::Ptr defaultDA() const;
1087   AMDGPUOperand::Ptr defaultR128() const;
1088   AMDGPUOperand::Ptr defaultLWE() const;
1089   AMDGPUOperand::Ptr defaultSMRDOffset8() const;
1090   AMDGPUOperand::Ptr defaultSMRDOffset20() const;
1091   AMDGPUOperand::Ptr defaultSMRDLiteralOffset() const;
1092   AMDGPUOperand::Ptr defaultOffsetU12() const;
1093   AMDGPUOperand::Ptr defaultOffsetS13() const;
1094 
1095   OperandMatchResultTy parseOModOperand(OperandVector &Operands);
1096 
1097   void cvtVOP3(MCInst &Inst, const OperandVector &Operands,
1098                OptionalImmIndexMap &OptionalIdx);
1099   void cvtVOP3OpSel(MCInst &Inst, const OperandVector &Operands);
1100   void cvtVOP3(MCInst &Inst, const OperandVector &Operands);
1101   void cvtVOP3P(MCInst &Inst, const OperandVector &Operands);
1102 
1103   void cvtVOP3Interp(MCInst &Inst, const OperandVector &Operands);
1104 
1105   void cvtMIMG(MCInst &Inst, const OperandVector &Operands,
1106                bool IsAtomic = false);
1107   void cvtMIMGAtomic(MCInst &Inst, const OperandVector &Operands);
1108 
1109   OperandMatchResultTy parseDPPCtrl(OperandVector &Operands);
1110   AMDGPUOperand::Ptr defaultRowMask() const;
1111   AMDGPUOperand::Ptr defaultBankMask() const;
1112   AMDGPUOperand::Ptr defaultBoundCtrl() const;
1113   void cvtDPP(MCInst &Inst, const OperandVector &Operands);
1114 
1115   OperandMatchResultTy parseSDWASel(OperandVector &Operands, StringRef Prefix,
1116                                     AMDGPUOperand::ImmTy Type);
1117   OperandMatchResultTy parseSDWADstUnused(OperandVector &Operands);
1118   void cvtSdwaVOP1(MCInst &Inst, const OperandVector &Operands);
1119   void cvtSdwaVOP2(MCInst &Inst, const OperandVector &Operands);
1120   void cvtSdwaVOP2b(MCInst &Inst, const OperandVector &Operands);
1121   void cvtSdwaVOPC(MCInst &Inst, const OperandVector &Operands);
1122   void cvtSDWA(MCInst &Inst, const OperandVector &Operands,
1123                 uint64_t BasicInstType, bool skipVcc = false);
1124 };
1125 
1126 struct OptionalOperand {
1127   const char *Name;
1128   AMDGPUOperand::ImmTy Type;
1129   bool IsBit;
1130   bool (*ConvertResult)(int64_t&);
1131 };
1132 
1133 } // end anonymous namespace
1134 
1135 // May be called with integer type with equivalent bitwidth.
1136 static const fltSemantics *getFltSemantics(unsigned Size) {
1137   switch (Size) {
1138   case 4:
1139     return &APFloat::IEEEsingle();
1140   case 8:
1141     return &APFloat::IEEEdouble();
1142   case 2:
1143     return &APFloat::IEEEhalf();
1144   default:
1145     llvm_unreachable("unsupported fp type");
1146   }
1147 }
1148 
1149 static const fltSemantics *getFltSemantics(MVT VT) {
1150   return getFltSemantics(VT.getSizeInBits() / 8);
1151 }
1152 
1153 static const fltSemantics *getOpFltSemantics(uint8_t OperandType) {
1154   switch (OperandType) {
1155   case AMDGPU::OPERAND_REG_IMM_INT32:
1156   case AMDGPU::OPERAND_REG_IMM_FP32:
1157   case AMDGPU::OPERAND_REG_INLINE_C_INT32:
1158   case AMDGPU::OPERAND_REG_INLINE_C_FP32:
1159     return &APFloat::IEEEsingle();
1160   case AMDGPU::OPERAND_REG_IMM_INT64:
1161   case AMDGPU::OPERAND_REG_IMM_FP64:
1162   case AMDGPU::OPERAND_REG_INLINE_C_INT64:
1163   case AMDGPU::OPERAND_REG_INLINE_C_FP64:
1164     return &APFloat::IEEEdouble();
1165   case AMDGPU::OPERAND_REG_IMM_INT16:
1166   case AMDGPU::OPERAND_REG_IMM_FP16:
1167   case AMDGPU::OPERAND_REG_INLINE_C_INT16:
1168   case AMDGPU::OPERAND_REG_INLINE_C_FP16:
1169   case AMDGPU::OPERAND_REG_INLINE_C_V2INT16:
1170   case AMDGPU::OPERAND_REG_INLINE_C_V2FP16:
1171     return &APFloat::IEEEhalf();
1172   default:
1173     llvm_unreachable("unsupported fp type");
1174   }
1175 }
1176 
1177 //===----------------------------------------------------------------------===//
1178 // Operand
1179 //===----------------------------------------------------------------------===//
1180 
1181 static bool canLosslesslyConvertToFPType(APFloat &FPLiteral, MVT VT) {
1182   bool Lost;
1183 
1184   // Convert literal to single precision
1185   APFloat::opStatus Status = FPLiteral.convert(*getFltSemantics(VT),
1186                                                APFloat::rmNearestTiesToEven,
1187                                                &Lost);
1188   // We allow precision lost but not overflow or underflow
1189   if (Status != APFloat::opOK &&
1190       Lost &&
1191       ((Status & APFloat::opOverflow)  != 0 ||
1192        (Status & APFloat::opUnderflow) != 0)) {
1193     return false;
1194   }
1195 
1196   return true;
1197 }
1198 
1199 bool AMDGPUOperand::isInlinableImm(MVT type) const {
1200   if (!isImmTy(ImmTyNone)) {
1201     // Only plain immediates are inlinable (e.g. "clamp" attribute is not)
1202     return false;
1203   }
1204   // TODO: We should avoid using host float here. It would be better to
1205   // check the float bit values which is what a few other places do.
1206   // We've had bot failures before due to weird NaN support on mips hosts.
1207 
1208   APInt Literal(64, Imm.Val);
1209 
1210   if (Imm.IsFPImm) { // We got fp literal token
1211     if (type == MVT::f64 || type == MVT::i64) { // Expected 64-bit operand
1212       return AMDGPU::isInlinableLiteral64(Imm.Val,
1213                                           AsmParser->hasInv2PiInlineImm());
1214     }
1215 
1216     APFloat FPLiteral(APFloat::IEEEdouble(), APInt(64, Imm.Val));
1217     if (!canLosslesslyConvertToFPType(FPLiteral, type))
1218       return false;
1219 
1220     if (type.getScalarSizeInBits() == 16) {
1221       return AMDGPU::isInlinableLiteral16(
1222         static_cast<int16_t>(FPLiteral.bitcastToAPInt().getZExtValue()),
1223         AsmParser->hasInv2PiInlineImm());
1224     }
1225 
1226     // Check if single precision literal is inlinable
1227     return AMDGPU::isInlinableLiteral32(
1228       static_cast<int32_t>(FPLiteral.bitcastToAPInt().getZExtValue()),
1229       AsmParser->hasInv2PiInlineImm());
1230   }
1231 
1232   // We got int literal token.
1233   if (type == MVT::f64 || type == MVT::i64) { // Expected 64-bit operand
1234     return AMDGPU::isInlinableLiteral64(Imm.Val,
1235                                         AsmParser->hasInv2PiInlineImm());
1236   }
1237 
1238   if (type.getScalarSizeInBits() == 16) {
1239     return AMDGPU::isInlinableLiteral16(
1240       static_cast<int16_t>(Literal.getLoBits(16).getSExtValue()),
1241       AsmParser->hasInv2PiInlineImm());
1242   }
1243 
1244   return AMDGPU::isInlinableLiteral32(
1245     static_cast<int32_t>(Literal.getLoBits(32).getZExtValue()),
1246     AsmParser->hasInv2PiInlineImm());
1247 }
1248 
1249 bool AMDGPUOperand::isLiteralImm(MVT type) const {
1250   // Check that this immediate can be added as literal
1251   if (!isImmTy(ImmTyNone)) {
1252     return false;
1253   }
1254 
1255   if (!Imm.IsFPImm) {
1256     // We got int literal token.
1257 
1258     if (type == MVT::f64 && hasFPModifiers()) {
1259       // Cannot apply fp modifiers to int literals preserving the same semantics
1260       // for VOP1/2/C and VOP3 because of integer truncation. To avoid ambiguity,
1261       // disable these cases.
1262       return false;
1263     }
1264 
1265     unsigned Size = type.getSizeInBits();
1266     if (Size == 64)
1267       Size = 32;
1268 
1269     // FIXME: 64-bit operands can zero extend, sign extend, or pad zeroes for FP
1270     // types.
1271     return isUIntN(Size, Imm.Val) || isIntN(Size, Imm.Val);
1272   }
1273 
1274   // We got fp literal token
1275   if (type == MVT::f64) { // Expected 64-bit fp operand
1276     // We would set low 64-bits of literal to zeroes but we accept this literals
1277     return true;
1278   }
1279 
1280   if (type == MVT::i64) { // Expected 64-bit int operand
1281     // We don't allow fp literals in 64-bit integer instructions. It is
1282     // unclear how we should encode them.
1283     return false;
1284   }
1285 
1286   APFloat FPLiteral(APFloat::IEEEdouble(), APInt(64, Imm.Val));
1287   return canLosslesslyConvertToFPType(FPLiteral, type);
1288 }
1289 
1290 bool AMDGPUOperand::isRegClass(unsigned RCID) const {
1291   return isRegKind() && AsmParser->getMRI()->getRegClass(RCID).contains(getReg());
1292 }
1293 
1294 bool AMDGPUOperand::isSDWAOperand(MVT type) const {
1295   if (AsmParser->isVI())
1296     return isVReg();
1297   else if (AsmParser->isGFX9())
1298     return isRegKind() || isInlinableImm(type);
1299   else
1300     return false;
1301 }
1302 
1303 bool AMDGPUOperand::isSDWAFP16Operand() const {
1304   return isSDWAOperand(MVT::f16);
1305 }
1306 
1307 bool AMDGPUOperand::isSDWAFP32Operand() const {
1308   return isSDWAOperand(MVT::f32);
1309 }
1310 
1311 bool AMDGPUOperand::isSDWAInt16Operand() const {
1312   return isSDWAOperand(MVT::i16);
1313 }
1314 
1315 bool AMDGPUOperand::isSDWAInt32Operand() const {
1316   return isSDWAOperand(MVT::i32);
1317 }
1318 
1319 uint64_t AMDGPUOperand::applyInputFPModifiers(uint64_t Val, unsigned Size) const
1320 {
1321   assert(isImmTy(ImmTyNone) && Imm.Mods.hasFPModifiers());
1322   assert(Size == 2 || Size == 4 || Size == 8);
1323 
1324   const uint64_t FpSignMask = (1ULL << (Size * 8 - 1));
1325 
1326   if (Imm.Mods.Abs) {
1327     Val &= ~FpSignMask;
1328   }
1329   if (Imm.Mods.Neg) {
1330     Val ^= FpSignMask;
1331   }
1332 
1333   return Val;
1334 }
1335 
1336 void AMDGPUOperand::addImmOperands(MCInst &Inst, unsigned N, bool ApplyModifiers) const {
1337   if (AMDGPU::isSISrcOperand(AsmParser->getMII()->get(Inst.getOpcode()),
1338                              Inst.getNumOperands())) {
1339     addLiteralImmOperand(Inst, Imm.Val,
1340                          ApplyModifiers &
1341                          isImmTy(ImmTyNone) && Imm.Mods.hasFPModifiers());
1342   } else {
1343     assert(!isImmTy(ImmTyNone) || !hasModifiers());
1344     Inst.addOperand(MCOperand::createImm(Imm.Val));
1345   }
1346 }
1347 
1348 void AMDGPUOperand::addLiteralImmOperand(MCInst &Inst, int64_t Val, bool ApplyModifiers) const {
1349   const auto& InstDesc = AsmParser->getMII()->get(Inst.getOpcode());
1350   auto OpNum = Inst.getNumOperands();
1351   // Check that this operand accepts literals
1352   assert(AMDGPU::isSISrcOperand(InstDesc, OpNum));
1353 
1354   if (ApplyModifiers) {
1355     assert(AMDGPU::isSISrcFPOperand(InstDesc, OpNum));
1356     const unsigned Size = Imm.IsFPImm ? sizeof(double) : getOperandSize(InstDesc, OpNum);
1357     Val = applyInputFPModifiers(Val, Size);
1358   }
1359 
1360   APInt Literal(64, Val);
1361   uint8_t OpTy = InstDesc.OpInfo[OpNum].OperandType;
1362 
1363   if (Imm.IsFPImm) { // We got fp literal token
1364     switch (OpTy) {
1365     case AMDGPU::OPERAND_REG_IMM_INT64:
1366     case AMDGPU::OPERAND_REG_IMM_FP64:
1367     case AMDGPU::OPERAND_REG_INLINE_C_INT64:
1368     case AMDGPU::OPERAND_REG_INLINE_C_FP64:
1369       if (AMDGPU::isInlinableLiteral64(Literal.getZExtValue(),
1370                                        AsmParser->hasInv2PiInlineImm())) {
1371         Inst.addOperand(MCOperand::createImm(Literal.getZExtValue()));
1372         return;
1373       }
1374 
1375       // Non-inlineable
1376       if (AMDGPU::isSISrcFPOperand(InstDesc, OpNum)) { // Expected 64-bit fp operand
1377         // For fp operands we check if low 32 bits are zeros
1378         if (Literal.getLoBits(32) != 0) {
1379           const_cast<AMDGPUAsmParser *>(AsmParser)->Warning(Inst.getLoc(),
1380           "Can't encode literal as exact 64-bit floating-point operand. "
1381           "Low 32-bits will be set to zero");
1382         }
1383 
1384         Inst.addOperand(MCOperand::createImm(Literal.lshr(32).getZExtValue()));
1385         return;
1386       }
1387 
1388       // We don't allow fp literals in 64-bit integer instructions. It is
1389       // unclear how we should encode them. This case should be checked earlier
1390       // in predicate methods (isLiteralImm())
1391       llvm_unreachable("fp literal in 64-bit integer instruction.");
1392 
1393     case AMDGPU::OPERAND_REG_IMM_INT32:
1394     case AMDGPU::OPERAND_REG_IMM_FP32:
1395     case AMDGPU::OPERAND_REG_INLINE_C_INT32:
1396     case AMDGPU::OPERAND_REG_INLINE_C_FP32:
1397     case AMDGPU::OPERAND_REG_IMM_INT16:
1398     case AMDGPU::OPERAND_REG_IMM_FP16:
1399     case AMDGPU::OPERAND_REG_INLINE_C_INT16:
1400     case AMDGPU::OPERAND_REG_INLINE_C_FP16:
1401     case AMDGPU::OPERAND_REG_INLINE_C_V2INT16:
1402     case AMDGPU::OPERAND_REG_INLINE_C_V2FP16: {
1403       bool lost;
1404       APFloat FPLiteral(APFloat::IEEEdouble(), Literal);
1405       // Convert literal to single precision
1406       FPLiteral.convert(*getOpFltSemantics(OpTy),
1407                         APFloat::rmNearestTiesToEven, &lost);
1408       // We allow precision lost but not overflow or underflow. This should be
1409       // checked earlier in isLiteralImm()
1410 
1411       uint64_t ImmVal = FPLiteral.bitcastToAPInt().getZExtValue();
1412       if (OpTy == AMDGPU::OPERAND_REG_INLINE_C_V2INT16 ||
1413           OpTy == AMDGPU::OPERAND_REG_INLINE_C_V2FP16) {
1414         ImmVal |= (ImmVal << 16);
1415       }
1416 
1417       Inst.addOperand(MCOperand::createImm(ImmVal));
1418       return;
1419     }
1420     default:
1421       llvm_unreachable("invalid operand size");
1422     }
1423 
1424     return;
1425   }
1426 
1427    // We got int literal token.
1428   // Only sign extend inline immediates.
1429   // FIXME: No errors on truncation
1430   switch (OpTy) {
1431   case AMDGPU::OPERAND_REG_IMM_INT32:
1432   case AMDGPU::OPERAND_REG_IMM_FP32:
1433   case AMDGPU::OPERAND_REG_INLINE_C_INT32:
1434   case AMDGPU::OPERAND_REG_INLINE_C_FP32:
1435     if (isInt<32>(Val) &&
1436         AMDGPU::isInlinableLiteral32(static_cast<int32_t>(Val),
1437                                      AsmParser->hasInv2PiInlineImm())) {
1438       Inst.addOperand(MCOperand::createImm(Val));
1439       return;
1440     }
1441 
1442     Inst.addOperand(MCOperand::createImm(Val & 0xffffffff));
1443     return;
1444 
1445   case AMDGPU::OPERAND_REG_IMM_INT64:
1446   case AMDGPU::OPERAND_REG_IMM_FP64:
1447   case AMDGPU::OPERAND_REG_INLINE_C_INT64:
1448   case AMDGPU::OPERAND_REG_INLINE_C_FP64:
1449     if (AMDGPU::isInlinableLiteral64(Val, AsmParser->hasInv2PiInlineImm())) {
1450       Inst.addOperand(MCOperand::createImm(Val));
1451       return;
1452     }
1453 
1454     Inst.addOperand(MCOperand::createImm(Lo_32(Val)));
1455     return;
1456 
1457   case AMDGPU::OPERAND_REG_IMM_INT16:
1458   case AMDGPU::OPERAND_REG_IMM_FP16:
1459   case AMDGPU::OPERAND_REG_INLINE_C_INT16:
1460   case AMDGPU::OPERAND_REG_INLINE_C_FP16:
1461     if (isInt<16>(Val) &&
1462         AMDGPU::isInlinableLiteral16(static_cast<int16_t>(Val),
1463                                      AsmParser->hasInv2PiInlineImm())) {
1464       Inst.addOperand(MCOperand::createImm(Val));
1465       return;
1466     }
1467 
1468     Inst.addOperand(MCOperand::createImm(Val & 0xffff));
1469     return;
1470 
1471   case AMDGPU::OPERAND_REG_INLINE_C_V2INT16:
1472   case AMDGPU::OPERAND_REG_INLINE_C_V2FP16: {
1473     auto LiteralVal = static_cast<uint16_t>(Literal.getLoBits(16).getZExtValue());
1474     assert(AMDGPU::isInlinableLiteral16(LiteralVal,
1475                                         AsmParser->hasInv2PiInlineImm()));
1476 
1477     uint32_t ImmVal = static_cast<uint32_t>(LiteralVal) << 16 |
1478                       static_cast<uint32_t>(LiteralVal);
1479     Inst.addOperand(MCOperand::createImm(ImmVal));
1480     return;
1481   }
1482   default:
1483     llvm_unreachable("invalid operand size");
1484   }
1485 }
1486 
1487 template <unsigned Bitwidth>
1488 void AMDGPUOperand::addKImmFPOperands(MCInst &Inst, unsigned N) const {
1489   APInt Literal(64, Imm.Val);
1490 
1491   if (!Imm.IsFPImm) {
1492     // We got int literal token.
1493     Inst.addOperand(MCOperand::createImm(Literal.getLoBits(Bitwidth).getZExtValue()));
1494     return;
1495   }
1496 
1497   bool Lost;
1498   APFloat FPLiteral(APFloat::IEEEdouble(), Literal);
1499   FPLiteral.convert(*getFltSemantics(Bitwidth / 8),
1500                     APFloat::rmNearestTiesToEven, &Lost);
1501   Inst.addOperand(MCOperand::createImm(FPLiteral.bitcastToAPInt().getZExtValue()));
1502 }
1503 
1504 void AMDGPUOperand::addRegOperands(MCInst &Inst, unsigned N) const {
1505   Inst.addOperand(MCOperand::createReg(AMDGPU::getMCReg(getReg(), AsmParser->getSTI())));
1506 }
1507 
1508 //===----------------------------------------------------------------------===//
1509 // AsmParser
1510 //===----------------------------------------------------------------------===//
1511 
1512 static int getRegClass(RegisterKind Is, unsigned RegWidth) {
1513   if (Is == IS_VGPR) {
1514     switch (RegWidth) {
1515       default: return -1;
1516       case 1: return AMDGPU::VGPR_32RegClassID;
1517       case 2: return AMDGPU::VReg_64RegClassID;
1518       case 3: return AMDGPU::VReg_96RegClassID;
1519       case 4: return AMDGPU::VReg_128RegClassID;
1520       case 8: return AMDGPU::VReg_256RegClassID;
1521       case 16: return AMDGPU::VReg_512RegClassID;
1522     }
1523   } else if (Is == IS_TTMP) {
1524     switch (RegWidth) {
1525       default: return -1;
1526       case 1: return AMDGPU::TTMP_32RegClassID;
1527       case 2: return AMDGPU::TTMP_64RegClassID;
1528       case 4: return AMDGPU::TTMP_128RegClassID;
1529       case 8: return AMDGPU::TTMP_256RegClassID;
1530       case 16: return AMDGPU::TTMP_512RegClassID;
1531     }
1532   } else if (Is == IS_SGPR) {
1533     switch (RegWidth) {
1534       default: return -1;
1535       case 1: return AMDGPU::SGPR_32RegClassID;
1536       case 2: return AMDGPU::SGPR_64RegClassID;
1537       case 4: return AMDGPU::SGPR_128RegClassID;
1538       case 8: return AMDGPU::SGPR_256RegClassID;
1539       case 16: return AMDGPU::SGPR_512RegClassID;
1540     }
1541   }
1542   return -1;
1543 }
1544 
1545 static unsigned getSpecialRegForName(StringRef RegName) {
1546   return StringSwitch<unsigned>(RegName)
1547     .Case("exec", AMDGPU::EXEC)
1548     .Case("vcc", AMDGPU::VCC)
1549     .Case("flat_scratch", AMDGPU::FLAT_SCR)
1550     .Case("xnack_mask", AMDGPU::XNACK_MASK)
1551     .Case("m0", AMDGPU::M0)
1552     .Case("scc", AMDGPU::SCC)
1553     .Case("tba", AMDGPU::TBA)
1554     .Case("tma", AMDGPU::TMA)
1555     .Case("flat_scratch_lo", AMDGPU::FLAT_SCR_LO)
1556     .Case("flat_scratch_hi", AMDGPU::FLAT_SCR_HI)
1557     .Case("xnack_mask_lo", AMDGPU::XNACK_MASK_LO)
1558     .Case("xnack_mask_hi", AMDGPU::XNACK_MASK_HI)
1559     .Case("vcc_lo", AMDGPU::VCC_LO)
1560     .Case("vcc_hi", AMDGPU::VCC_HI)
1561     .Case("exec_lo", AMDGPU::EXEC_LO)
1562     .Case("exec_hi", AMDGPU::EXEC_HI)
1563     .Case("tma_lo", AMDGPU::TMA_LO)
1564     .Case("tma_hi", AMDGPU::TMA_HI)
1565     .Case("tba_lo", AMDGPU::TBA_LO)
1566     .Case("tba_hi", AMDGPU::TBA_HI)
1567     .Default(0);
1568 }
1569 
1570 bool AMDGPUAsmParser::ParseRegister(unsigned &RegNo, SMLoc &StartLoc,
1571                                     SMLoc &EndLoc) {
1572   auto R = parseRegister();
1573   if (!R) return true;
1574   assert(R->isReg());
1575   RegNo = R->getReg();
1576   StartLoc = R->getStartLoc();
1577   EndLoc = R->getEndLoc();
1578   return false;
1579 }
1580 
1581 bool AMDGPUAsmParser::AddNextRegisterToList(unsigned &Reg, unsigned &RegWidth,
1582                                             RegisterKind RegKind, unsigned Reg1,
1583                                             unsigned RegNum) {
1584   switch (RegKind) {
1585   case IS_SPECIAL:
1586     if (Reg == AMDGPU::EXEC_LO && Reg1 == AMDGPU::EXEC_HI) {
1587       Reg = AMDGPU::EXEC;
1588       RegWidth = 2;
1589       return true;
1590     }
1591     if (Reg == AMDGPU::FLAT_SCR_LO && Reg1 == AMDGPU::FLAT_SCR_HI) {
1592       Reg = AMDGPU::FLAT_SCR;
1593       RegWidth = 2;
1594       return true;
1595     }
1596     if (Reg == AMDGPU::XNACK_MASK_LO && Reg1 == AMDGPU::XNACK_MASK_HI) {
1597       Reg = AMDGPU::XNACK_MASK;
1598       RegWidth = 2;
1599       return true;
1600     }
1601     if (Reg == AMDGPU::VCC_LO && Reg1 == AMDGPU::VCC_HI) {
1602       Reg = AMDGPU::VCC;
1603       RegWidth = 2;
1604       return true;
1605     }
1606     if (Reg == AMDGPU::TBA_LO && Reg1 == AMDGPU::TBA_HI) {
1607       Reg = AMDGPU::TBA;
1608       RegWidth = 2;
1609       return true;
1610     }
1611     if (Reg == AMDGPU::TMA_LO && Reg1 == AMDGPU::TMA_HI) {
1612       Reg = AMDGPU::TMA;
1613       RegWidth = 2;
1614       return true;
1615     }
1616     return false;
1617   case IS_VGPR:
1618   case IS_SGPR:
1619   case IS_TTMP:
1620     if (Reg1 != Reg + RegWidth) {
1621       return false;
1622     }
1623     RegWidth++;
1624     return true;
1625   default:
1626     llvm_unreachable("unexpected register kind");
1627   }
1628 }
1629 
1630 bool AMDGPUAsmParser::ParseAMDGPURegister(RegisterKind &RegKind, unsigned &Reg,
1631                                           unsigned &RegNum, unsigned &RegWidth,
1632                                           unsigned *DwordRegIndex) {
1633   if (DwordRegIndex) { *DwordRegIndex = 0; }
1634   const MCRegisterInfo *TRI = getContext().getRegisterInfo();
1635   if (getLexer().is(AsmToken::Identifier)) {
1636     StringRef RegName = Parser.getTok().getString();
1637     if ((Reg = getSpecialRegForName(RegName))) {
1638       Parser.Lex();
1639       RegKind = IS_SPECIAL;
1640     } else {
1641       unsigned RegNumIndex = 0;
1642       if (RegName[0] == 'v') {
1643         RegNumIndex = 1;
1644         RegKind = IS_VGPR;
1645       } else if (RegName[0] == 's') {
1646         RegNumIndex = 1;
1647         RegKind = IS_SGPR;
1648       } else if (RegName.startswith("ttmp")) {
1649         RegNumIndex = strlen("ttmp");
1650         RegKind = IS_TTMP;
1651       } else {
1652         return false;
1653       }
1654       if (RegName.size() > RegNumIndex) {
1655         // Single 32-bit register: vXX.
1656         if (RegName.substr(RegNumIndex).getAsInteger(10, RegNum))
1657           return false;
1658         Parser.Lex();
1659         RegWidth = 1;
1660       } else {
1661         // Range of registers: v[XX:YY]. ":YY" is optional.
1662         Parser.Lex();
1663         int64_t RegLo, RegHi;
1664         if (getLexer().isNot(AsmToken::LBrac))
1665           return false;
1666         Parser.Lex();
1667 
1668         if (getParser().parseAbsoluteExpression(RegLo))
1669           return false;
1670 
1671         const bool isRBrace = getLexer().is(AsmToken::RBrac);
1672         if (!isRBrace && getLexer().isNot(AsmToken::Colon))
1673           return false;
1674         Parser.Lex();
1675 
1676         if (isRBrace) {
1677           RegHi = RegLo;
1678         } else {
1679           if (getParser().parseAbsoluteExpression(RegHi))
1680             return false;
1681 
1682           if (getLexer().isNot(AsmToken::RBrac))
1683             return false;
1684           Parser.Lex();
1685         }
1686         RegNum = (unsigned) RegLo;
1687         RegWidth = (RegHi - RegLo) + 1;
1688       }
1689     }
1690   } else if (getLexer().is(AsmToken::LBrac)) {
1691     // List of consecutive registers: [s0,s1,s2,s3]
1692     Parser.Lex();
1693     if (!ParseAMDGPURegister(RegKind, Reg, RegNum, RegWidth, nullptr))
1694       return false;
1695     if (RegWidth != 1)
1696       return false;
1697     RegisterKind RegKind1;
1698     unsigned Reg1, RegNum1, RegWidth1;
1699     do {
1700       if (getLexer().is(AsmToken::Comma)) {
1701         Parser.Lex();
1702       } else if (getLexer().is(AsmToken::RBrac)) {
1703         Parser.Lex();
1704         break;
1705       } else if (ParseAMDGPURegister(RegKind1, Reg1, RegNum1, RegWidth1, nullptr)) {
1706         if (RegWidth1 != 1) {
1707           return false;
1708         }
1709         if (RegKind1 != RegKind) {
1710           return false;
1711         }
1712         if (!AddNextRegisterToList(Reg, RegWidth, RegKind1, Reg1, RegNum1)) {
1713           return false;
1714         }
1715       } else {
1716         return false;
1717       }
1718     } while (true);
1719   } else {
1720     return false;
1721   }
1722   switch (RegKind) {
1723   case IS_SPECIAL:
1724     RegNum = 0;
1725     RegWidth = 1;
1726     break;
1727   case IS_VGPR:
1728   case IS_SGPR:
1729   case IS_TTMP:
1730   {
1731     unsigned Size = 1;
1732     if (RegKind == IS_SGPR || RegKind == IS_TTMP) {
1733       // SGPR and TTMP registers must be aligned. Max required alignment is 4 dwords.
1734       Size = std::min(RegWidth, 4u);
1735     }
1736     if (RegNum % Size != 0)
1737       return false;
1738     if (DwordRegIndex) { *DwordRegIndex = RegNum; }
1739     RegNum = RegNum / Size;
1740     int RCID = getRegClass(RegKind, RegWidth);
1741     if (RCID == -1)
1742       return false;
1743     const MCRegisterClass RC = TRI->getRegClass(RCID);
1744     if (RegNum >= RC.getNumRegs())
1745       return false;
1746     Reg = RC.getRegister(RegNum);
1747     break;
1748   }
1749 
1750   default:
1751     llvm_unreachable("unexpected register kind");
1752   }
1753 
1754   if (!subtargetHasRegister(*TRI, Reg))
1755     return false;
1756   return true;
1757 }
1758 
1759 std::unique_ptr<AMDGPUOperand> AMDGPUAsmParser::parseRegister() {
1760   const auto &Tok = Parser.getTok();
1761   SMLoc StartLoc = Tok.getLoc();
1762   SMLoc EndLoc = Tok.getEndLoc();
1763   RegisterKind RegKind;
1764   unsigned Reg, RegNum, RegWidth, DwordRegIndex;
1765 
1766   if (!ParseAMDGPURegister(RegKind, Reg, RegNum, RegWidth, &DwordRegIndex)) {
1767     return nullptr;
1768   }
1769   KernelScope.usesRegister(RegKind, DwordRegIndex, RegWidth);
1770   return AMDGPUOperand::CreateReg(this, Reg, StartLoc, EndLoc, false);
1771 }
1772 
1773 bool
1774 AMDGPUAsmParser::parseAbsoluteExpr(int64_t &Val, bool AbsMod) {
1775   if (AbsMod && getLexer().peekTok().is(AsmToken::Pipe) &&
1776       (getLexer().getKind() == AsmToken::Integer ||
1777        getLexer().getKind() == AsmToken::Real)) {
1778     // This is a workaround for handling operands like these:
1779     //     |1.0|
1780     //     |-1|
1781     // This syntax is not compatible with syntax of standard
1782     // MC expressions (due to the trailing '|').
1783 
1784     SMLoc EndLoc;
1785     const MCExpr *Expr;
1786 
1787     if (getParser().parsePrimaryExpr(Expr, EndLoc)) {
1788       return true;
1789     }
1790 
1791     return !Expr->evaluateAsAbsolute(Val);
1792   }
1793 
1794   return getParser().parseAbsoluteExpression(Val);
1795 }
1796 
1797 OperandMatchResultTy
1798 AMDGPUAsmParser::parseImm(OperandVector &Operands, bool AbsMod) {
1799   // TODO: add syntactic sugar for 1/(2*PI)
1800   bool Minus = false;
1801   if (getLexer().getKind() == AsmToken::Minus) {
1802     const AsmToken NextToken = getLexer().peekTok();
1803     if (!NextToken.is(AsmToken::Integer) &&
1804         !NextToken.is(AsmToken::Real)) {
1805         return MatchOperand_NoMatch;
1806     }
1807     Minus = true;
1808     Parser.Lex();
1809   }
1810 
1811   SMLoc S = Parser.getTok().getLoc();
1812   switch(getLexer().getKind()) {
1813   case AsmToken::Integer: {
1814     int64_t IntVal;
1815     if (parseAbsoluteExpr(IntVal, AbsMod))
1816       return MatchOperand_ParseFail;
1817     if (Minus)
1818       IntVal *= -1;
1819     Operands.push_back(AMDGPUOperand::CreateImm(this, IntVal, S));
1820     return MatchOperand_Success;
1821   }
1822   case AsmToken::Real: {
1823     int64_t IntVal;
1824     if (parseAbsoluteExpr(IntVal, AbsMod))
1825       return MatchOperand_ParseFail;
1826 
1827     APFloat F(BitsToDouble(IntVal));
1828     if (Minus)
1829       F.changeSign();
1830     Operands.push_back(
1831         AMDGPUOperand::CreateImm(this, F.bitcastToAPInt().getZExtValue(), S,
1832                                  AMDGPUOperand::ImmTyNone, true));
1833     return MatchOperand_Success;
1834   }
1835   default:
1836     return MatchOperand_NoMatch;
1837   }
1838 }
1839 
1840 OperandMatchResultTy
1841 AMDGPUAsmParser::parseReg(OperandVector &Operands) {
1842   if (auto R = parseRegister()) {
1843     assert(R->isReg());
1844     R->Reg.IsForcedVOP3 = isForcedVOP3();
1845     Operands.push_back(std::move(R));
1846     return MatchOperand_Success;
1847   }
1848   return MatchOperand_NoMatch;
1849 }
1850 
1851 OperandMatchResultTy
1852 AMDGPUAsmParser::parseRegOrImm(OperandVector &Operands, bool AbsMod) {
1853   auto res = parseImm(Operands, AbsMod);
1854   if (res != MatchOperand_NoMatch) {
1855     return res;
1856   }
1857 
1858   return parseReg(Operands);
1859 }
1860 
1861 OperandMatchResultTy
1862 AMDGPUAsmParser::parseRegOrImmWithFPInputMods(OperandVector &Operands,
1863                                               bool AllowImm) {
1864   bool Negate = false, Negate2 = false, Abs = false, Abs2 = false;
1865 
1866   if (getLexer().getKind()== AsmToken::Minus) {
1867     const AsmToken NextToken = getLexer().peekTok();
1868 
1869     // Disable ambiguous constructs like '--1' etc. Should use neg(-1) instead.
1870     if (NextToken.is(AsmToken::Minus)) {
1871       Error(Parser.getTok().getLoc(), "invalid syntax, expected 'neg' modifier");
1872       return MatchOperand_ParseFail;
1873     }
1874 
1875     // '-' followed by an integer literal N should be interpreted as integer
1876     // negation rather than a floating-point NEG modifier applied to N.
1877     // Beside being contr-intuitive, such use of floating-point NEG modifier
1878     // results in different meaning of integer literals used with VOP1/2/C
1879     // and VOP3, for example:
1880     //    v_exp_f32_e32 v5, -1 // VOP1: src0 = 0xFFFFFFFF
1881     //    v_exp_f32_e64 v5, -1 // VOP3: src0 = 0x80000001
1882     // Negative fp literals should be handled likewise for unifomtity
1883     if (!NextToken.is(AsmToken::Integer) && !NextToken.is(AsmToken::Real)) {
1884       Parser.Lex();
1885       Negate = true;
1886     }
1887   }
1888 
1889   if (getLexer().getKind() == AsmToken::Identifier &&
1890       Parser.getTok().getString() == "neg") {
1891     if (Negate) {
1892       Error(Parser.getTok().getLoc(), "expected register or immediate");
1893       return MatchOperand_ParseFail;
1894     }
1895     Parser.Lex();
1896     Negate2 = true;
1897     if (getLexer().isNot(AsmToken::LParen)) {
1898       Error(Parser.getTok().getLoc(), "expected left paren after neg");
1899       return MatchOperand_ParseFail;
1900     }
1901     Parser.Lex();
1902   }
1903 
1904   if (getLexer().getKind() == AsmToken::Identifier &&
1905       Parser.getTok().getString() == "abs") {
1906     Parser.Lex();
1907     Abs2 = true;
1908     if (getLexer().isNot(AsmToken::LParen)) {
1909       Error(Parser.getTok().getLoc(), "expected left paren after abs");
1910       return MatchOperand_ParseFail;
1911     }
1912     Parser.Lex();
1913   }
1914 
1915   if (getLexer().getKind() == AsmToken::Pipe) {
1916     if (Abs2) {
1917       Error(Parser.getTok().getLoc(), "expected register or immediate");
1918       return MatchOperand_ParseFail;
1919     }
1920     Parser.Lex();
1921     Abs = true;
1922   }
1923 
1924   OperandMatchResultTy Res;
1925   if (AllowImm) {
1926     Res = parseRegOrImm(Operands, Abs);
1927   } else {
1928     Res = parseReg(Operands);
1929   }
1930   if (Res != MatchOperand_Success) {
1931     return Res;
1932   }
1933 
1934   AMDGPUOperand::Modifiers Mods;
1935   if (Abs) {
1936     if (getLexer().getKind() != AsmToken::Pipe) {
1937       Error(Parser.getTok().getLoc(), "expected vertical bar");
1938       return MatchOperand_ParseFail;
1939     }
1940     Parser.Lex();
1941     Mods.Abs = true;
1942   }
1943   if (Abs2) {
1944     if (getLexer().isNot(AsmToken::RParen)) {
1945       Error(Parser.getTok().getLoc(), "expected closing parentheses");
1946       return MatchOperand_ParseFail;
1947     }
1948     Parser.Lex();
1949     Mods.Abs = true;
1950   }
1951 
1952   if (Negate) {
1953     Mods.Neg = true;
1954   } else if (Negate2) {
1955     if (getLexer().isNot(AsmToken::RParen)) {
1956       Error(Parser.getTok().getLoc(), "expected closing parentheses");
1957       return MatchOperand_ParseFail;
1958     }
1959     Parser.Lex();
1960     Mods.Neg = true;
1961   }
1962 
1963   if (Mods.hasFPModifiers()) {
1964     AMDGPUOperand &Op = static_cast<AMDGPUOperand &>(*Operands.back());
1965     Op.setModifiers(Mods);
1966   }
1967   return MatchOperand_Success;
1968 }
1969 
1970 OperandMatchResultTy
1971 AMDGPUAsmParser::parseRegOrImmWithIntInputMods(OperandVector &Operands,
1972                                                bool AllowImm) {
1973   bool Sext = false;
1974 
1975   if (getLexer().getKind() == AsmToken::Identifier &&
1976       Parser.getTok().getString() == "sext") {
1977     Parser.Lex();
1978     Sext = true;
1979     if (getLexer().isNot(AsmToken::LParen)) {
1980       Error(Parser.getTok().getLoc(), "expected left paren after sext");
1981       return MatchOperand_ParseFail;
1982     }
1983     Parser.Lex();
1984   }
1985 
1986   OperandMatchResultTy Res;
1987   if (AllowImm) {
1988     Res = parseRegOrImm(Operands);
1989   } else {
1990     Res = parseReg(Operands);
1991   }
1992   if (Res != MatchOperand_Success) {
1993     return Res;
1994   }
1995 
1996   AMDGPUOperand::Modifiers Mods;
1997   if (Sext) {
1998     if (getLexer().isNot(AsmToken::RParen)) {
1999       Error(Parser.getTok().getLoc(), "expected closing parentheses");
2000       return MatchOperand_ParseFail;
2001     }
2002     Parser.Lex();
2003     Mods.Sext = true;
2004   }
2005 
2006   if (Mods.hasIntModifiers()) {
2007     AMDGPUOperand &Op = static_cast<AMDGPUOperand &>(*Operands.back());
2008     Op.setModifiers(Mods);
2009   }
2010 
2011   return MatchOperand_Success;
2012 }
2013 
2014 OperandMatchResultTy
2015 AMDGPUAsmParser::parseRegWithFPInputMods(OperandVector &Operands) {
2016   return parseRegOrImmWithFPInputMods(Operands, false);
2017 }
2018 
2019 OperandMatchResultTy
2020 AMDGPUAsmParser::parseRegWithIntInputMods(OperandVector &Operands) {
2021   return parseRegOrImmWithIntInputMods(Operands, false);
2022 }
2023 
2024 OperandMatchResultTy AMDGPUAsmParser::parseVReg32OrOff(OperandVector &Operands) {
2025   std::unique_ptr<AMDGPUOperand> Reg = parseRegister();
2026   if (Reg) {
2027     Operands.push_back(std::move(Reg));
2028     return MatchOperand_Success;
2029   }
2030 
2031   const AsmToken &Tok = Parser.getTok();
2032   if (Tok.getString() == "off") {
2033     Operands.push_back(AMDGPUOperand::CreateImm(this, 0, Tok.getLoc(),
2034                                                 AMDGPUOperand::ImmTyOff, false));
2035     Parser.Lex();
2036     return MatchOperand_Success;
2037   }
2038 
2039   return MatchOperand_NoMatch;
2040 }
2041 
2042 unsigned AMDGPUAsmParser::checkTargetMatchPredicate(MCInst &Inst) {
2043   uint64_t TSFlags = MII.get(Inst.getOpcode()).TSFlags;
2044 
2045   if ((getForcedEncodingSize() == 32 && (TSFlags & SIInstrFlags::VOP3)) ||
2046       (getForcedEncodingSize() == 64 && !(TSFlags & SIInstrFlags::VOP3)) ||
2047       (isForcedDPP() && !(TSFlags & SIInstrFlags::DPP)) ||
2048       (isForcedSDWA() && !(TSFlags & SIInstrFlags::SDWA)) )
2049     return Match_InvalidOperand;
2050 
2051   if ((TSFlags & SIInstrFlags::VOP3) &&
2052       (TSFlags & SIInstrFlags::VOPAsmPrefer32Bit) &&
2053       getForcedEncodingSize() != 64)
2054     return Match_PreferE32;
2055 
2056   if (Inst.getOpcode() == AMDGPU::V_MAC_F32_sdwa_vi ||
2057       Inst.getOpcode() == AMDGPU::V_MAC_F16_sdwa_vi) {
2058     // v_mac_f32/16 allow only dst_sel == DWORD;
2059     auto OpNum =
2060         AMDGPU::getNamedOperandIdx(Inst.getOpcode(), AMDGPU::OpName::dst_sel);
2061     const auto &Op = Inst.getOperand(OpNum);
2062     if (!Op.isImm() || Op.getImm() != AMDGPU::SDWA::SdwaSel::DWORD) {
2063       return Match_InvalidOperand;
2064     }
2065   }
2066 
2067   if ((TSFlags & SIInstrFlags::FLAT) && !hasFlatOffsets()) {
2068     // FIXME: Produces error without correct column reported.
2069     auto OpNum =
2070         AMDGPU::getNamedOperandIdx(Inst.getOpcode(), AMDGPU::OpName::offset);
2071     const auto &Op = Inst.getOperand(OpNum);
2072     if (Op.getImm() != 0)
2073       return Match_InvalidOperand;
2074   }
2075 
2076   return Match_Success;
2077 }
2078 
2079 // What asm variants we should check
2080 ArrayRef<unsigned> AMDGPUAsmParser::getMatchedVariants() const {
2081   if (getForcedEncodingSize() == 32) {
2082     static const unsigned Variants[] = {AMDGPUAsmVariants::DEFAULT};
2083     return makeArrayRef(Variants);
2084   }
2085 
2086   if (isForcedVOP3()) {
2087     static const unsigned Variants[] = {AMDGPUAsmVariants::VOP3};
2088     return makeArrayRef(Variants);
2089   }
2090 
2091   if (isForcedSDWA()) {
2092     static const unsigned Variants[] = {AMDGPUAsmVariants::SDWA,
2093                                         AMDGPUAsmVariants::SDWA9};
2094     return makeArrayRef(Variants);
2095   }
2096 
2097   if (isForcedDPP()) {
2098     static const unsigned Variants[] = {AMDGPUAsmVariants::DPP};
2099     return makeArrayRef(Variants);
2100   }
2101 
2102   static const unsigned Variants[] = {
2103     AMDGPUAsmVariants::DEFAULT, AMDGPUAsmVariants::VOP3,
2104     AMDGPUAsmVariants::SDWA, AMDGPUAsmVariants::SDWA9, AMDGPUAsmVariants::DPP
2105   };
2106 
2107   return makeArrayRef(Variants);
2108 }
2109 
2110 unsigned AMDGPUAsmParser::findImplicitSGPRReadInVOP(const MCInst &Inst) const {
2111   const MCInstrDesc &Desc = MII.get(Inst.getOpcode());
2112   const unsigned Num = Desc.getNumImplicitUses();
2113   for (unsigned i = 0; i < Num; ++i) {
2114     unsigned Reg = Desc.ImplicitUses[i];
2115     switch (Reg) {
2116     case AMDGPU::FLAT_SCR:
2117     case AMDGPU::VCC:
2118     case AMDGPU::M0:
2119       return Reg;
2120     default:
2121       break;
2122     }
2123   }
2124   return AMDGPU::NoRegister;
2125 }
2126 
2127 // NB: This code is correct only when used to check constant
2128 // bus limitations because GFX7 support no f16 inline constants.
2129 // Note that there are no cases when a GFX7 opcode violates
2130 // constant bus limitations due to the use of an f16 constant.
2131 bool AMDGPUAsmParser::isInlineConstant(const MCInst &Inst,
2132                                        unsigned OpIdx) const {
2133   const MCInstrDesc &Desc = MII.get(Inst.getOpcode());
2134 
2135   if (!AMDGPU::isSISrcOperand(Desc, OpIdx)) {
2136     return false;
2137   }
2138 
2139   const MCOperand &MO = Inst.getOperand(OpIdx);
2140 
2141   int64_t Val = MO.getImm();
2142   auto OpSize = AMDGPU::getOperandSize(Desc, OpIdx);
2143 
2144   switch (OpSize) { // expected operand size
2145   case 8:
2146     return AMDGPU::isInlinableLiteral64(Val, hasInv2PiInlineImm());
2147   case 4:
2148     return AMDGPU::isInlinableLiteral32(Val, hasInv2PiInlineImm());
2149   case 2: {
2150     const unsigned OperandType = Desc.OpInfo[OpIdx].OperandType;
2151     if (OperandType == AMDGPU::OPERAND_REG_INLINE_C_V2INT16 ||
2152         OperandType == AMDGPU::OPERAND_REG_INLINE_C_V2FP16) {
2153       return AMDGPU::isInlinableLiteralV216(Val, hasInv2PiInlineImm());
2154     } else {
2155       return AMDGPU::isInlinableLiteral16(Val, hasInv2PiInlineImm());
2156     }
2157   }
2158   default:
2159     llvm_unreachable("invalid operand size");
2160   }
2161 }
2162 
2163 bool AMDGPUAsmParser::usesConstantBus(const MCInst &Inst, unsigned OpIdx) {
2164   const MCOperand &MO = Inst.getOperand(OpIdx);
2165   if (MO.isImm()) {
2166     return !isInlineConstant(Inst, OpIdx);
2167   }
2168   return !MO.isReg() ||
2169          isSGPR(mc2PseudoReg(MO.getReg()), getContext().getRegisterInfo());
2170 }
2171 
2172 bool AMDGPUAsmParser::validateConstantBusLimitations(const MCInst &Inst) {
2173   const unsigned Opcode = Inst.getOpcode();
2174   const MCInstrDesc &Desc = MII.get(Opcode);
2175   unsigned ConstantBusUseCount = 0;
2176 
2177   if (Desc.TSFlags &
2178       (SIInstrFlags::VOPC |
2179        SIInstrFlags::VOP1 | SIInstrFlags::VOP2 |
2180        SIInstrFlags::VOP3 | SIInstrFlags::VOP3P |
2181        SIInstrFlags::SDWA)) {
2182     // Check special imm operands (used by madmk, etc)
2183     if (AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::imm) != -1) {
2184       ++ConstantBusUseCount;
2185     }
2186 
2187     unsigned SGPRUsed = findImplicitSGPRReadInVOP(Inst);
2188     if (SGPRUsed != AMDGPU::NoRegister) {
2189       ++ConstantBusUseCount;
2190     }
2191 
2192     const int Src0Idx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::src0);
2193     const int Src1Idx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::src1);
2194     const int Src2Idx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::src2);
2195 
2196     const int OpIndices[] = { Src0Idx, Src1Idx, Src2Idx };
2197 
2198     for (int OpIdx : OpIndices) {
2199       if (OpIdx == -1) break;
2200 
2201       const MCOperand &MO = Inst.getOperand(OpIdx);
2202       if (usesConstantBus(Inst, OpIdx)) {
2203         if (MO.isReg()) {
2204           const unsigned Reg = mc2PseudoReg(MO.getReg());
2205           // Pairs of registers with a partial intersections like these
2206           //   s0, s[0:1]
2207           //   flat_scratch_lo, flat_scratch
2208           //   flat_scratch_lo, flat_scratch_hi
2209           // are theoretically valid but they are disabled anyway.
2210           // Note that this code mimics SIInstrInfo::verifyInstruction
2211           if (Reg != SGPRUsed) {
2212             ++ConstantBusUseCount;
2213           }
2214           SGPRUsed = Reg;
2215         } else { // Expression or a literal
2216           ++ConstantBusUseCount;
2217         }
2218       }
2219     }
2220   }
2221 
2222   return ConstantBusUseCount <= 1;
2223 }
2224 
2225 bool AMDGPUAsmParser::validateEarlyClobberLimitations(const MCInst &Inst) {
2226   const unsigned Opcode = Inst.getOpcode();
2227   const MCInstrDesc &Desc = MII.get(Opcode);
2228 
2229   const int DstIdx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::vdst);
2230   if (DstIdx == -1 ||
2231       Desc.getOperandConstraint(DstIdx, MCOI::EARLY_CLOBBER) == -1) {
2232     return true;
2233   }
2234 
2235   const MCRegisterInfo *TRI = getContext().getRegisterInfo();
2236 
2237   const int Src0Idx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::src0);
2238   const int Src1Idx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::src1);
2239   const int Src2Idx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::src2);
2240 
2241   assert(DstIdx != -1);
2242   const MCOperand &Dst = Inst.getOperand(DstIdx);
2243   assert(Dst.isReg());
2244   const unsigned DstReg = mc2PseudoReg(Dst.getReg());
2245 
2246   const int SrcIndices[] = { Src0Idx, Src1Idx, Src2Idx };
2247 
2248   for (int SrcIdx : SrcIndices) {
2249     if (SrcIdx == -1) break;
2250     const MCOperand &Src = Inst.getOperand(SrcIdx);
2251     if (Src.isReg()) {
2252       const unsigned SrcReg = mc2PseudoReg(Src.getReg());
2253       if (isRegIntersect(DstReg, SrcReg, TRI)) {
2254         return false;
2255       }
2256     }
2257   }
2258 
2259   return true;
2260 }
2261 
2262 bool AMDGPUAsmParser::validateIntClampSupported(const MCInst &Inst) {
2263 
2264   const unsigned Opc = Inst.getOpcode();
2265   const MCInstrDesc &Desc = MII.get(Opc);
2266 
2267   if ((Desc.TSFlags & SIInstrFlags::IntClamp) != 0 && !hasIntClamp()) {
2268     int ClampIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::clamp);
2269     assert(ClampIdx != -1);
2270     return Inst.getOperand(ClampIdx).getImm() == 0;
2271   }
2272 
2273   return true;
2274 }
2275 
2276 bool AMDGPUAsmParser::validateMIMGDataSize(const MCInst &Inst) {
2277 
2278   const unsigned Opc = Inst.getOpcode();
2279   const MCInstrDesc &Desc = MII.get(Opc);
2280 
2281   if ((Desc.TSFlags & SIInstrFlags::MIMG) == 0)
2282     return true;
2283 
2284   // Gather4 instructions seem to have special rules not described in spec.
2285   if (Desc.TSFlags & SIInstrFlags::Gather4)
2286     return true;
2287 
2288   int VDataIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::vdata);
2289   int DMaskIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::dmask);
2290   int TFEIdx   = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::tfe);
2291 
2292   assert(VDataIdx != -1);
2293   assert(DMaskIdx != -1);
2294   assert(TFEIdx != -1);
2295 
2296   unsigned VDataSize = AMDGPU::getRegOperandSize(getMRI(), Desc, VDataIdx);
2297   unsigned TFESize = Inst.getOperand(TFEIdx).getImm()? 1 : 0;
2298   unsigned DMask = Inst.getOperand(DMaskIdx).getImm() & 0xf;
2299   if (DMask == 0)
2300     DMask = 1;
2301 
2302   return (VDataSize / 4) == countPopulation(DMask) + TFESize;
2303 }
2304 
2305 bool AMDGPUAsmParser::validateMIMGAtomicDMask(const MCInst &Inst) {
2306 
2307   const unsigned Opc = Inst.getOpcode();
2308   const MCInstrDesc &Desc = MII.get(Opc);
2309 
2310   if ((Desc.TSFlags & SIInstrFlags::MIMG) == 0)
2311     return true;
2312   if (!Desc.mayLoad() || !Desc.mayStore())
2313     return true; // Not atomic
2314 
2315   int DMaskIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::dmask);
2316   unsigned DMask = Inst.getOperand(DMaskIdx).getImm() & 0xf;
2317 
2318   // This is an incomplete check because image_atomic_cmpswap
2319   // may only use 0x3 and 0xf while other atomic operations
2320   // may use 0x1 and 0x3. However these limitations are
2321   // verified when we check that dmask matches dst size.
2322   return DMask == 0x1 || DMask == 0x3 || DMask == 0xf;
2323 }
2324 
2325 bool AMDGPUAsmParser::validateInstruction(const MCInst &Inst,
2326                                           const SMLoc &IDLoc) {
2327   if (!validateConstantBusLimitations(Inst)) {
2328     Error(IDLoc,
2329       "invalid operand (violates constant bus restrictions)");
2330     return false;
2331   }
2332   if (!validateEarlyClobberLimitations(Inst)) {
2333     Error(IDLoc,
2334       "destination must be different than all sources");
2335     return false;
2336   }
2337   if (!validateIntClampSupported(Inst)) {
2338     Error(IDLoc,
2339       "integer clamping is not supported on this GPU");
2340     return false;
2341   }
2342   if (!validateMIMGDataSize(Inst)) {
2343     Error(IDLoc,
2344       "image data size does not match dmask and tfe");
2345     return false;
2346   }
2347   if (!validateMIMGAtomicDMask(Inst)) {
2348     Error(IDLoc,
2349       "invalid atomic image dmask");
2350     return false;
2351   }
2352 
2353   return true;
2354 }
2355 
2356 static std::string AMDGPUMnemonicSpellCheck(StringRef S, uint64_t FBS,
2357                                             unsigned VariantID = 0);
2358 
2359 bool AMDGPUAsmParser::MatchAndEmitInstruction(SMLoc IDLoc, unsigned &Opcode,
2360                                               OperandVector &Operands,
2361                                               MCStreamer &Out,
2362                                               uint64_t &ErrorInfo,
2363                                               bool MatchingInlineAsm) {
2364   MCInst Inst;
2365   unsigned Result = Match_Success;
2366   for (auto Variant : getMatchedVariants()) {
2367     uint64_t EI;
2368     auto R = MatchInstructionImpl(Operands, Inst, EI, MatchingInlineAsm,
2369                                   Variant);
2370     // We order match statuses from least to most specific. We use most specific
2371     // status as resulting
2372     // Match_MnemonicFail < Match_InvalidOperand < Match_MissingFeature < Match_PreferE32
2373     if ((R == Match_Success) ||
2374         (R == Match_PreferE32) ||
2375         (R == Match_MissingFeature && Result != Match_PreferE32) ||
2376         (R == Match_InvalidOperand && Result != Match_MissingFeature
2377                                    && Result != Match_PreferE32) ||
2378         (R == Match_MnemonicFail   && Result != Match_InvalidOperand
2379                                    && Result != Match_MissingFeature
2380                                    && Result != Match_PreferE32)) {
2381       Result = R;
2382       ErrorInfo = EI;
2383     }
2384     if (R == Match_Success)
2385       break;
2386   }
2387 
2388   switch (Result) {
2389   default: break;
2390   case Match_Success:
2391     if (!validateInstruction(Inst, IDLoc)) {
2392       return true;
2393     }
2394     Inst.setLoc(IDLoc);
2395     Out.EmitInstruction(Inst, getSTI());
2396     return false;
2397 
2398   case Match_MissingFeature:
2399     return Error(IDLoc, "instruction not supported on this GPU");
2400 
2401   case Match_MnemonicFail: {
2402     uint64_t FBS = ComputeAvailableFeatures(getSTI().getFeatureBits());
2403     std::string Suggestion = AMDGPUMnemonicSpellCheck(
2404         ((AMDGPUOperand &)*Operands[0]).getToken(), FBS);
2405     return Error(IDLoc, "invalid instruction" + Suggestion,
2406                  ((AMDGPUOperand &)*Operands[0]).getLocRange());
2407   }
2408 
2409   case Match_InvalidOperand: {
2410     SMLoc ErrorLoc = IDLoc;
2411     if (ErrorInfo != ~0ULL) {
2412       if (ErrorInfo >= Operands.size()) {
2413         return Error(IDLoc, "too few operands for instruction");
2414       }
2415       ErrorLoc = ((AMDGPUOperand &)*Operands[ErrorInfo]).getStartLoc();
2416       if (ErrorLoc == SMLoc())
2417         ErrorLoc = IDLoc;
2418     }
2419     return Error(ErrorLoc, "invalid operand for instruction");
2420   }
2421 
2422   case Match_PreferE32:
2423     return Error(IDLoc, "internal error: instruction without _e64 suffix "
2424                         "should be encoded as e32");
2425   }
2426   llvm_unreachable("Implement any new match types added!");
2427 }
2428 
2429 bool AMDGPUAsmParser::ParseAsAbsoluteExpression(uint32_t &Ret) {
2430   int64_t Tmp = -1;
2431   if (getLexer().isNot(AsmToken::Integer) && getLexer().isNot(AsmToken::Identifier)) {
2432     return true;
2433   }
2434   if (getParser().parseAbsoluteExpression(Tmp)) {
2435     return true;
2436   }
2437   Ret = static_cast<uint32_t>(Tmp);
2438   return false;
2439 }
2440 
2441 bool AMDGPUAsmParser::ParseDirectiveMajorMinor(uint32_t &Major,
2442                                                uint32_t &Minor) {
2443   if (ParseAsAbsoluteExpression(Major))
2444     return TokError("invalid major version");
2445 
2446   if (getLexer().isNot(AsmToken::Comma))
2447     return TokError("minor version number required, comma expected");
2448   Lex();
2449 
2450   if (ParseAsAbsoluteExpression(Minor))
2451     return TokError("invalid minor version");
2452 
2453   return false;
2454 }
2455 
2456 bool AMDGPUAsmParser::ParseDirectiveHSACodeObjectVersion() {
2457   uint32_t Major;
2458   uint32_t Minor;
2459 
2460   if (ParseDirectiveMajorMinor(Major, Minor))
2461     return true;
2462 
2463   getTargetStreamer().EmitDirectiveHSACodeObjectVersion(Major, Minor);
2464   return false;
2465 }
2466 
2467 bool AMDGPUAsmParser::ParseDirectiveHSACodeObjectISA() {
2468   uint32_t Major;
2469   uint32_t Minor;
2470   uint32_t Stepping;
2471   StringRef VendorName;
2472   StringRef ArchName;
2473 
2474   // If this directive has no arguments, then use the ISA version for the
2475   // targeted GPU.
2476   if (getLexer().is(AsmToken::EndOfStatement)) {
2477     AMDGPU::IsaInfo::IsaVersion ISA =
2478         AMDGPU::IsaInfo::getIsaVersion(getFeatureBits());
2479     getTargetStreamer().EmitDirectiveHSACodeObjectISA(ISA.Major, ISA.Minor,
2480                                                       ISA.Stepping,
2481                                                       "AMD", "AMDGPU");
2482     return false;
2483   }
2484 
2485   if (ParseDirectiveMajorMinor(Major, Minor))
2486     return true;
2487 
2488   if (getLexer().isNot(AsmToken::Comma))
2489     return TokError("stepping version number required, comma expected");
2490   Lex();
2491 
2492   if (ParseAsAbsoluteExpression(Stepping))
2493     return TokError("invalid stepping version");
2494 
2495   if (getLexer().isNot(AsmToken::Comma))
2496     return TokError("vendor name required, comma expected");
2497   Lex();
2498 
2499   if (getLexer().isNot(AsmToken::String))
2500     return TokError("invalid vendor name");
2501 
2502   VendorName = getLexer().getTok().getStringContents();
2503   Lex();
2504 
2505   if (getLexer().isNot(AsmToken::Comma))
2506     return TokError("arch name required, comma expected");
2507   Lex();
2508 
2509   if (getLexer().isNot(AsmToken::String))
2510     return TokError("invalid arch name");
2511 
2512   ArchName = getLexer().getTok().getStringContents();
2513   Lex();
2514 
2515   getTargetStreamer().EmitDirectiveHSACodeObjectISA(Major, Minor, Stepping,
2516                                                     VendorName, ArchName);
2517   return false;
2518 }
2519 
2520 bool AMDGPUAsmParser::ParseAMDKernelCodeTValue(StringRef ID,
2521                                                amd_kernel_code_t &Header) {
2522   SmallString<40> ErrStr;
2523   raw_svector_ostream Err(ErrStr);
2524   if (!parseAmdKernelCodeField(ID, getParser(), Header, Err)) {
2525     return TokError(Err.str());
2526   }
2527   Lex();
2528   return false;
2529 }
2530 
2531 bool AMDGPUAsmParser::ParseDirectiveAMDKernelCodeT() {
2532   amd_kernel_code_t Header;
2533   AMDGPU::initDefaultAMDKernelCodeT(Header, getFeatureBits());
2534 
2535   while (true) {
2536     // Lex EndOfStatement.  This is in a while loop, because lexing a comment
2537     // will set the current token to EndOfStatement.
2538     while(getLexer().is(AsmToken::EndOfStatement))
2539       Lex();
2540 
2541     if (getLexer().isNot(AsmToken::Identifier))
2542       return TokError("expected value identifier or .end_amd_kernel_code_t");
2543 
2544     StringRef ID = getLexer().getTok().getIdentifier();
2545     Lex();
2546 
2547     if (ID == ".end_amd_kernel_code_t")
2548       break;
2549 
2550     if (ParseAMDKernelCodeTValue(ID, Header))
2551       return true;
2552   }
2553 
2554   getTargetStreamer().EmitAMDKernelCodeT(Header);
2555 
2556   return false;
2557 }
2558 
2559 bool AMDGPUAsmParser::ParseDirectiveAMDGPUHsaKernel() {
2560   if (getLexer().isNot(AsmToken::Identifier))
2561     return TokError("expected symbol name");
2562 
2563   StringRef KernelName = Parser.getTok().getString();
2564 
2565   getTargetStreamer().EmitAMDGPUSymbolType(KernelName,
2566                                            ELF::STT_AMDGPU_HSA_KERNEL);
2567   Lex();
2568   KernelScope.initialize(getContext());
2569   return false;
2570 }
2571 
2572 bool AMDGPUAsmParser::ParseDirectiveISAVersion() {
2573   if (getSTI().getTargetTriple().getArch() != Triple::amdgcn) {
2574     return Error(getParser().getTok().getLoc(),
2575                  ".amd_amdgpu_isa directive is not available on non-amdgcn "
2576                  "architectures");
2577   }
2578 
2579   auto ISAVersionStringFromASM = getLexer().getTok().getStringContents();
2580 
2581   std::string ISAVersionStringFromSTI;
2582   raw_string_ostream ISAVersionStreamFromSTI(ISAVersionStringFromSTI);
2583   IsaInfo::streamIsaVersion(&getSTI(), ISAVersionStreamFromSTI);
2584 
2585   if (ISAVersionStringFromASM != ISAVersionStreamFromSTI.str()) {
2586     return Error(getParser().getTok().getLoc(),
2587                  ".amd_amdgpu_isa directive does not match triple and/or mcpu "
2588                  "arguments specified through the command line");
2589   }
2590 
2591   getTargetStreamer().EmitISAVersion(ISAVersionStreamFromSTI.str());
2592   Lex();
2593 
2594   return false;
2595 }
2596 
2597 bool AMDGPUAsmParser::ParseDirectiveHSAMetadata() {
2598   if (getSTI().getTargetTriple().getOS() != Triple::AMDHSA) {
2599     return Error(getParser().getTok().getLoc(),
2600                  (Twine(HSAMD::AssemblerDirectiveBegin) + Twine(" directive is "
2601                  "not available on non-amdhsa OSes")).str());
2602   }
2603 
2604   std::string HSAMetadataString;
2605   raw_string_ostream YamlStream(HSAMetadataString);
2606 
2607   getLexer().setSkipSpace(false);
2608 
2609   bool FoundEnd = false;
2610   while (!getLexer().is(AsmToken::Eof)) {
2611     while (getLexer().is(AsmToken::Space)) {
2612       YamlStream << getLexer().getTok().getString();
2613       Lex();
2614     }
2615 
2616     if (getLexer().is(AsmToken::Identifier)) {
2617       StringRef ID = getLexer().getTok().getIdentifier();
2618       if (ID == AMDGPU::HSAMD::AssemblerDirectiveEnd) {
2619         Lex();
2620         FoundEnd = true;
2621         break;
2622       }
2623     }
2624 
2625     YamlStream << Parser.parseStringToEndOfStatement()
2626                << getContext().getAsmInfo()->getSeparatorString();
2627 
2628     Parser.eatToEndOfStatement();
2629   }
2630 
2631   getLexer().setSkipSpace(true);
2632 
2633   if (getLexer().is(AsmToken::Eof) && !FoundEnd) {
2634     return TokError(Twine("expected directive ") +
2635                     Twine(HSAMD::AssemblerDirectiveEnd) + Twine(" not found"));
2636   }
2637 
2638   YamlStream.flush();
2639 
2640   if (!getTargetStreamer().EmitHSAMetadata(HSAMetadataString))
2641     return Error(getParser().getTok().getLoc(), "invalid HSA metadata");
2642 
2643   return false;
2644 }
2645 
2646 bool AMDGPUAsmParser::ParseDirectivePALMetadata() {
2647   if (getSTI().getTargetTriple().getOS() != Triple::AMDPAL) {
2648     return Error(getParser().getTok().getLoc(),
2649                  (Twine(PALMD::AssemblerDirective) + Twine(" directive is "
2650                  "not available on non-amdpal OSes")).str());
2651   }
2652 
2653   PALMD::Metadata PALMetadata;
2654   for (;;) {
2655     uint32_t Value;
2656     if (ParseAsAbsoluteExpression(Value)) {
2657       return TokError(Twine("invalid value in ") +
2658                       Twine(PALMD::AssemblerDirective));
2659     }
2660     PALMetadata.push_back(Value);
2661     if (getLexer().isNot(AsmToken::Comma))
2662       break;
2663     Lex();
2664   }
2665   getTargetStreamer().EmitPALMetadata(PALMetadata);
2666   return false;
2667 }
2668 
2669 bool AMDGPUAsmParser::ParseDirective(AsmToken DirectiveID) {
2670   StringRef IDVal = DirectiveID.getString();
2671 
2672   if (IDVal == ".hsa_code_object_version")
2673     return ParseDirectiveHSACodeObjectVersion();
2674 
2675   if (IDVal == ".hsa_code_object_isa")
2676     return ParseDirectiveHSACodeObjectISA();
2677 
2678   if (IDVal == ".amd_kernel_code_t")
2679     return ParseDirectiveAMDKernelCodeT();
2680 
2681   if (IDVal == ".amdgpu_hsa_kernel")
2682     return ParseDirectiveAMDGPUHsaKernel();
2683 
2684   if (IDVal == ".amd_amdgpu_isa")
2685     return ParseDirectiveISAVersion();
2686 
2687   if (IDVal == AMDGPU::HSAMD::AssemblerDirectiveBegin)
2688     return ParseDirectiveHSAMetadata();
2689 
2690   if (IDVal == PALMD::AssemblerDirective)
2691     return ParseDirectivePALMetadata();
2692 
2693   return true;
2694 }
2695 
2696 bool AMDGPUAsmParser::subtargetHasRegister(const MCRegisterInfo &MRI,
2697                                            unsigned RegNo) const {
2698 
2699   for (MCRegAliasIterator R(AMDGPU::TTMP12_TTMP13_TTMP14_TTMP15, &MRI, true);
2700        R.isValid(); ++R) {
2701     if (*R == RegNo)
2702       return isGFX9();
2703   }
2704 
2705   switch (RegNo) {
2706   case AMDGPU::TBA:
2707   case AMDGPU::TBA_LO:
2708   case AMDGPU::TBA_HI:
2709   case AMDGPU::TMA:
2710   case AMDGPU::TMA_LO:
2711   case AMDGPU::TMA_HI:
2712     return !isGFX9();
2713   case AMDGPU::XNACK_MASK:
2714   case AMDGPU::XNACK_MASK_LO:
2715   case AMDGPU::XNACK_MASK_HI:
2716     return !isCI() && !isSI() && hasXNACK();
2717   default:
2718     break;
2719   }
2720 
2721   if (isCI())
2722     return true;
2723 
2724   if (isSI()) {
2725     // No flat_scr
2726     switch (RegNo) {
2727     case AMDGPU::FLAT_SCR:
2728     case AMDGPU::FLAT_SCR_LO:
2729     case AMDGPU::FLAT_SCR_HI:
2730       return false;
2731     default:
2732       return true;
2733     }
2734   }
2735 
2736   // VI only has 102 SGPRs, so make sure we aren't trying to use the 2 more that
2737   // SI/CI have.
2738   for (MCRegAliasIterator R(AMDGPU::SGPR102_SGPR103, &MRI, true);
2739        R.isValid(); ++R) {
2740     if (*R == RegNo)
2741       return false;
2742   }
2743 
2744   return true;
2745 }
2746 
2747 OperandMatchResultTy
2748 AMDGPUAsmParser::parseOperand(OperandVector &Operands, StringRef Mnemonic) {
2749   // Try to parse with a custom parser
2750   OperandMatchResultTy ResTy = MatchOperandParserImpl(Operands, Mnemonic);
2751 
2752   // If we successfully parsed the operand or if there as an error parsing,
2753   // we are done.
2754   //
2755   // If we are parsing after we reach EndOfStatement then this means we
2756   // are appending default values to the Operands list.  This is only done
2757   // by custom parser, so we shouldn't continue on to the generic parsing.
2758   if (ResTy == MatchOperand_Success || ResTy == MatchOperand_ParseFail ||
2759       getLexer().is(AsmToken::EndOfStatement))
2760     return ResTy;
2761 
2762   ResTy = parseRegOrImm(Operands);
2763 
2764   if (ResTy == MatchOperand_Success)
2765     return ResTy;
2766 
2767   const auto &Tok = Parser.getTok();
2768   SMLoc S = Tok.getLoc();
2769 
2770   const MCExpr *Expr = nullptr;
2771   if (!Parser.parseExpression(Expr)) {
2772     Operands.push_back(AMDGPUOperand::CreateExpr(this, Expr, S));
2773     return MatchOperand_Success;
2774   }
2775 
2776   // Possibly this is an instruction flag like 'gds'.
2777   if (Tok.getKind() == AsmToken::Identifier) {
2778     Operands.push_back(AMDGPUOperand::CreateToken(this, Tok.getString(), S));
2779     Parser.Lex();
2780     return MatchOperand_Success;
2781   }
2782 
2783   return MatchOperand_NoMatch;
2784 }
2785 
2786 StringRef AMDGPUAsmParser::parseMnemonicSuffix(StringRef Name) {
2787   // Clear any forced encodings from the previous instruction.
2788   setForcedEncodingSize(0);
2789   setForcedDPP(false);
2790   setForcedSDWA(false);
2791 
2792   if (Name.endswith("_e64")) {
2793     setForcedEncodingSize(64);
2794     return Name.substr(0, Name.size() - 4);
2795   } else if (Name.endswith("_e32")) {
2796     setForcedEncodingSize(32);
2797     return Name.substr(0, Name.size() - 4);
2798   } else if (Name.endswith("_dpp")) {
2799     setForcedDPP(true);
2800     return Name.substr(0, Name.size() - 4);
2801   } else if (Name.endswith("_sdwa")) {
2802     setForcedSDWA(true);
2803     return Name.substr(0, Name.size() - 5);
2804   }
2805   return Name;
2806 }
2807 
2808 bool AMDGPUAsmParser::ParseInstruction(ParseInstructionInfo &Info,
2809                                        StringRef Name,
2810                                        SMLoc NameLoc, OperandVector &Operands) {
2811   // Add the instruction mnemonic
2812   Name = parseMnemonicSuffix(Name);
2813   Operands.push_back(AMDGPUOperand::CreateToken(this, Name, NameLoc));
2814 
2815   while (!getLexer().is(AsmToken::EndOfStatement)) {
2816     OperandMatchResultTy Res = parseOperand(Operands, Name);
2817 
2818     // Eat the comma or space if there is one.
2819     if (getLexer().is(AsmToken::Comma))
2820       Parser.Lex();
2821 
2822     switch (Res) {
2823       case MatchOperand_Success: break;
2824       case MatchOperand_ParseFail:
2825         Error(getLexer().getLoc(), "failed parsing operand.");
2826         while (!getLexer().is(AsmToken::EndOfStatement)) {
2827           Parser.Lex();
2828         }
2829         return true;
2830       case MatchOperand_NoMatch:
2831         Error(getLexer().getLoc(), "not a valid operand.");
2832         while (!getLexer().is(AsmToken::EndOfStatement)) {
2833           Parser.Lex();
2834         }
2835         return true;
2836     }
2837   }
2838 
2839   return false;
2840 }
2841 
2842 //===----------------------------------------------------------------------===//
2843 // Utility functions
2844 //===----------------------------------------------------------------------===//
2845 
2846 OperandMatchResultTy
2847 AMDGPUAsmParser::parseIntWithPrefix(const char *Prefix, int64_t &Int) {
2848   switch(getLexer().getKind()) {
2849     default: return MatchOperand_NoMatch;
2850     case AsmToken::Identifier: {
2851       StringRef Name = Parser.getTok().getString();
2852       if (!Name.equals(Prefix)) {
2853         return MatchOperand_NoMatch;
2854       }
2855 
2856       Parser.Lex();
2857       if (getLexer().isNot(AsmToken::Colon))
2858         return MatchOperand_ParseFail;
2859 
2860       Parser.Lex();
2861 
2862       bool IsMinus = false;
2863       if (getLexer().getKind() == AsmToken::Minus) {
2864         Parser.Lex();
2865         IsMinus = true;
2866       }
2867 
2868       if (getLexer().isNot(AsmToken::Integer))
2869         return MatchOperand_ParseFail;
2870 
2871       if (getParser().parseAbsoluteExpression(Int))
2872         return MatchOperand_ParseFail;
2873 
2874       if (IsMinus)
2875         Int = -Int;
2876       break;
2877     }
2878   }
2879   return MatchOperand_Success;
2880 }
2881 
2882 OperandMatchResultTy
2883 AMDGPUAsmParser::parseIntWithPrefix(const char *Prefix, OperandVector &Operands,
2884                                     AMDGPUOperand::ImmTy ImmTy,
2885                                     bool (*ConvertResult)(int64_t&)) {
2886   SMLoc S = Parser.getTok().getLoc();
2887   int64_t Value = 0;
2888 
2889   OperandMatchResultTy Res = parseIntWithPrefix(Prefix, Value);
2890   if (Res != MatchOperand_Success)
2891     return Res;
2892 
2893   if (ConvertResult && !ConvertResult(Value)) {
2894     return MatchOperand_ParseFail;
2895   }
2896 
2897   Operands.push_back(AMDGPUOperand::CreateImm(this, Value, S, ImmTy));
2898   return MatchOperand_Success;
2899 }
2900 
2901 OperandMatchResultTy AMDGPUAsmParser::parseOperandArrayWithPrefix(
2902   const char *Prefix,
2903   OperandVector &Operands,
2904   AMDGPUOperand::ImmTy ImmTy,
2905   bool (*ConvertResult)(int64_t&)) {
2906   StringRef Name = Parser.getTok().getString();
2907   if (!Name.equals(Prefix))
2908     return MatchOperand_NoMatch;
2909 
2910   Parser.Lex();
2911   if (getLexer().isNot(AsmToken::Colon))
2912     return MatchOperand_ParseFail;
2913 
2914   Parser.Lex();
2915   if (getLexer().isNot(AsmToken::LBrac))
2916     return MatchOperand_ParseFail;
2917   Parser.Lex();
2918 
2919   unsigned Val = 0;
2920   SMLoc S = Parser.getTok().getLoc();
2921 
2922   // FIXME: How to verify the number of elements matches the number of src
2923   // operands?
2924   for (int I = 0; I < 4; ++I) {
2925     if (I != 0) {
2926       if (getLexer().is(AsmToken::RBrac))
2927         break;
2928 
2929       if (getLexer().isNot(AsmToken::Comma))
2930         return MatchOperand_ParseFail;
2931       Parser.Lex();
2932     }
2933 
2934     if (getLexer().isNot(AsmToken::Integer))
2935       return MatchOperand_ParseFail;
2936 
2937     int64_t Op;
2938     if (getParser().parseAbsoluteExpression(Op))
2939       return MatchOperand_ParseFail;
2940 
2941     if (Op != 0 && Op != 1)
2942       return MatchOperand_ParseFail;
2943     Val |= (Op << I);
2944   }
2945 
2946   Parser.Lex();
2947   Operands.push_back(AMDGPUOperand::CreateImm(this, Val, S, ImmTy));
2948   return MatchOperand_Success;
2949 }
2950 
2951 OperandMatchResultTy
2952 AMDGPUAsmParser::parseNamedBit(const char *Name, OperandVector &Operands,
2953                                AMDGPUOperand::ImmTy ImmTy) {
2954   int64_t Bit = 0;
2955   SMLoc S = Parser.getTok().getLoc();
2956 
2957   // We are at the end of the statement, and this is a default argument, so
2958   // use a default value.
2959   if (getLexer().isNot(AsmToken::EndOfStatement)) {
2960     switch(getLexer().getKind()) {
2961       case AsmToken::Identifier: {
2962         StringRef Tok = Parser.getTok().getString();
2963         if (Tok == Name) {
2964           Bit = 1;
2965           Parser.Lex();
2966         } else if (Tok.startswith("no") && Tok.endswith(Name)) {
2967           Bit = 0;
2968           Parser.Lex();
2969         } else {
2970           return MatchOperand_NoMatch;
2971         }
2972         break;
2973       }
2974       default:
2975         return MatchOperand_NoMatch;
2976     }
2977   }
2978 
2979   Operands.push_back(AMDGPUOperand::CreateImm(this, Bit, S, ImmTy));
2980   return MatchOperand_Success;
2981 }
2982 
2983 static void addOptionalImmOperand(
2984   MCInst& Inst, const OperandVector& Operands,
2985   AMDGPUAsmParser::OptionalImmIndexMap& OptionalIdx,
2986   AMDGPUOperand::ImmTy ImmT,
2987   int64_t Default = 0) {
2988   auto i = OptionalIdx.find(ImmT);
2989   if (i != OptionalIdx.end()) {
2990     unsigned Idx = i->second;
2991     ((AMDGPUOperand &)*Operands[Idx]).addImmOperands(Inst, 1);
2992   } else {
2993     Inst.addOperand(MCOperand::createImm(Default));
2994   }
2995 }
2996 
2997 OperandMatchResultTy
2998 AMDGPUAsmParser::parseStringWithPrefix(StringRef Prefix, StringRef &Value) {
2999   if (getLexer().isNot(AsmToken::Identifier)) {
3000     return MatchOperand_NoMatch;
3001   }
3002   StringRef Tok = Parser.getTok().getString();
3003   if (Tok != Prefix) {
3004     return MatchOperand_NoMatch;
3005   }
3006 
3007   Parser.Lex();
3008   if (getLexer().isNot(AsmToken::Colon)) {
3009     return MatchOperand_ParseFail;
3010   }
3011 
3012   Parser.Lex();
3013   if (getLexer().isNot(AsmToken::Identifier)) {
3014     return MatchOperand_ParseFail;
3015   }
3016 
3017   Value = Parser.getTok().getString();
3018   return MatchOperand_Success;
3019 }
3020 
3021 //===----------------------------------------------------------------------===//
3022 // ds
3023 //===----------------------------------------------------------------------===//
3024 
3025 void AMDGPUAsmParser::cvtDSOffset01(MCInst &Inst,
3026                                     const OperandVector &Operands) {
3027   OptionalImmIndexMap OptionalIdx;
3028 
3029   for (unsigned i = 1, e = Operands.size(); i != e; ++i) {
3030     AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[i]);
3031 
3032     // Add the register arguments
3033     if (Op.isReg()) {
3034       Op.addRegOperands(Inst, 1);
3035       continue;
3036     }
3037 
3038     // Handle optional arguments
3039     OptionalIdx[Op.getImmTy()] = i;
3040   }
3041 
3042   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyOffset0);
3043   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyOffset1);
3044   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyGDS);
3045 
3046   Inst.addOperand(MCOperand::createReg(AMDGPU::M0)); // m0
3047 }
3048 
3049 void AMDGPUAsmParser::cvtDSImpl(MCInst &Inst, const OperandVector &Operands,
3050                                 bool IsGdsHardcoded) {
3051   OptionalImmIndexMap OptionalIdx;
3052 
3053   for (unsigned i = 1, e = Operands.size(); i != e; ++i) {
3054     AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[i]);
3055 
3056     // Add the register arguments
3057     if (Op.isReg()) {
3058       Op.addRegOperands(Inst, 1);
3059       continue;
3060     }
3061 
3062     if (Op.isToken() && Op.getToken() == "gds") {
3063       IsGdsHardcoded = true;
3064       continue;
3065     }
3066 
3067     // Handle optional arguments
3068     OptionalIdx[Op.getImmTy()] = i;
3069   }
3070 
3071   AMDGPUOperand::ImmTy OffsetType =
3072     (Inst.getOpcode() == AMDGPU::DS_SWIZZLE_B32_si ||
3073      Inst.getOpcode() == AMDGPU::DS_SWIZZLE_B32_vi) ? AMDGPUOperand::ImmTySwizzle :
3074                                                       AMDGPUOperand::ImmTyOffset;
3075 
3076   addOptionalImmOperand(Inst, Operands, OptionalIdx, OffsetType);
3077 
3078   if (!IsGdsHardcoded) {
3079     addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyGDS);
3080   }
3081   Inst.addOperand(MCOperand::createReg(AMDGPU::M0)); // m0
3082 }
3083 
3084 void AMDGPUAsmParser::cvtExp(MCInst &Inst, const OperandVector &Operands) {
3085   OptionalImmIndexMap OptionalIdx;
3086 
3087   unsigned OperandIdx[4];
3088   unsigned EnMask = 0;
3089   int SrcIdx = 0;
3090 
3091   for (unsigned i = 1, e = Operands.size(); i != e; ++i) {
3092     AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[i]);
3093 
3094     // Add the register arguments
3095     if (Op.isReg()) {
3096       assert(SrcIdx < 4);
3097       OperandIdx[SrcIdx] = Inst.size();
3098       Op.addRegOperands(Inst, 1);
3099       ++SrcIdx;
3100       continue;
3101     }
3102 
3103     if (Op.isOff()) {
3104       assert(SrcIdx < 4);
3105       OperandIdx[SrcIdx] = Inst.size();
3106       Inst.addOperand(MCOperand::createReg(AMDGPU::NoRegister));
3107       ++SrcIdx;
3108       continue;
3109     }
3110 
3111     if (Op.isImm() && Op.getImmTy() == AMDGPUOperand::ImmTyExpTgt) {
3112       Op.addImmOperands(Inst, 1);
3113       continue;
3114     }
3115 
3116     if (Op.isToken() && Op.getToken() == "done")
3117       continue;
3118 
3119     // Handle optional arguments
3120     OptionalIdx[Op.getImmTy()] = i;
3121   }
3122 
3123   assert(SrcIdx == 4);
3124 
3125   bool Compr = false;
3126   if (OptionalIdx.find(AMDGPUOperand::ImmTyExpCompr) != OptionalIdx.end()) {
3127     Compr = true;
3128     Inst.getOperand(OperandIdx[1]) = Inst.getOperand(OperandIdx[2]);
3129     Inst.getOperand(OperandIdx[2]).setReg(AMDGPU::NoRegister);
3130     Inst.getOperand(OperandIdx[3]).setReg(AMDGPU::NoRegister);
3131   }
3132 
3133   for (auto i = 0; i < SrcIdx; ++i) {
3134     if (Inst.getOperand(OperandIdx[i]).getReg() != AMDGPU::NoRegister) {
3135       EnMask |= Compr? (0x3 << i * 2) : (0x1 << i);
3136     }
3137   }
3138 
3139   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyExpVM);
3140   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyExpCompr);
3141 
3142   Inst.addOperand(MCOperand::createImm(EnMask));
3143 }
3144 
3145 //===----------------------------------------------------------------------===//
3146 // s_waitcnt
3147 //===----------------------------------------------------------------------===//
3148 
3149 static bool
3150 encodeCnt(
3151   const AMDGPU::IsaInfo::IsaVersion ISA,
3152   int64_t &IntVal,
3153   int64_t CntVal,
3154   bool Saturate,
3155   unsigned (*encode)(const IsaInfo::IsaVersion &Version, unsigned, unsigned),
3156   unsigned (*decode)(const IsaInfo::IsaVersion &Version, unsigned))
3157 {
3158   bool Failed = false;
3159 
3160   IntVal = encode(ISA, IntVal, CntVal);
3161   if (CntVal != decode(ISA, IntVal)) {
3162     if (Saturate) {
3163       IntVal = encode(ISA, IntVal, -1);
3164     } else {
3165       Failed = true;
3166     }
3167   }
3168   return Failed;
3169 }
3170 
3171 bool AMDGPUAsmParser::parseCnt(int64_t &IntVal) {
3172   StringRef CntName = Parser.getTok().getString();
3173   int64_t CntVal;
3174 
3175   Parser.Lex();
3176   if (getLexer().isNot(AsmToken::LParen))
3177     return true;
3178 
3179   Parser.Lex();
3180   if (getLexer().isNot(AsmToken::Integer))
3181     return true;
3182 
3183   SMLoc ValLoc = Parser.getTok().getLoc();
3184   if (getParser().parseAbsoluteExpression(CntVal))
3185     return true;
3186 
3187   AMDGPU::IsaInfo::IsaVersion ISA =
3188       AMDGPU::IsaInfo::getIsaVersion(getFeatureBits());
3189 
3190   bool Failed = true;
3191   bool Sat = CntName.endswith("_sat");
3192 
3193   if (CntName == "vmcnt" || CntName == "vmcnt_sat") {
3194     Failed = encodeCnt(ISA, IntVal, CntVal, Sat, encodeVmcnt, decodeVmcnt);
3195   } else if (CntName == "expcnt" || CntName == "expcnt_sat") {
3196     Failed = encodeCnt(ISA, IntVal, CntVal, Sat, encodeExpcnt, decodeExpcnt);
3197   } else if (CntName == "lgkmcnt" || CntName == "lgkmcnt_sat") {
3198     Failed = encodeCnt(ISA, IntVal, CntVal, Sat, encodeLgkmcnt, decodeLgkmcnt);
3199   }
3200 
3201   if (Failed) {
3202     Error(ValLoc, "too large value for " + CntName);
3203     return true;
3204   }
3205 
3206   if (getLexer().isNot(AsmToken::RParen)) {
3207     return true;
3208   }
3209 
3210   Parser.Lex();
3211   if (getLexer().is(AsmToken::Amp) || getLexer().is(AsmToken::Comma)) {
3212     const AsmToken NextToken = getLexer().peekTok();
3213     if (NextToken.is(AsmToken::Identifier)) {
3214       Parser.Lex();
3215     }
3216   }
3217 
3218   return false;
3219 }
3220 
3221 OperandMatchResultTy
3222 AMDGPUAsmParser::parseSWaitCntOps(OperandVector &Operands) {
3223   AMDGPU::IsaInfo::IsaVersion ISA =
3224       AMDGPU::IsaInfo::getIsaVersion(getFeatureBits());
3225   int64_t Waitcnt = getWaitcntBitMask(ISA);
3226   SMLoc S = Parser.getTok().getLoc();
3227 
3228   switch(getLexer().getKind()) {
3229     default: return MatchOperand_ParseFail;
3230     case AsmToken::Integer:
3231       // The operand can be an integer value.
3232       if (getParser().parseAbsoluteExpression(Waitcnt))
3233         return MatchOperand_ParseFail;
3234       break;
3235 
3236     case AsmToken::Identifier:
3237       do {
3238         if (parseCnt(Waitcnt))
3239           return MatchOperand_ParseFail;
3240       } while(getLexer().isNot(AsmToken::EndOfStatement));
3241       break;
3242   }
3243   Operands.push_back(AMDGPUOperand::CreateImm(this, Waitcnt, S));
3244   return MatchOperand_Success;
3245 }
3246 
3247 bool AMDGPUAsmParser::parseHwregConstruct(OperandInfoTy &HwReg, int64_t &Offset,
3248                                           int64_t &Width) {
3249   using namespace llvm::AMDGPU::Hwreg;
3250 
3251   if (Parser.getTok().getString() != "hwreg")
3252     return true;
3253   Parser.Lex();
3254 
3255   if (getLexer().isNot(AsmToken::LParen))
3256     return true;
3257   Parser.Lex();
3258 
3259   if (getLexer().is(AsmToken::Identifier)) {
3260     HwReg.IsSymbolic = true;
3261     HwReg.Id = ID_UNKNOWN_;
3262     const StringRef tok = Parser.getTok().getString();
3263     int Last = ID_SYMBOLIC_LAST_;
3264     if (isSI() || isCI() || isVI())
3265       Last = ID_SYMBOLIC_FIRST_GFX9_;
3266     for (int i = ID_SYMBOLIC_FIRST_; i < Last; ++i) {
3267       if (tok == IdSymbolic[i]) {
3268         HwReg.Id = i;
3269         break;
3270       }
3271     }
3272     Parser.Lex();
3273   } else {
3274     HwReg.IsSymbolic = false;
3275     if (getLexer().isNot(AsmToken::Integer))
3276       return true;
3277     if (getParser().parseAbsoluteExpression(HwReg.Id))
3278       return true;
3279   }
3280 
3281   if (getLexer().is(AsmToken::RParen)) {
3282     Parser.Lex();
3283     return false;
3284   }
3285 
3286   // optional params
3287   if (getLexer().isNot(AsmToken::Comma))
3288     return true;
3289   Parser.Lex();
3290 
3291   if (getLexer().isNot(AsmToken::Integer))
3292     return true;
3293   if (getParser().parseAbsoluteExpression(Offset))
3294     return true;
3295 
3296   if (getLexer().isNot(AsmToken::Comma))
3297     return true;
3298   Parser.Lex();
3299 
3300   if (getLexer().isNot(AsmToken::Integer))
3301     return true;
3302   if (getParser().parseAbsoluteExpression(Width))
3303     return true;
3304 
3305   if (getLexer().isNot(AsmToken::RParen))
3306     return true;
3307   Parser.Lex();
3308 
3309   return false;
3310 }
3311 
3312 OperandMatchResultTy AMDGPUAsmParser::parseHwreg(OperandVector &Operands) {
3313   using namespace llvm::AMDGPU::Hwreg;
3314 
3315   int64_t Imm16Val = 0;
3316   SMLoc S = Parser.getTok().getLoc();
3317 
3318   switch(getLexer().getKind()) {
3319     default: return MatchOperand_NoMatch;
3320     case AsmToken::Integer:
3321       // The operand can be an integer value.
3322       if (getParser().parseAbsoluteExpression(Imm16Val))
3323         return MatchOperand_NoMatch;
3324       if (Imm16Val < 0 || !isUInt<16>(Imm16Val)) {
3325         Error(S, "invalid immediate: only 16-bit values are legal");
3326         // Do not return error code, but create an imm operand anyway and proceed
3327         // to the next operand, if any. That avoids unneccessary error messages.
3328       }
3329       break;
3330 
3331     case AsmToken::Identifier: {
3332         OperandInfoTy HwReg(ID_UNKNOWN_);
3333         int64_t Offset = OFFSET_DEFAULT_;
3334         int64_t Width = WIDTH_M1_DEFAULT_ + 1;
3335         if (parseHwregConstruct(HwReg, Offset, Width))
3336           return MatchOperand_ParseFail;
3337         if (HwReg.Id < 0 || !isUInt<ID_WIDTH_>(HwReg.Id)) {
3338           if (HwReg.IsSymbolic)
3339             Error(S, "invalid symbolic name of hardware register");
3340           else
3341             Error(S, "invalid code of hardware register: only 6-bit values are legal");
3342         }
3343         if (Offset < 0 || !isUInt<OFFSET_WIDTH_>(Offset))
3344           Error(S, "invalid bit offset: only 5-bit values are legal");
3345         if ((Width-1) < 0 || !isUInt<WIDTH_M1_WIDTH_>(Width-1))
3346           Error(S, "invalid bitfield width: only values from 1 to 32 are legal");
3347         Imm16Val = (HwReg.Id << ID_SHIFT_) | (Offset << OFFSET_SHIFT_) | ((Width-1) << WIDTH_M1_SHIFT_);
3348       }
3349       break;
3350   }
3351   Operands.push_back(AMDGPUOperand::CreateImm(this, Imm16Val, S, AMDGPUOperand::ImmTyHwreg));
3352   return MatchOperand_Success;
3353 }
3354 
3355 bool AMDGPUOperand::isSWaitCnt() const {
3356   return isImm();
3357 }
3358 
3359 bool AMDGPUOperand::isHwreg() const {
3360   return isImmTy(ImmTyHwreg);
3361 }
3362 
3363 bool AMDGPUAsmParser::parseSendMsgConstruct(OperandInfoTy &Msg, OperandInfoTy &Operation, int64_t &StreamId) {
3364   using namespace llvm::AMDGPU::SendMsg;
3365 
3366   if (Parser.getTok().getString() != "sendmsg")
3367     return true;
3368   Parser.Lex();
3369 
3370   if (getLexer().isNot(AsmToken::LParen))
3371     return true;
3372   Parser.Lex();
3373 
3374   if (getLexer().is(AsmToken::Identifier)) {
3375     Msg.IsSymbolic = true;
3376     Msg.Id = ID_UNKNOWN_;
3377     const std::string tok = Parser.getTok().getString();
3378     for (int i = ID_GAPS_FIRST_; i < ID_GAPS_LAST_; ++i) {
3379       switch(i) {
3380         default: continue; // Omit gaps.
3381         case ID_INTERRUPT: case ID_GS: case ID_GS_DONE:  case ID_SYSMSG: break;
3382       }
3383       if (tok == IdSymbolic[i]) {
3384         Msg.Id = i;
3385         break;
3386       }
3387     }
3388     Parser.Lex();
3389   } else {
3390     Msg.IsSymbolic = false;
3391     if (getLexer().isNot(AsmToken::Integer))
3392       return true;
3393     if (getParser().parseAbsoluteExpression(Msg.Id))
3394       return true;
3395     if (getLexer().is(AsmToken::Integer))
3396       if (getParser().parseAbsoluteExpression(Msg.Id))
3397         Msg.Id = ID_UNKNOWN_;
3398   }
3399   if (Msg.Id == ID_UNKNOWN_) // Don't know how to parse the rest.
3400     return false;
3401 
3402   if (!(Msg.Id == ID_GS || Msg.Id == ID_GS_DONE || Msg.Id == ID_SYSMSG)) {
3403     if (getLexer().isNot(AsmToken::RParen))
3404       return true;
3405     Parser.Lex();
3406     return false;
3407   }
3408 
3409   if (getLexer().isNot(AsmToken::Comma))
3410     return true;
3411   Parser.Lex();
3412 
3413   assert(Msg.Id == ID_GS || Msg.Id == ID_GS_DONE || Msg.Id == ID_SYSMSG);
3414   Operation.Id = ID_UNKNOWN_;
3415   if (getLexer().is(AsmToken::Identifier)) {
3416     Operation.IsSymbolic = true;
3417     const char* const *S = (Msg.Id == ID_SYSMSG) ? OpSysSymbolic : OpGsSymbolic;
3418     const int F = (Msg.Id == ID_SYSMSG) ? OP_SYS_FIRST_ : OP_GS_FIRST_;
3419     const int L = (Msg.Id == ID_SYSMSG) ? OP_SYS_LAST_ : OP_GS_LAST_;
3420     const StringRef Tok = Parser.getTok().getString();
3421     for (int i = F; i < L; ++i) {
3422       if (Tok == S[i]) {
3423         Operation.Id = i;
3424         break;
3425       }
3426     }
3427     Parser.Lex();
3428   } else {
3429     Operation.IsSymbolic = false;
3430     if (getLexer().isNot(AsmToken::Integer))
3431       return true;
3432     if (getParser().parseAbsoluteExpression(Operation.Id))
3433       return true;
3434   }
3435 
3436   if ((Msg.Id == ID_GS || Msg.Id == ID_GS_DONE) && Operation.Id != OP_GS_NOP) {
3437     // Stream id is optional.
3438     if (getLexer().is(AsmToken::RParen)) {
3439       Parser.Lex();
3440       return false;
3441     }
3442 
3443     if (getLexer().isNot(AsmToken::Comma))
3444       return true;
3445     Parser.Lex();
3446 
3447     if (getLexer().isNot(AsmToken::Integer))
3448       return true;
3449     if (getParser().parseAbsoluteExpression(StreamId))
3450       return true;
3451   }
3452 
3453   if (getLexer().isNot(AsmToken::RParen))
3454     return true;
3455   Parser.Lex();
3456   return false;
3457 }
3458 
3459 OperandMatchResultTy AMDGPUAsmParser::parseInterpSlot(OperandVector &Operands) {
3460   if (getLexer().getKind() != AsmToken::Identifier)
3461     return MatchOperand_NoMatch;
3462 
3463   StringRef Str = Parser.getTok().getString();
3464   int Slot = StringSwitch<int>(Str)
3465     .Case("p10", 0)
3466     .Case("p20", 1)
3467     .Case("p0", 2)
3468     .Default(-1);
3469 
3470   SMLoc S = Parser.getTok().getLoc();
3471   if (Slot == -1)
3472     return MatchOperand_ParseFail;
3473 
3474   Parser.Lex();
3475   Operands.push_back(AMDGPUOperand::CreateImm(this, Slot, S,
3476                                               AMDGPUOperand::ImmTyInterpSlot));
3477   return MatchOperand_Success;
3478 }
3479 
3480 OperandMatchResultTy AMDGPUAsmParser::parseInterpAttr(OperandVector &Operands) {
3481   if (getLexer().getKind() != AsmToken::Identifier)
3482     return MatchOperand_NoMatch;
3483 
3484   StringRef Str = Parser.getTok().getString();
3485   if (!Str.startswith("attr"))
3486     return MatchOperand_NoMatch;
3487 
3488   StringRef Chan = Str.take_back(2);
3489   int AttrChan = StringSwitch<int>(Chan)
3490     .Case(".x", 0)
3491     .Case(".y", 1)
3492     .Case(".z", 2)
3493     .Case(".w", 3)
3494     .Default(-1);
3495   if (AttrChan == -1)
3496     return MatchOperand_ParseFail;
3497 
3498   Str = Str.drop_back(2).drop_front(4);
3499 
3500   uint8_t Attr;
3501   if (Str.getAsInteger(10, Attr))
3502     return MatchOperand_ParseFail;
3503 
3504   SMLoc S = Parser.getTok().getLoc();
3505   Parser.Lex();
3506   if (Attr > 63) {
3507     Error(S, "out of bounds attr");
3508     return MatchOperand_Success;
3509   }
3510 
3511   SMLoc SChan = SMLoc::getFromPointer(Chan.data());
3512 
3513   Operands.push_back(AMDGPUOperand::CreateImm(this, Attr, S,
3514                                               AMDGPUOperand::ImmTyInterpAttr));
3515   Operands.push_back(AMDGPUOperand::CreateImm(this, AttrChan, SChan,
3516                                               AMDGPUOperand::ImmTyAttrChan));
3517   return MatchOperand_Success;
3518 }
3519 
3520 void AMDGPUAsmParser::errorExpTgt() {
3521   Error(Parser.getTok().getLoc(), "invalid exp target");
3522 }
3523 
3524 OperandMatchResultTy AMDGPUAsmParser::parseExpTgtImpl(StringRef Str,
3525                                                       uint8_t &Val) {
3526   if (Str == "null") {
3527     Val = 9;
3528     return MatchOperand_Success;
3529   }
3530 
3531   if (Str.startswith("mrt")) {
3532     Str = Str.drop_front(3);
3533     if (Str == "z") { // == mrtz
3534       Val = 8;
3535       return MatchOperand_Success;
3536     }
3537 
3538     if (Str.getAsInteger(10, Val))
3539       return MatchOperand_ParseFail;
3540 
3541     if (Val > 7)
3542       errorExpTgt();
3543 
3544     return MatchOperand_Success;
3545   }
3546 
3547   if (Str.startswith("pos")) {
3548     Str = Str.drop_front(3);
3549     if (Str.getAsInteger(10, Val))
3550       return MatchOperand_ParseFail;
3551 
3552     if (Val > 3)
3553       errorExpTgt();
3554 
3555     Val += 12;
3556     return MatchOperand_Success;
3557   }
3558 
3559   if (Str.startswith("param")) {
3560     Str = Str.drop_front(5);
3561     if (Str.getAsInteger(10, Val))
3562       return MatchOperand_ParseFail;
3563 
3564     if (Val >= 32)
3565       errorExpTgt();
3566 
3567     Val += 32;
3568     return MatchOperand_Success;
3569   }
3570 
3571   if (Str.startswith("invalid_target_")) {
3572     Str = Str.drop_front(15);
3573     if (Str.getAsInteger(10, Val))
3574       return MatchOperand_ParseFail;
3575 
3576     errorExpTgt();
3577     return MatchOperand_Success;
3578   }
3579 
3580   return MatchOperand_NoMatch;
3581 }
3582 
3583 OperandMatchResultTy AMDGPUAsmParser::parseExpTgt(OperandVector &Operands) {
3584   uint8_t Val;
3585   StringRef Str = Parser.getTok().getString();
3586 
3587   auto Res = parseExpTgtImpl(Str, Val);
3588   if (Res != MatchOperand_Success)
3589     return Res;
3590 
3591   SMLoc S = Parser.getTok().getLoc();
3592   Parser.Lex();
3593 
3594   Operands.push_back(AMDGPUOperand::CreateImm(this, Val, S,
3595                                               AMDGPUOperand::ImmTyExpTgt));
3596   return MatchOperand_Success;
3597 }
3598 
3599 OperandMatchResultTy
3600 AMDGPUAsmParser::parseSendMsgOp(OperandVector &Operands) {
3601   using namespace llvm::AMDGPU::SendMsg;
3602 
3603   int64_t Imm16Val = 0;
3604   SMLoc S = Parser.getTok().getLoc();
3605 
3606   switch(getLexer().getKind()) {
3607   default:
3608     return MatchOperand_NoMatch;
3609   case AsmToken::Integer:
3610     // The operand can be an integer value.
3611     if (getParser().parseAbsoluteExpression(Imm16Val))
3612       return MatchOperand_NoMatch;
3613     if (Imm16Val < 0 || !isUInt<16>(Imm16Val)) {
3614       Error(S, "invalid immediate: only 16-bit values are legal");
3615       // Do not return error code, but create an imm operand anyway and proceed
3616       // to the next operand, if any. That avoids unneccessary error messages.
3617     }
3618     break;
3619   case AsmToken::Identifier: {
3620       OperandInfoTy Msg(ID_UNKNOWN_);
3621       OperandInfoTy Operation(OP_UNKNOWN_);
3622       int64_t StreamId = STREAM_ID_DEFAULT_;
3623       if (parseSendMsgConstruct(Msg, Operation, StreamId))
3624         return MatchOperand_ParseFail;
3625       do {
3626         // Validate and encode message ID.
3627         if (! ((ID_INTERRUPT <= Msg.Id && Msg.Id <= ID_GS_DONE)
3628                 || Msg.Id == ID_SYSMSG)) {
3629           if (Msg.IsSymbolic)
3630             Error(S, "invalid/unsupported symbolic name of message");
3631           else
3632             Error(S, "invalid/unsupported code of message");
3633           break;
3634         }
3635         Imm16Val = (Msg.Id << ID_SHIFT_);
3636         // Validate and encode operation ID.
3637         if (Msg.Id == ID_GS || Msg.Id == ID_GS_DONE) {
3638           if (! (OP_GS_FIRST_ <= Operation.Id && Operation.Id < OP_GS_LAST_)) {
3639             if (Operation.IsSymbolic)
3640               Error(S, "invalid symbolic name of GS_OP");
3641             else
3642               Error(S, "invalid code of GS_OP: only 2-bit values are legal");
3643             break;
3644           }
3645           if (Operation.Id == OP_GS_NOP
3646               && Msg.Id != ID_GS_DONE) {
3647             Error(S, "invalid GS_OP: NOP is for GS_DONE only");
3648             break;
3649           }
3650           Imm16Val |= (Operation.Id << OP_SHIFT_);
3651         }
3652         if (Msg.Id == ID_SYSMSG) {
3653           if (! (OP_SYS_FIRST_ <= Operation.Id && Operation.Id < OP_SYS_LAST_)) {
3654             if (Operation.IsSymbolic)
3655               Error(S, "invalid/unsupported symbolic name of SYSMSG_OP");
3656             else
3657               Error(S, "invalid/unsupported code of SYSMSG_OP");
3658             break;
3659           }
3660           Imm16Val |= (Operation.Id << OP_SHIFT_);
3661         }
3662         // Validate and encode stream ID.
3663         if ((Msg.Id == ID_GS || Msg.Id == ID_GS_DONE) && Operation.Id != OP_GS_NOP) {
3664           if (! (STREAM_ID_FIRST_ <= StreamId && StreamId < STREAM_ID_LAST_)) {
3665             Error(S, "invalid stream id: only 2-bit values are legal");
3666             break;
3667           }
3668           Imm16Val |= (StreamId << STREAM_ID_SHIFT_);
3669         }
3670       } while (false);
3671     }
3672     break;
3673   }
3674   Operands.push_back(AMDGPUOperand::CreateImm(this, Imm16Val, S, AMDGPUOperand::ImmTySendMsg));
3675   return MatchOperand_Success;
3676 }
3677 
3678 bool AMDGPUOperand::isSendMsg() const {
3679   return isImmTy(ImmTySendMsg);
3680 }
3681 
3682 //===----------------------------------------------------------------------===//
3683 // parser helpers
3684 //===----------------------------------------------------------------------===//
3685 
3686 bool
3687 AMDGPUAsmParser::trySkipId(const StringRef Id) {
3688   if (getLexer().getKind() == AsmToken::Identifier &&
3689       Parser.getTok().getString() == Id) {
3690     Parser.Lex();
3691     return true;
3692   }
3693   return false;
3694 }
3695 
3696 bool
3697 AMDGPUAsmParser::trySkipToken(const AsmToken::TokenKind Kind) {
3698   if (getLexer().getKind() == Kind) {
3699     Parser.Lex();
3700     return true;
3701   }
3702   return false;
3703 }
3704 
3705 bool
3706 AMDGPUAsmParser::skipToken(const AsmToken::TokenKind Kind,
3707                            const StringRef ErrMsg) {
3708   if (!trySkipToken(Kind)) {
3709     Error(Parser.getTok().getLoc(), ErrMsg);
3710     return false;
3711   }
3712   return true;
3713 }
3714 
3715 bool
3716 AMDGPUAsmParser::parseExpr(int64_t &Imm) {
3717   return !getParser().parseAbsoluteExpression(Imm);
3718 }
3719 
3720 bool
3721 AMDGPUAsmParser::parseString(StringRef &Val, const StringRef ErrMsg) {
3722   SMLoc S = Parser.getTok().getLoc();
3723   if (getLexer().getKind() == AsmToken::String) {
3724     Val = Parser.getTok().getStringContents();
3725     Parser.Lex();
3726     return true;
3727   } else {
3728     Error(S, ErrMsg);
3729     return false;
3730   }
3731 }
3732 
3733 //===----------------------------------------------------------------------===//
3734 // swizzle
3735 //===----------------------------------------------------------------------===//
3736 
3737 LLVM_READNONE
3738 static unsigned
3739 encodeBitmaskPerm(const unsigned AndMask,
3740                   const unsigned OrMask,
3741                   const unsigned XorMask) {
3742   using namespace llvm::AMDGPU::Swizzle;
3743 
3744   return BITMASK_PERM_ENC |
3745          (AndMask << BITMASK_AND_SHIFT) |
3746          (OrMask  << BITMASK_OR_SHIFT)  |
3747          (XorMask << BITMASK_XOR_SHIFT);
3748 }
3749 
3750 bool
3751 AMDGPUAsmParser::parseSwizzleOperands(const unsigned OpNum, int64_t* Op,
3752                                       const unsigned MinVal,
3753                                       const unsigned MaxVal,
3754                                       const StringRef ErrMsg) {
3755   for (unsigned i = 0; i < OpNum; ++i) {
3756     if (!skipToken(AsmToken::Comma, "expected a comma")){
3757       return false;
3758     }
3759     SMLoc ExprLoc = Parser.getTok().getLoc();
3760     if (!parseExpr(Op[i])) {
3761       return false;
3762     }
3763     if (Op[i] < MinVal || Op[i] > MaxVal) {
3764       Error(ExprLoc, ErrMsg);
3765       return false;
3766     }
3767   }
3768 
3769   return true;
3770 }
3771 
3772 bool
3773 AMDGPUAsmParser::parseSwizzleQuadPerm(int64_t &Imm) {
3774   using namespace llvm::AMDGPU::Swizzle;
3775 
3776   int64_t Lane[LANE_NUM];
3777   if (parseSwizzleOperands(LANE_NUM, Lane, 0, LANE_MAX,
3778                            "expected a 2-bit lane id")) {
3779     Imm = QUAD_PERM_ENC;
3780     for (auto i = 0; i < LANE_NUM; ++i) {
3781       Imm |= Lane[i] << (LANE_SHIFT * i);
3782     }
3783     return true;
3784   }
3785   return false;
3786 }
3787 
3788 bool
3789 AMDGPUAsmParser::parseSwizzleBroadcast(int64_t &Imm) {
3790   using namespace llvm::AMDGPU::Swizzle;
3791 
3792   SMLoc S = Parser.getTok().getLoc();
3793   int64_t GroupSize;
3794   int64_t LaneIdx;
3795 
3796   if (!parseSwizzleOperands(1, &GroupSize,
3797                             2, 32,
3798                             "group size must be in the interval [2,32]")) {
3799     return false;
3800   }
3801   if (!isPowerOf2_64(GroupSize)) {
3802     Error(S, "group size must be a power of two");
3803     return false;
3804   }
3805   if (parseSwizzleOperands(1, &LaneIdx,
3806                            0, GroupSize - 1,
3807                            "lane id must be in the interval [0,group size - 1]")) {
3808     Imm = encodeBitmaskPerm(BITMASK_MAX - GroupSize + 1, LaneIdx, 0);
3809     return true;
3810   }
3811   return false;
3812 }
3813 
3814 bool
3815 AMDGPUAsmParser::parseSwizzleReverse(int64_t &Imm) {
3816   using namespace llvm::AMDGPU::Swizzle;
3817 
3818   SMLoc S = Parser.getTok().getLoc();
3819   int64_t GroupSize;
3820 
3821   if (!parseSwizzleOperands(1, &GroupSize,
3822       2, 32, "group size must be in the interval [2,32]")) {
3823     return false;
3824   }
3825   if (!isPowerOf2_64(GroupSize)) {
3826     Error(S, "group size must be a power of two");
3827     return false;
3828   }
3829 
3830   Imm = encodeBitmaskPerm(BITMASK_MAX, 0, GroupSize - 1);
3831   return true;
3832 }
3833 
3834 bool
3835 AMDGPUAsmParser::parseSwizzleSwap(int64_t &Imm) {
3836   using namespace llvm::AMDGPU::Swizzle;
3837 
3838   SMLoc S = Parser.getTok().getLoc();
3839   int64_t GroupSize;
3840 
3841   if (!parseSwizzleOperands(1, &GroupSize,
3842       1, 16, "group size must be in the interval [1,16]")) {
3843     return false;
3844   }
3845   if (!isPowerOf2_64(GroupSize)) {
3846     Error(S, "group size must be a power of two");
3847     return false;
3848   }
3849 
3850   Imm = encodeBitmaskPerm(BITMASK_MAX, 0, GroupSize);
3851   return true;
3852 }
3853 
3854 bool
3855 AMDGPUAsmParser::parseSwizzleBitmaskPerm(int64_t &Imm) {
3856   using namespace llvm::AMDGPU::Swizzle;
3857 
3858   if (!skipToken(AsmToken::Comma, "expected a comma")) {
3859     return false;
3860   }
3861 
3862   StringRef Ctl;
3863   SMLoc StrLoc = Parser.getTok().getLoc();
3864   if (!parseString(Ctl)) {
3865     return false;
3866   }
3867   if (Ctl.size() != BITMASK_WIDTH) {
3868     Error(StrLoc, "expected a 5-character mask");
3869     return false;
3870   }
3871 
3872   unsigned AndMask = 0;
3873   unsigned OrMask = 0;
3874   unsigned XorMask = 0;
3875 
3876   for (size_t i = 0; i < Ctl.size(); ++i) {
3877     unsigned Mask = 1 << (BITMASK_WIDTH - 1 - i);
3878     switch(Ctl[i]) {
3879     default:
3880       Error(StrLoc, "invalid mask");
3881       return false;
3882     case '0':
3883       break;
3884     case '1':
3885       OrMask |= Mask;
3886       break;
3887     case 'p':
3888       AndMask |= Mask;
3889       break;
3890     case 'i':
3891       AndMask |= Mask;
3892       XorMask |= Mask;
3893       break;
3894     }
3895   }
3896 
3897   Imm = encodeBitmaskPerm(AndMask, OrMask, XorMask);
3898   return true;
3899 }
3900 
3901 bool
3902 AMDGPUAsmParser::parseSwizzleOffset(int64_t &Imm) {
3903 
3904   SMLoc OffsetLoc = Parser.getTok().getLoc();
3905 
3906   if (!parseExpr(Imm)) {
3907     return false;
3908   }
3909   if (!isUInt<16>(Imm)) {
3910     Error(OffsetLoc, "expected a 16-bit offset");
3911     return false;
3912   }
3913   return true;
3914 }
3915 
3916 bool
3917 AMDGPUAsmParser::parseSwizzleMacro(int64_t &Imm) {
3918   using namespace llvm::AMDGPU::Swizzle;
3919 
3920   if (skipToken(AsmToken::LParen, "expected a left parentheses")) {
3921 
3922     SMLoc ModeLoc = Parser.getTok().getLoc();
3923     bool Ok = false;
3924 
3925     if (trySkipId(IdSymbolic[ID_QUAD_PERM])) {
3926       Ok = parseSwizzleQuadPerm(Imm);
3927     } else if (trySkipId(IdSymbolic[ID_BITMASK_PERM])) {
3928       Ok = parseSwizzleBitmaskPerm(Imm);
3929     } else if (trySkipId(IdSymbolic[ID_BROADCAST])) {
3930       Ok = parseSwizzleBroadcast(Imm);
3931     } else if (trySkipId(IdSymbolic[ID_SWAP])) {
3932       Ok = parseSwizzleSwap(Imm);
3933     } else if (trySkipId(IdSymbolic[ID_REVERSE])) {
3934       Ok = parseSwizzleReverse(Imm);
3935     } else {
3936       Error(ModeLoc, "expected a swizzle mode");
3937     }
3938 
3939     return Ok && skipToken(AsmToken::RParen, "expected a closing parentheses");
3940   }
3941 
3942   return false;
3943 }
3944 
3945 OperandMatchResultTy
3946 AMDGPUAsmParser::parseSwizzleOp(OperandVector &Operands) {
3947   SMLoc S = Parser.getTok().getLoc();
3948   int64_t Imm = 0;
3949 
3950   if (trySkipId("offset")) {
3951 
3952     bool Ok = false;
3953     if (skipToken(AsmToken::Colon, "expected a colon")) {
3954       if (trySkipId("swizzle")) {
3955         Ok = parseSwizzleMacro(Imm);
3956       } else {
3957         Ok = parseSwizzleOffset(Imm);
3958       }
3959     }
3960 
3961     Operands.push_back(AMDGPUOperand::CreateImm(this, Imm, S, AMDGPUOperand::ImmTySwizzle));
3962 
3963     return Ok? MatchOperand_Success : MatchOperand_ParseFail;
3964   } else {
3965     // Swizzle "offset" operand is optional.
3966     // If it is omitted, try parsing other optional operands.
3967     return parseOptionalOpr(Operands);
3968   }
3969 }
3970 
3971 bool
3972 AMDGPUOperand::isSwizzle() const {
3973   return isImmTy(ImmTySwizzle);
3974 }
3975 
3976 //===----------------------------------------------------------------------===//
3977 // sopp branch targets
3978 //===----------------------------------------------------------------------===//
3979 
3980 OperandMatchResultTy
3981 AMDGPUAsmParser::parseSOppBrTarget(OperandVector &Operands) {
3982   SMLoc S = Parser.getTok().getLoc();
3983 
3984   switch (getLexer().getKind()) {
3985     default: return MatchOperand_ParseFail;
3986     case AsmToken::Integer: {
3987       int64_t Imm;
3988       if (getParser().parseAbsoluteExpression(Imm))
3989         return MatchOperand_ParseFail;
3990       Operands.push_back(AMDGPUOperand::CreateImm(this, Imm, S));
3991       return MatchOperand_Success;
3992     }
3993 
3994     case AsmToken::Identifier:
3995       Operands.push_back(AMDGPUOperand::CreateExpr(this,
3996           MCSymbolRefExpr::create(getContext().getOrCreateSymbol(
3997                                   Parser.getTok().getString()), getContext()), S));
3998       Parser.Lex();
3999       return MatchOperand_Success;
4000   }
4001 }
4002 
4003 //===----------------------------------------------------------------------===//
4004 // mubuf
4005 //===----------------------------------------------------------------------===//
4006 
4007 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultGLC() const {
4008   return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTyGLC);
4009 }
4010 
4011 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultSLC() const {
4012   return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTySLC);
4013 }
4014 
4015 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultTFE() const {
4016   return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTyTFE);
4017 }
4018 
4019 void AMDGPUAsmParser::cvtMubufImpl(MCInst &Inst,
4020                                const OperandVector &Operands,
4021                                bool IsAtomic, bool IsAtomicReturn) {
4022   OptionalImmIndexMap OptionalIdx;
4023   assert(IsAtomicReturn ? IsAtomic : true);
4024 
4025   for (unsigned i = 1, e = Operands.size(); i != e; ++i) {
4026     AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[i]);
4027 
4028     // Add the register arguments
4029     if (Op.isReg()) {
4030       Op.addRegOperands(Inst, 1);
4031       continue;
4032     }
4033 
4034     // Handle the case where soffset is an immediate
4035     if (Op.isImm() && Op.getImmTy() == AMDGPUOperand::ImmTyNone) {
4036       Op.addImmOperands(Inst, 1);
4037       continue;
4038     }
4039 
4040     // Handle tokens like 'offen' which are sometimes hard-coded into the
4041     // asm string.  There are no MCInst operands for these.
4042     if (Op.isToken()) {
4043       continue;
4044     }
4045     assert(Op.isImm());
4046 
4047     // Handle optional arguments
4048     OptionalIdx[Op.getImmTy()] = i;
4049   }
4050 
4051   // Copy $vdata_in operand and insert as $vdata for MUBUF_Atomic RTN insns.
4052   if (IsAtomicReturn) {
4053     MCInst::iterator I = Inst.begin(); // $vdata_in is always at the beginning.
4054     Inst.insert(I, *I);
4055   }
4056 
4057   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyOffset);
4058   if (!IsAtomic) { // glc is hard-coded.
4059     addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyGLC);
4060   }
4061   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySLC);
4062   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyTFE);
4063 }
4064 
4065 void AMDGPUAsmParser::cvtMtbuf(MCInst &Inst, const OperandVector &Operands) {
4066   OptionalImmIndexMap OptionalIdx;
4067 
4068   for (unsigned i = 1, e = Operands.size(); i != e; ++i) {
4069     AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[i]);
4070 
4071     // Add the register arguments
4072     if (Op.isReg()) {
4073       Op.addRegOperands(Inst, 1);
4074       continue;
4075     }
4076 
4077     // Handle the case where soffset is an immediate
4078     if (Op.isImm() && Op.getImmTy() == AMDGPUOperand::ImmTyNone) {
4079       Op.addImmOperands(Inst, 1);
4080       continue;
4081     }
4082 
4083     // Handle tokens like 'offen' which are sometimes hard-coded into the
4084     // asm string.  There are no MCInst operands for these.
4085     if (Op.isToken()) {
4086       continue;
4087     }
4088     assert(Op.isImm());
4089 
4090     // Handle optional arguments
4091     OptionalIdx[Op.getImmTy()] = i;
4092   }
4093 
4094   addOptionalImmOperand(Inst, Operands, OptionalIdx,
4095                         AMDGPUOperand::ImmTyOffset);
4096   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyDFMT);
4097   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyNFMT);
4098   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyGLC);
4099   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySLC);
4100   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyTFE);
4101 }
4102 
4103 //===----------------------------------------------------------------------===//
4104 // mimg
4105 //===----------------------------------------------------------------------===//
4106 
4107 void AMDGPUAsmParser::cvtMIMG(MCInst &Inst, const OperandVector &Operands,
4108                               bool IsAtomic) {
4109   unsigned I = 1;
4110   const MCInstrDesc &Desc = MII.get(Inst.getOpcode());
4111   for (unsigned J = 0; J < Desc.getNumDefs(); ++J) {
4112     ((AMDGPUOperand &)*Operands[I++]).addRegOperands(Inst, 1);
4113   }
4114 
4115   if (IsAtomic) {
4116     // Add src, same as dst
4117     assert(Desc.getNumDefs() == 1);
4118     ((AMDGPUOperand &)*Operands[I - 1]).addRegOperands(Inst, 1);
4119   }
4120 
4121   OptionalImmIndexMap OptionalIdx;
4122 
4123   for (unsigned E = Operands.size(); I != E; ++I) {
4124     AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[I]);
4125 
4126     // Add the register arguments
4127     if (Op.isReg()) {
4128       Op.addRegOperands(Inst, 1);
4129     } else if (Op.isImmModifier()) {
4130       OptionalIdx[Op.getImmTy()] = I;
4131     } else {
4132       llvm_unreachable("unexpected operand type");
4133     }
4134   }
4135 
4136   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyDMask);
4137   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyUNorm);
4138   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyGLC);
4139   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySLC);
4140   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyR128);
4141   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyTFE);
4142   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyLWE);
4143   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyDA);
4144 }
4145 
4146 void AMDGPUAsmParser::cvtMIMGAtomic(MCInst &Inst, const OperandVector &Operands) {
4147   cvtMIMG(Inst, Operands, true);
4148 }
4149 
4150 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultDMask() const {
4151   return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTyDMask);
4152 }
4153 
4154 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultUNorm() const {
4155   return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTyUNorm);
4156 }
4157 
4158 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultDA() const {
4159   return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTyDA);
4160 }
4161 
4162 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultR128() const {
4163   return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTyR128);
4164 }
4165 
4166 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultLWE() const {
4167   return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTyLWE);
4168 }
4169 
4170 //===----------------------------------------------------------------------===//
4171 // smrd
4172 //===----------------------------------------------------------------------===//
4173 
4174 bool AMDGPUOperand::isSMRDOffset8() const {
4175   return isImm() && isUInt<8>(getImm());
4176 }
4177 
4178 bool AMDGPUOperand::isSMRDOffset20() const {
4179   return isImm() && isUInt<20>(getImm());
4180 }
4181 
4182 bool AMDGPUOperand::isSMRDLiteralOffset() const {
4183   // 32-bit literals are only supported on CI and we only want to use them
4184   // when the offset is > 8-bits.
4185   return isImm() && !isUInt<8>(getImm()) && isUInt<32>(getImm());
4186 }
4187 
4188 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultSMRDOffset8() const {
4189   return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTyOffset);
4190 }
4191 
4192 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultSMRDOffset20() const {
4193   return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTyOffset);
4194 }
4195 
4196 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultSMRDLiteralOffset() const {
4197   return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTyOffset);
4198 }
4199 
4200 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultOffsetU12() const {
4201   return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTyOffset);
4202 }
4203 
4204 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultOffsetS13() const {
4205   return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTyOffset);
4206 }
4207 
4208 //===----------------------------------------------------------------------===//
4209 // vop3
4210 //===----------------------------------------------------------------------===//
4211 
4212 static bool ConvertOmodMul(int64_t &Mul) {
4213   if (Mul != 1 && Mul != 2 && Mul != 4)
4214     return false;
4215 
4216   Mul >>= 1;
4217   return true;
4218 }
4219 
4220 static bool ConvertOmodDiv(int64_t &Div) {
4221   if (Div == 1) {
4222     Div = 0;
4223     return true;
4224   }
4225 
4226   if (Div == 2) {
4227     Div = 3;
4228     return true;
4229   }
4230 
4231   return false;
4232 }
4233 
4234 static bool ConvertBoundCtrl(int64_t &BoundCtrl) {
4235   if (BoundCtrl == 0) {
4236     BoundCtrl = 1;
4237     return true;
4238   }
4239 
4240   if (BoundCtrl == -1) {
4241     BoundCtrl = 0;
4242     return true;
4243   }
4244 
4245   return false;
4246 }
4247 
4248 // Note: the order in this table matches the order of operands in AsmString.
4249 static const OptionalOperand AMDGPUOptionalOperandTable[] = {
4250   {"offen",   AMDGPUOperand::ImmTyOffen, true, nullptr},
4251   {"idxen",   AMDGPUOperand::ImmTyIdxen, true, nullptr},
4252   {"addr64",  AMDGPUOperand::ImmTyAddr64, true, nullptr},
4253   {"offset0", AMDGPUOperand::ImmTyOffset0, false, nullptr},
4254   {"offset1", AMDGPUOperand::ImmTyOffset1, false, nullptr},
4255   {"gds",     AMDGPUOperand::ImmTyGDS, true, nullptr},
4256   {"offset",  AMDGPUOperand::ImmTyOffset, false, nullptr},
4257   {"inst_offset", AMDGPUOperand::ImmTyInstOffset, false, nullptr},
4258   {"dfmt",    AMDGPUOperand::ImmTyDFMT, false, nullptr},
4259   {"nfmt",    AMDGPUOperand::ImmTyNFMT, false, nullptr},
4260   {"glc",     AMDGPUOperand::ImmTyGLC, true, nullptr},
4261   {"slc",     AMDGPUOperand::ImmTySLC, true, nullptr},
4262   {"tfe",     AMDGPUOperand::ImmTyTFE, true, nullptr},
4263   {"high",    AMDGPUOperand::ImmTyHigh, true, nullptr},
4264   {"clamp",   AMDGPUOperand::ImmTyClampSI, true, nullptr},
4265   {"omod",    AMDGPUOperand::ImmTyOModSI, false, ConvertOmodMul},
4266   {"unorm",   AMDGPUOperand::ImmTyUNorm, true, nullptr},
4267   {"da",      AMDGPUOperand::ImmTyDA,    true, nullptr},
4268   {"r128",    AMDGPUOperand::ImmTyR128,  true, nullptr},
4269   {"lwe",     AMDGPUOperand::ImmTyLWE,   true, nullptr},
4270   {"dmask",   AMDGPUOperand::ImmTyDMask, false, nullptr},
4271   {"row_mask",   AMDGPUOperand::ImmTyDppRowMask, false, nullptr},
4272   {"bank_mask",  AMDGPUOperand::ImmTyDppBankMask, false, nullptr},
4273   {"bound_ctrl", AMDGPUOperand::ImmTyDppBoundCtrl, false, ConvertBoundCtrl},
4274   {"dst_sel",    AMDGPUOperand::ImmTySdwaDstSel, false, nullptr},
4275   {"src0_sel",   AMDGPUOperand::ImmTySdwaSrc0Sel, false, nullptr},
4276   {"src1_sel",   AMDGPUOperand::ImmTySdwaSrc1Sel, false, nullptr},
4277   {"dst_unused", AMDGPUOperand::ImmTySdwaDstUnused, false, nullptr},
4278   {"compr", AMDGPUOperand::ImmTyExpCompr, true, nullptr },
4279   {"vm", AMDGPUOperand::ImmTyExpVM, true, nullptr},
4280   {"op_sel", AMDGPUOperand::ImmTyOpSel, false, nullptr},
4281   {"op_sel_hi", AMDGPUOperand::ImmTyOpSelHi, false, nullptr},
4282   {"neg_lo", AMDGPUOperand::ImmTyNegLo, false, nullptr},
4283   {"neg_hi", AMDGPUOperand::ImmTyNegHi, false, nullptr}
4284 };
4285 
4286 OperandMatchResultTy AMDGPUAsmParser::parseOptionalOperand(OperandVector &Operands) {
4287   unsigned size = Operands.size();
4288   assert(size > 0);
4289 
4290   OperandMatchResultTy res = parseOptionalOpr(Operands);
4291 
4292   // This is a hack to enable hardcoded mandatory operands which follow
4293   // optional operands.
4294   //
4295   // Current design assumes that all operands after the first optional operand
4296   // are also optional. However implementation of some instructions violates
4297   // this rule (see e.g. flat/global atomic which have hardcoded 'glc' operands).
4298   //
4299   // To alleviate this problem, we have to (implicitly) parse extra operands
4300   // to make sure autogenerated parser of custom operands never hit hardcoded
4301   // mandatory operands.
4302 
4303   if (size == 1 || ((AMDGPUOperand &)*Operands[size - 1]).isRegKind()) {
4304 
4305     // We have parsed the first optional operand.
4306     // Parse as many operands as necessary to skip all mandatory operands.
4307 
4308     for (unsigned i = 0; i < MAX_OPR_LOOKAHEAD; ++i) {
4309       if (res != MatchOperand_Success ||
4310           getLexer().is(AsmToken::EndOfStatement)) break;
4311       if (getLexer().is(AsmToken::Comma)) Parser.Lex();
4312       res = parseOptionalOpr(Operands);
4313     }
4314   }
4315 
4316   return res;
4317 }
4318 
4319 OperandMatchResultTy AMDGPUAsmParser::parseOptionalOpr(OperandVector &Operands) {
4320   OperandMatchResultTy res;
4321   for (const OptionalOperand &Op : AMDGPUOptionalOperandTable) {
4322     // try to parse any optional operand here
4323     if (Op.IsBit) {
4324       res = parseNamedBit(Op.Name, Operands, Op.Type);
4325     } else if (Op.Type == AMDGPUOperand::ImmTyOModSI) {
4326       res = parseOModOperand(Operands);
4327     } else if (Op.Type == AMDGPUOperand::ImmTySdwaDstSel ||
4328                Op.Type == AMDGPUOperand::ImmTySdwaSrc0Sel ||
4329                Op.Type == AMDGPUOperand::ImmTySdwaSrc1Sel) {
4330       res = parseSDWASel(Operands, Op.Name, Op.Type);
4331     } else if (Op.Type == AMDGPUOperand::ImmTySdwaDstUnused) {
4332       res = parseSDWADstUnused(Operands);
4333     } else if (Op.Type == AMDGPUOperand::ImmTyOpSel ||
4334                Op.Type == AMDGPUOperand::ImmTyOpSelHi ||
4335                Op.Type == AMDGPUOperand::ImmTyNegLo ||
4336                Op.Type == AMDGPUOperand::ImmTyNegHi) {
4337       res = parseOperandArrayWithPrefix(Op.Name, Operands, Op.Type,
4338                                         Op.ConvertResult);
4339     } else {
4340       res = parseIntWithPrefix(Op.Name, Operands, Op.Type, Op.ConvertResult);
4341     }
4342     if (res != MatchOperand_NoMatch) {
4343       return res;
4344     }
4345   }
4346   return MatchOperand_NoMatch;
4347 }
4348 
4349 OperandMatchResultTy AMDGPUAsmParser::parseOModOperand(OperandVector &Operands) {
4350   StringRef Name = Parser.getTok().getString();
4351   if (Name == "mul") {
4352     return parseIntWithPrefix("mul", Operands,
4353                               AMDGPUOperand::ImmTyOModSI, ConvertOmodMul);
4354   }
4355 
4356   if (Name == "div") {
4357     return parseIntWithPrefix("div", Operands,
4358                               AMDGPUOperand::ImmTyOModSI, ConvertOmodDiv);
4359   }
4360 
4361   return MatchOperand_NoMatch;
4362 }
4363 
4364 void AMDGPUAsmParser::cvtVOP3OpSel(MCInst &Inst, const OperandVector &Operands) {
4365   cvtVOP3P(Inst, Operands);
4366 
4367   int Opc = Inst.getOpcode();
4368 
4369   int SrcNum;
4370   const int Ops[] = { AMDGPU::OpName::src0,
4371                       AMDGPU::OpName::src1,
4372                       AMDGPU::OpName::src2 };
4373   for (SrcNum = 0;
4374        SrcNum < 3 && AMDGPU::getNamedOperandIdx(Opc, Ops[SrcNum]) != -1;
4375        ++SrcNum);
4376   assert(SrcNum > 0);
4377 
4378   int OpSelIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::op_sel);
4379   unsigned OpSel = Inst.getOperand(OpSelIdx).getImm();
4380 
4381   if ((OpSel & (1 << SrcNum)) != 0) {
4382     int ModIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src0_modifiers);
4383     uint32_t ModVal = Inst.getOperand(ModIdx).getImm();
4384     Inst.getOperand(ModIdx).setImm(ModVal | SISrcMods::DST_OP_SEL);
4385   }
4386 }
4387 
4388 static bool isRegOrImmWithInputMods(const MCInstrDesc &Desc, unsigned OpNum) {
4389       // 1. This operand is input modifiers
4390   return Desc.OpInfo[OpNum].OperandType == AMDGPU::OPERAND_INPUT_MODS
4391       // 2. This is not last operand
4392       && Desc.NumOperands > (OpNum + 1)
4393       // 3. Next operand is register class
4394       && Desc.OpInfo[OpNum + 1].RegClass != -1
4395       // 4. Next register is not tied to any other operand
4396       && Desc.getOperandConstraint(OpNum + 1, MCOI::OperandConstraint::TIED_TO) == -1;
4397 }
4398 
4399 void AMDGPUAsmParser::cvtVOP3Interp(MCInst &Inst, const OperandVector &Operands)
4400 {
4401   OptionalImmIndexMap OptionalIdx;
4402   unsigned Opc = Inst.getOpcode();
4403 
4404   unsigned I = 1;
4405   const MCInstrDesc &Desc = MII.get(Inst.getOpcode());
4406   for (unsigned J = 0; J < Desc.getNumDefs(); ++J) {
4407     ((AMDGPUOperand &)*Operands[I++]).addRegOperands(Inst, 1);
4408   }
4409 
4410   for (unsigned E = Operands.size(); I != E; ++I) {
4411     AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[I]);
4412     if (isRegOrImmWithInputMods(Desc, Inst.getNumOperands())) {
4413       Op.addRegOrImmWithFPInputModsOperands(Inst, 2);
4414     } else if (Op.isInterpSlot() ||
4415                Op.isInterpAttr() ||
4416                Op.isAttrChan()) {
4417       Inst.addOperand(MCOperand::createImm(Op.Imm.Val));
4418     } else if (Op.isImmModifier()) {
4419       OptionalIdx[Op.getImmTy()] = I;
4420     } else {
4421       llvm_unreachable("unhandled operand type");
4422     }
4423   }
4424 
4425   if (AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::high) != -1) {
4426     addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyHigh);
4427   }
4428 
4429   if (AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::clamp) != -1) {
4430     addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyClampSI);
4431   }
4432 
4433   if (AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::omod) != -1) {
4434     addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyOModSI);
4435   }
4436 }
4437 
4438 void AMDGPUAsmParser::cvtVOP3(MCInst &Inst, const OperandVector &Operands,
4439                               OptionalImmIndexMap &OptionalIdx) {
4440   unsigned Opc = Inst.getOpcode();
4441 
4442   unsigned I = 1;
4443   const MCInstrDesc &Desc = MII.get(Inst.getOpcode());
4444   for (unsigned J = 0; J < Desc.getNumDefs(); ++J) {
4445     ((AMDGPUOperand &)*Operands[I++]).addRegOperands(Inst, 1);
4446   }
4447 
4448   if (AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src0_modifiers) != -1) {
4449     // This instruction has src modifiers
4450     for (unsigned E = Operands.size(); I != E; ++I) {
4451       AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[I]);
4452       if (isRegOrImmWithInputMods(Desc, Inst.getNumOperands())) {
4453         Op.addRegOrImmWithFPInputModsOperands(Inst, 2);
4454       } else if (Op.isImmModifier()) {
4455         OptionalIdx[Op.getImmTy()] = I;
4456       } else if (Op.isRegOrImm()) {
4457         Op.addRegOrImmOperands(Inst, 1);
4458       } else {
4459         llvm_unreachable("unhandled operand type");
4460       }
4461     }
4462   } else {
4463     // No src modifiers
4464     for (unsigned E = Operands.size(); I != E; ++I) {
4465       AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[I]);
4466       if (Op.isMod()) {
4467         OptionalIdx[Op.getImmTy()] = I;
4468       } else {
4469         Op.addRegOrImmOperands(Inst, 1);
4470       }
4471     }
4472   }
4473 
4474   if (AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::clamp) != -1) {
4475     addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyClampSI);
4476   }
4477 
4478   if (AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::omod) != -1) {
4479     addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyOModSI);
4480   }
4481 
4482   // special case v_mac_{f16, f32}:
4483   // it has src2 register operand that is tied to dst operand
4484   // we don't allow modifiers for this operand in assembler so src2_modifiers
4485   // should be 0
4486   if (Opc == AMDGPU::V_MAC_F32_e64_si || Opc == AMDGPU::V_MAC_F32_e64_vi ||
4487       Opc == AMDGPU::V_MAC_F16_e64_vi) {
4488     auto it = Inst.begin();
4489     std::advance(it, AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src2_modifiers));
4490     it = Inst.insert(it, MCOperand::createImm(0)); // no modifiers for src2
4491     ++it;
4492     Inst.insert(it, Inst.getOperand(0)); // src2 = dst
4493   }
4494 }
4495 
4496 void AMDGPUAsmParser::cvtVOP3(MCInst &Inst, const OperandVector &Operands) {
4497   OptionalImmIndexMap OptionalIdx;
4498   cvtVOP3(Inst, Operands, OptionalIdx);
4499 }
4500 
4501 void AMDGPUAsmParser::cvtVOP3P(MCInst &Inst,
4502                                const OperandVector &Operands) {
4503   OptionalImmIndexMap OptIdx;
4504   const int Opc = Inst.getOpcode();
4505   const MCInstrDesc &Desc = MII.get(Opc);
4506 
4507   const bool IsPacked = (Desc.TSFlags & SIInstrFlags::IsPacked) != 0;
4508 
4509   cvtVOP3(Inst, Operands, OptIdx);
4510 
4511   if (AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::vdst_in) != -1) {
4512     assert(!IsPacked);
4513     Inst.addOperand(Inst.getOperand(0));
4514   }
4515 
4516   // FIXME: This is messy. Parse the modifiers as if it was a normal VOP3
4517   // instruction, and then figure out where to actually put the modifiers
4518 
4519   addOptionalImmOperand(Inst, Operands, OptIdx, AMDGPUOperand::ImmTyOpSel);
4520 
4521   int OpSelHiIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::op_sel_hi);
4522   if (OpSelHiIdx != -1) {
4523     int DefaultVal = IsPacked ? -1 : 0;
4524     addOptionalImmOperand(Inst, Operands, OptIdx, AMDGPUOperand::ImmTyOpSelHi,
4525                           DefaultVal);
4526   }
4527 
4528   int NegLoIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::neg_lo);
4529   if (NegLoIdx != -1) {
4530     assert(IsPacked);
4531     addOptionalImmOperand(Inst, Operands, OptIdx, AMDGPUOperand::ImmTyNegLo);
4532     addOptionalImmOperand(Inst, Operands, OptIdx, AMDGPUOperand::ImmTyNegHi);
4533   }
4534 
4535   const int Ops[] = { AMDGPU::OpName::src0,
4536                       AMDGPU::OpName::src1,
4537                       AMDGPU::OpName::src2 };
4538   const int ModOps[] = { AMDGPU::OpName::src0_modifiers,
4539                          AMDGPU::OpName::src1_modifiers,
4540                          AMDGPU::OpName::src2_modifiers };
4541 
4542   int OpSelIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::op_sel);
4543 
4544   unsigned OpSel = Inst.getOperand(OpSelIdx).getImm();
4545   unsigned OpSelHi = 0;
4546   unsigned NegLo = 0;
4547   unsigned NegHi = 0;
4548 
4549   if (OpSelHiIdx != -1) {
4550     OpSelHi = Inst.getOperand(OpSelHiIdx).getImm();
4551   }
4552 
4553   if (NegLoIdx != -1) {
4554     int NegHiIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::neg_hi);
4555     NegLo = Inst.getOperand(NegLoIdx).getImm();
4556     NegHi = Inst.getOperand(NegHiIdx).getImm();
4557   }
4558 
4559   for (int J = 0; J < 3; ++J) {
4560     int OpIdx = AMDGPU::getNamedOperandIdx(Opc, Ops[J]);
4561     if (OpIdx == -1)
4562       break;
4563 
4564     uint32_t ModVal = 0;
4565 
4566     if ((OpSel & (1 << J)) != 0)
4567       ModVal |= SISrcMods::OP_SEL_0;
4568 
4569     if ((OpSelHi & (1 << J)) != 0)
4570       ModVal |= SISrcMods::OP_SEL_1;
4571 
4572     if ((NegLo & (1 << J)) != 0)
4573       ModVal |= SISrcMods::NEG;
4574 
4575     if ((NegHi & (1 << J)) != 0)
4576       ModVal |= SISrcMods::NEG_HI;
4577 
4578     int ModIdx = AMDGPU::getNamedOperandIdx(Opc, ModOps[J]);
4579 
4580     Inst.getOperand(ModIdx).setImm(Inst.getOperand(ModIdx).getImm() | ModVal);
4581   }
4582 }
4583 
4584 //===----------------------------------------------------------------------===//
4585 // dpp
4586 //===----------------------------------------------------------------------===//
4587 
4588 bool AMDGPUOperand::isDPPCtrl() const {
4589   bool result = isImm() && getImmTy() == ImmTyDppCtrl && isUInt<9>(getImm());
4590   if (result) {
4591     int64_t Imm = getImm();
4592     return ((Imm >= 0x000) && (Imm <= 0x0ff)) ||
4593            ((Imm >= 0x101) && (Imm <= 0x10f)) ||
4594            ((Imm >= 0x111) && (Imm <= 0x11f)) ||
4595            ((Imm >= 0x121) && (Imm <= 0x12f)) ||
4596            (Imm == 0x130) ||
4597            (Imm == 0x134) ||
4598            (Imm == 0x138) ||
4599            (Imm == 0x13c) ||
4600            (Imm == 0x140) ||
4601            (Imm == 0x141) ||
4602            (Imm == 0x142) ||
4603            (Imm == 0x143);
4604   }
4605   return false;
4606 }
4607 
4608 bool AMDGPUOperand::isGPRIdxMode() const {
4609   return isImm() && isUInt<4>(getImm());
4610 }
4611 
4612 bool AMDGPUOperand::isS16Imm() const {
4613   return isImm() && (isInt<16>(getImm()) || isUInt<16>(getImm()));
4614 }
4615 
4616 bool AMDGPUOperand::isU16Imm() const {
4617   return isImm() && isUInt<16>(getImm());
4618 }
4619 
4620 OperandMatchResultTy
4621 AMDGPUAsmParser::parseDPPCtrl(OperandVector &Operands) {
4622   SMLoc S = Parser.getTok().getLoc();
4623   StringRef Prefix;
4624   int64_t Int;
4625 
4626   if (getLexer().getKind() == AsmToken::Identifier) {
4627     Prefix = Parser.getTok().getString();
4628   } else {
4629     return MatchOperand_NoMatch;
4630   }
4631 
4632   if (Prefix == "row_mirror") {
4633     Int = 0x140;
4634     Parser.Lex();
4635   } else if (Prefix == "row_half_mirror") {
4636     Int = 0x141;
4637     Parser.Lex();
4638   } else {
4639     // Check to prevent parseDPPCtrlOps from eating invalid tokens
4640     if (Prefix != "quad_perm"
4641         && Prefix != "row_shl"
4642         && Prefix != "row_shr"
4643         && Prefix != "row_ror"
4644         && Prefix != "wave_shl"
4645         && Prefix != "wave_rol"
4646         && Prefix != "wave_shr"
4647         && Prefix != "wave_ror"
4648         && Prefix != "row_bcast") {
4649       return MatchOperand_NoMatch;
4650     }
4651 
4652     Parser.Lex();
4653     if (getLexer().isNot(AsmToken::Colon))
4654       return MatchOperand_ParseFail;
4655 
4656     if (Prefix == "quad_perm") {
4657       // quad_perm:[%d,%d,%d,%d]
4658       Parser.Lex();
4659       if (getLexer().isNot(AsmToken::LBrac))
4660         return MatchOperand_ParseFail;
4661       Parser.Lex();
4662 
4663       if (getParser().parseAbsoluteExpression(Int) || !(0 <= Int && Int <=3))
4664         return MatchOperand_ParseFail;
4665 
4666       for (int i = 0; i < 3; ++i) {
4667         if (getLexer().isNot(AsmToken::Comma))
4668           return MatchOperand_ParseFail;
4669         Parser.Lex();
4670 
4671         int64_t Temp;
4672         if (getParser().parseAbsoluteExpression(Temp) || !(0 <= Temp && Temp <=3))
4673           return MatchOperand_ParseFail;
4674         const int shift = i*2 + 2;
4675         Int += (Temp << shift);
4676       }
4677 
4678       if (getLexer().isNot(AsmToken::RBrac))
4679         return MatchOperand_ParseFail;
4680       Parser.Lex();
4681     } else {
4682       // sel:%d
4683       Parser.Lex();
4684       if (getParser().parseAbsoluteExpression(Int))
4685         return MatchOperand_ParseFail;
4686 
4687       if (Prefix == "row_shl" && 1 <= Int && Int <= 15) {
4688         Int |= 0x100;
4689       } else if (Prefix == "row_shr" && 1 <= Int && Int <= 15) {
4690         Int |= 0x110;
4691       } else if (Prefix == "row_ror" && 1 <= Int && Int <= 15) {
4692         Int |= 0x120;
4693       } else if (Prefix == "wave_shl" && 1 == Int) {
4694         Int = 0x130;
4695       } else if (Prefix == "wave_rol" && 1 == Int) {
4696         Int = 0x134;
4697       } else if (Prefix == "wave_shr" && 1 == Int) {
4698         Int = 0x138;
4699       } else if (Prefix == "wave_ror" && 1 == Int) {
4700         Int = 0x13C;
4701       } else if (Prefix == "row_bcast") {
4702         if (Int == 15) {
4703           Int = 0x142;
4704         } else if (Int == 31) {
4705           Int = 0x143;
4706         } else {
4707           return MatchOperand_ParseFail;
4708         }
4709       } else {
4710         return MatchOperand_ParseFail;
4711       }
4712     }
4713   }
4714 
4715   Operands.push_back(AMDGPUOperand::CreateImm(this, Int, S, AMDGPUOperand::ImmTyDppCtrl));
4716   return MatchOperand_Success;
4717 }
4718 
4719 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultRowMask() const {
4720   return AMDGPUOperand::CreateImm(this, 0xf, SMLoc(), AMDGPUOperand::ImmTyDppRowMask);
4721 }
4722 
4723 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultBankMask() const {
4724   return AMDGPUOperand::CreateImm(this, 0xf, SMLoc(), AMDGPUOperand::ImmTyDppBankMask);
4725 }
4726 
4727 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultBoundCtrl() const {
4728   return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTyDppBoundCtrl);
4729 }
4730 
4731 void AMDGPUAsmParser::cvtDPP(MCInst &Inst, const OperandVector &Operands) {
4732   OptionalImmIndexMap OptionalIdx;
4733 
4734   unsigned I = 1;
4735   const MCInstrDesc &Desc = MII.get(Inst.getOpcode());
4736   for (unsigned J = 0; J < Desc.getNumDefs(); ++J) {
4737     ((AMDGPUOperand &)*Operands[I++]).addRegOperands(Inst, 1);
4738   }
4739 
4740   // All DPP instructions with at least one source operand have a fake "old"
4741   // source at the beginning that's tied to the dst operand. Handle it here.
4742   if (Desc.getNumOperands() >= 2)
4743     Inst.addOperand(Inst.getOperand(0));
4744 
4745   for (unsigned E = Operands.size(); I != E; ++I) {
4746     AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[I]);
4747     // Add the register arguments
4748     if (Op.isReg() && Op.Reg.RegNo == AMDGPU::VCC) {
4749       // VOP2b (v_add_u32, v_sub_u32 ...) dpp use "vcc" token.
4750       // Skip it.
4751       continue;
4752     } if (isRegOrImmWithInputMods(Desc, Inst.getNumOperands())) {
4753       Op.addRegWithFPInputModsOperands(Inst, 2);
4754     } else if (Op.isDPPCtrl()) {
4755       Op.addImmOperands(Inst, 1);
4756     } else if (Op.isImm()) {
4757       // Handle optional arguments
4758       OptionalIdx[Op.getImmTy()] = I;
4759     } else {
4760       llvm_unreachable("Invalid operand type");
4761     }
4762   }
4763 
4764   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyDppRowMask, 0xf);
4765   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyDppBankMask, 0xf);
4766   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyDppBoundCtrl);
4767 }
4768 
4769 //===----------------------------------------------------------------------===//
4770 // sdwa
4771 //===----------------------------------------------------------------------===//
4772 
4773 OperandMatchResultTy
4774 AMDGPUAsmParser::parseSDWASel(OperandVector &Operands, StringRef Prefix,
4775                               AMDGPUOperand::ImmTy Type) {
4776   using namespace llvm::AMDGPU::SDWA;
4777 
4778   SMLoc S = Parser.getTok().getLoc();
4779   StringRef Value;
4780   OperandMatchResultTy res;
4781 
4782   res = parseStringWithPrefix(Prefix, Value);
4783   if (res != MatchOperand_Success) {
4784     return res;
4785   }
4786 
4787   int64_t Int;
4788   Int = StringSwitch<int64_t>(Value)
4789         .Case("BYTE_0", SdwaSel::BYTE_0)
4790         .Case("BYTE_1", SdwaSel::BYTE_1)
4791         .Case("BYTE_2", SdwaSel::BYTE_2)
4792         .Case("BYTE_3", SdwaSel::BYTE_3)
4793         .Case("WORD_0", SdwaSel::WORD_0)
4794         .Case("WORD_1", SdwaSel::WORD_1)
4795         .Case("DWORD", SdwaSel::DWORD)
4796         .Default(0xffffffff);
4797   Parser.Lex(); // eat last token
4798 
4799   if (Int == 0xffffffff) {
4800     return MatchOperand_ParseFail;
4801   }
4802 
4803   Operands.push_back(AMDGPUOperand::CreateImm(this, Int, S, Type));
4804   return MatchOperand_Success;
4805 }
4806 
4807 OperandMatchResultTy
4808 AMDGPUAsmParser::parseSDWADstUnused(OperandVector &Operands) {
4809   using namespace llvm::AMDGPU::SDWA;
4810 
4811   SMLoc S = Parser.getTok().getLoc();
4812   StringRef Value;
4813   OperandMatchResultTy res;
4814 
4815   res = parseStringWithPrefix("dst_unused", Value);
4816   if (res != MatchOperand_Success) {
4817     return res;
4818   }
4819 
4820   int64_t Int;
4821   Int = StringSwitch<int64_t>(Value)
4822         .Case("UNUSED_PAD", DstUnused::UNUSED_PAD)
4823         .Case("UNUSED_SEXT", DstUnused::UNUSED_SEXT)
4824         .Case("UNUSED_PRESERVE", DstUnused::UNUSED_PRESERVE)
4825         .Default(0xffffffff);
4826   Parser.Lex(); // eat last token
4827 
4828   if (Int == 0xffffffff) {
4829     return MatchOperand_ParseFail;
4830   }
4831 
4832   Operands.push_back(AMDGPUOperand::CreateImm(this, Int, S, AMDGPUOperand::ImmTySdwaDstUnused));
4833   return MatchOperand_Success;
4834 }
4835 
4836 void AMDGPUAsmParser::cvtSdwaVOP1(MCInst &Inst, const OperandVector &Operands) {
4837   cvtSDWA(Inst, Operands, SIInstrFlags::VOP1);
4838 }
4839 
4840 void AMDGPUAsmParser::cvtSdwaVOP2(MCInst &Inst, const OperandVector &Operands) {
4841   cvtSDWA(Inst, Operands, SIInstrFlags::VOP2);
4842 }
4843 
4844 void AMDGPUAsmParser::cvtSdwaVOP2b(MCInst &Inst, const OperandVector &Operands) {
4845   cvtSDWA(Inst, Operands, SIInstrFlags::VOP2, true);
4846 }
4847 
4848 void AMDGPUAsmParser::cvtSdwaVOPC(MCInst &Inst, const OperandVector &Operands) {
4849   cvtSDWA(Inst, Operands, SIInstrFlags::VOPC, isVI());
4850 }
4851 
4852 void AMDGPUAsmParser::cvtSDWA(MCInst &Inst, const OperandVector &Operands,
4853                               uint64_t BasicInstType, bool skipVcc) {
4854   using namespace llvm::AMDGPU::SDWA;
4855 
4856   OptionalImmIndexMap OptionalIdx;
4857   bool skippedVcc = false;
4858 
4859   unsigned I = 1;
4860   const MCInstrDesc &Desc = MII.get(Inst.getOpcode());
4861   for (unsigned J = 0; J < Desc.getNumDefs(); ++J) {
4862     ((AMDGPUOperand &)*Operands[I++]).addRegOperands(Inst, 1);
4863   }
4864 
4865   for (unsigned E = Operands.size(); I != E; ++I) {
4866     AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[I]);
4867     if (skipVcc && !skippedVcc && Op.isReg() && Op.Reg.RegNo == AMDGPU::VCC) {
4868       // VOP2b (v_add_u32, v_sub_u32 ...) sdwa use "vcc" token as dst.
4869       // Skip it if it's 2nd (e.g. v_add_i32_sdwa v1, vcc, v2, v3)
4870       // or 4th (v_addc_u32_sdwa v1, vcc, v2, v3, vcc) operand.
4871       // Skip VCC only if we didn't skip it on previous iteration.
4872       if (BasicInstType == SIInstrFlags::VOP2 &&
4873           (Inst.getNumOperands() == 1 || Inst.getNumOperands() == 5)) {
4874         skippedVcc = true;
4875         continue;
4876       } else if (BasicInstType == SIInstrFlags::VOPC &&
4877                  Inst.getNumOperands() == 0) {
4878         skippedVcc = true;
4879         continue;
4880       }
4881     }
4882     if (isRegOrImmWithInputMods(Desc, Inst.getNumOperands())) {
4883       Op.addRegOrImmWithInputModsOperands(Inst, 2);
4884     } else if (Op.isImm()) {
4885       // Handle optional arguments
4886       OptionalIdx[Op.getImmTy()] = I;
4887     } else {
4888       llvm_unreachable("Invalid operand type");
4889     }
4890     skippedVcc = false;
4891   }
4892 
4893   if (Inst.getOpcode() != AMDGPU::V_NOP_sdwa_gfx9 &&
4894       Inst.getOpcode() != AMDGPU::V_NOP_sdwa_vi) {
4895     // v_nop_sdwa_sdwa_vi/gfx9 has no optional sdwa arguments
4896     switch (BasicInstType) {
4897     case SIInstrFlags::VOP1:
4898       addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyClampSI, 0);
4899       if (AMDGPU::getNamedOperandIdx(Inst.getOpcode(), AMDGPU::OpName::omod) != -1) {
4900         addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyOModSI, 0);
4901       }
4902       addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySdwaDstSel, SdwaSel::DWORD);
4903       addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySdwaDstUnused, DstUnused::UNUSED_PRESERVE);
4904       addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySdwaSrc0Sel, SdwaSel::DWORD);
4905       break;
4906 
4907     case SIInstrFlags::VOP2:
4908       addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyClampSI, 0);
4909       if (AMDGPU::getNamedOperandIdx(Inst.getOpcode(), AMDGPU::OpName::omod) != -1) {
4910         addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyOModSI, 0);
4911       }
4912       addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySdwaDstSel, SdwaSel::DWORD);
4913       addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySdwaDstUnused, DstUnused::UNUSED_PRESERVE);
4914       addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySdwaSrc0Sel, SdwaSel::DWORD);
4915       addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySdwaSrc1Sel, SdwaSel::DWORD);
4916       break;
4917 
4918     case SIInstrFlags::VOPC:
4919       addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyClampSI, 0);
4920       addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySdwaSrc0Sel, SdwaSel::DWORD);
4921       addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySdwaSrc1Sel, SdwaSel::DWORD);
4922       break;
4923 
4924     default:
4925       llvm_unreachable("Invalid instruction type. Only VOP1, VOP2 and VOPC allowed");
4926     }
4927   }
4928 
4929   // special case v_mac_{f16, f32}:
4930   // it has src2 register operand that is tied to dst operand
4931   if (Inst.getOpcode() == AMDGPU::V_MAC_F32_sdwa_vi ||
4932       Inst.getOpcode() == AMDGPU::V_MAC_F16_sdwa_vi)  {
4933     auto it = Inst.begin();
4934     std::advance(
4935       it, AMDGPU::getNamedOperandIdx(Inst.getOpcode(), AMDGPU::OpName::src2));
4936     Inst.insert(it, Inst.getOperand(0)); // src2 = dst
4937   }
4938 }
4939 
4940 /// Force static initialization.
4941 extern "C" void LLVMInitializeAMDGPUAsmParser() {
4942   RegisterMCAsmParser<AMDGPUAsmParser> A(getTheAMDGPUTarget());
4943   RegisterMCAsmParser<AMDGPUAsmParser> B(getTheGCNTarget());
4944 }
4945 
4946 #define GET_REGISTER_MATCHER
4947 #define GET_MATCHER_IMPLEMENTATION
4948 #define GET_MNEMONIC_SPELL_CHECKER
4949 #include "AMDGPUGenAsmMatcher.inc"
4950 
4951 // This fuction should be defined after auto-generated include so that we have
4952 // MatchClassKind enum defined
4953 unsigned AMDGPUAsmParser::validateTargetOperandClass(MCParsedAsmOperand &Op,
4954                                                      unsigned Kind) {
4955   // Tokens like "glc" would be parsed as immediate operands in ParseOperand().
4956   // But MatchInstructionImpl() expects to meet token and fails to validate
4957   // operand. This method checks if we are given immediate operand but expect to
4958   // get corresponding token.
4959   AMDGPUOperand &Operand = (AMDGPUOperand&)Op;
4960   switch (Kind) {
4961   case MCK_addr64:
4962     return Operand.isAddr64() ? Match_Success : Match_InvalidOperand;
4963   case MCK_gds:
4964     return Operand.isGDS() ? Match_Success : Match_InvalidOperand;
4965   case MCK_glc:
4966     return Operand.isGLC() ? Match_Success : Match_InvalidOperand;
4967   case MCK_idxen:
4968     return Operand.isIdxen() ? Match_Success : Match_InvalidOperand;
4969   case MCK_offen:
4970     return Operand.isOffen() ? Match_Success : Match_InvalidOperand;
4971   case MCK_SSrcB32:
4972     // When operands have expression values, they will return true for isToken,
4973     // because it is not possible to distinguish between a token and an
4974     // expression at parse time. MatchInstructionImpl() will always try to
4975     // match an operand as a token, when isToken returns true, and when the
4976     // name of the expression is not a valid token, the match will fail,
4977     // so we need to handle it here.
4978     return Operand.isSSrcB32() ? Match_Success : Match_InvalidOperand;
4979   case MCK_SSrcF32:
4980     return Operand.isSSrcF32() ? Match_Success : Match_InvalidOperand;
4981   case MCK_SoppBrTarget:
4982     return Operand.isSoppBrTarget() ? Match_Success : Match_InvalidOperand;
4983   case MCK_VReg32OrOff:
4984     return Operand.isVReg32OrOff() ? Match_Success : Match_InvalidOperand;
4985   case MCK_InterpSlot:
4986     return Operand.isInterpSlot() ? Match_Success : Match_InvalidOperand;
4987   case MCK_Attr:
4988     return Operand.isInterpAttr() ? Match_Success : Match_InvalidOperand;
4989   case MCK_AttrChan:
4990     return Operand.isAttrChan() ? Match_Success : Match_InvalidOperand;
4991   default:
4992     return Match_InvalidOperand;
4993   }
4994 }
4995