1 //===-- AMDGPUAsmParser.cpp - Parse SI asm to MCInst instructions ---------===//
2 //
3 //                     The LLVM Compiler Infrastructure
4 //
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
7 //
8 //===----------------------------------------------------------------------===//
9 
10 #include "AMDKernelCodeT.h"
11 #include "MCTargetDesc/AMDGPUMCTargetDesc.h"
12 #include "MCTargetDesc/AMDGPUTargetStreamer.h"
13 #include "SIDefines.h"
14 #include "Utils/AMDGPUAsmUtils.h"
15 #include "Utils/AMDGPUBaseInfo.h"
16 #include "Utils/AMDKernelCodeTUtils.h"
17 #include "llvm/ADT/APFloat.h"
18 #include "llvm/ADT/APInt.h"
19 #include "llvm/ADT/ArrayRef.h"
20 #include "llvm/ADT/STLExtras.h"
21 #include "llvm/ADT/SmallBitVector.h"
22 #include "llvm/ADT/SmallString.h"
23 #include "llvm/ADT/StringRef.h"
24 #include "llvm/ADT/StringSwitch.h"
25 #include "llvm/ADT/Twine.h"
26 #include "llvm/BinaryFormat/ELF.h"
27 #include "llvm/CodeGen/MachineValueType.h"
28 #include "llvm/MC/MCAsmInfo.h"
29 #include "llvm/MC/MCContext.h"
30 #include "llvm/MC/MCExpr.h"
31 #include "llvm/MC/MCInst.h"
32 #include "llvm/MC/MCInstrDesc.h"
33 #include "llvm/MC/MCInstrInfo.h"
34 #include "llvm/MC/MCParser/MCAsmLexer.h"
35 #include "llvm/MC/MCParser/MCAsmParser.h"
36 #include "llvm/MC/MCParser/MCAsmParserExtension.h"
37 #include "llvm/MC/MCParser/MCParsedAsmOperand.h"
38 #include "llvm/MC/MCParser/MCTargetAsmParser.h"
39 #include "llvm/MC/MCRegisterInfo.h"
40 #include "llvm/MC/MCStreamer.h"
41 #include "llvm/MC/MCSubtargetInfo.h"
42 #include "llvm/MC/MCSymbol.h"
43 #include "llvm/Support/Casting.h"
44 #include "llvm/Support/ErrorHandling.h"
45 #include "llvm/Support/MathExtras.h"
46 #include "llvm/Support/SMLoc.h"
47 #include "llvm/Support/TargetRegistry.h"
48 #include "llvm/Support/raw_ostream.h"
49 #include <algorithm>
50 #include <cassert>
51 #include <cstdint>
52 #include <cstring>
53 #include <iterator>
54 #include <map>
55 #include <memory>
56 #include <string>
57 
58 using namespace llvm;
59 using namespace llvm::AMDGPU;
60 
61 namespace {
62 
63 class AMDGPUAsmParser;
64 
65 enum RegisterKind { IS_UNKNOWN, IS_VGPR, IS_SGPR, IS_TTMP, IS_SPECIAL };
66 
67 //===----------------------------------------------------------------------===//
68 // Operand
69 //===----------------------------------------------------------------------===//
70 
71 class AMDGPUOperand : public MCParsedAsmOperand {
72   enum KindTy {
73     Token,
74     Immediate,
75     Register,
76     Expression
77   } Kind;
78 
79   SMLoc StartLoc, EndLoc;
80   const AMDGPUAsmParser *AsmParser;
81 
82 public:
83   AMDGPUOperand(KindTy Kind_, const AMDGPUAsmParser *AsmParser_)
84     : MCParsedAsmOperand(), Kind(Kind_), AsmParser(AsmParser_) {}
85 
86   typedef std::unique_ptr<AMDGPUOperand> Ptr;
87 
88   struct Modifiers {
89     bool Abs = false;
90     bool Neg = false;
91     bool Sext = false;
92 
93     bool hasFPModifiers() const { return Abs || Neg; }
94     bool hasIntModifiers() const { return Sext; }
95     bool hasModifiers() const { return hasFPModifiers() || hasIntModifiers(); }
96 
97     int64_t getFPModifiersOperand() const {
98       int64_t Operand = 0;
99       Operand |= Abs ? SISrcMods::ABS : 0;
100       Operand |= Neg ? SISrcMods::NEG : 0;
101       return Operand;
102     }
103 
104     int64_t getIntModifiersOperand() const {
105       int64_t Operand = 0;
106       Operand |= Sext ? SISrcMods::SEXT : 0;
107       return Operand;
108     }
109 
110     int64_t getModifiersOperand() const {
111       assert(!(hasFPModifiers() && hasIntModifiers())
112            && "fp and int modifiers should not be used simultaneously");
113       if (hasFPModifiers()) {
114         return getFPModifiersOperand();
115       } else if (hasIntModifiers()) {
116         return getIntModifiersOperand();
117       } else {
118         return 0;
119       }
120     }
121 
122     friend raw_ostream &operator <<(raw_ostream &OS, AMDGPUOperand::Modifiers Mods);
123   };
124 
125   enum ImmTy {
126     ImmTyNone,
127     ImmTyGDS,
128     ImmTyOffen,
129     ImmTyIdxen,
130     ImmTyAddr64,
131     ImmTyOffset,
132     ImmTyOffset0,
133     ImmTyOffset1,
134     ImmTyGLC,
135     ImmTySLC,
136     ImmTyTFE,
137     ImmTyClampSI,
138     ImmTyOModSI,
139     ImmTyDppCtrl,
140     ImmTyDppRowMask,
141     ImmTyDppBankMask,
142     ImmTyDppBoundCtrl,
143     ImmTySdwaDstSel,
144     ImmTySdwaSrc0Sel,
145     ImmTySdwaSrc1Sel,
146     ImmTySdwaDstUnused,
147     ImmTyDMask,
148     ImmTyUNorm,
149     ImmTyDA,
150     ImmTyR128,
151     ImmTyLWE,
152     ImmTyExpTgt,
153     ImmTyExpCompr,
154     ImmTyExpVM,
155     ImmTyHwreg,
156     ImmTyOff,
157     ImmTySendMsg,
158     ImmTyInterpSlot,
159     ImmTyInterpAttr,
160     ImmTyAttrChan,
161     ImmTyOpSel,
162     ImmTyOpSelHi,
163     ImmTyNegLo,
164     ImmTyNegHi,
165     ImmTySwizzle
166   };
167 
168   struct TokOp {
169     const char *Data;
170     unsigned Length;
171   };
172 
173   struct ImmOp {
174     int64_t Val;
175     ImmTy Type;
176     bool IsFPImm;
177     Modifiers Mods;
178   };
179 
180   struct RegOp {
181     unsigned RegNo;
182     bool IsForcedVOP3;
183     Modifiers Mods;
184   };
185 
186   union {
187     TokOp Tok;
188     ImmOp Imm;
189     RegOp Reg;
190     const MCExpr *Expr;
191   };
192 
193   bool isToken() const override {
194     if (Kind == Token)
195       return true;
196 
197     if (Kind != Expression || !Expr)
198       return false;
199 
200     // When parsing operands, we can't always tell if something was meant to be
201     // a token, like 'gds', or an expression that references a global variable.
202     // In this case, we assume the string is an expression, and if we need to
203     // interpret is a token, then we treat the symbol name as the token.
204     return isa<MCSymbolRefExpr>(Expr);
205   }
206 
207   bool isImm() const override {
208     return Kind == Immediate;
209   }
210 
211   bool isInlinableImm(MVT type) const;
212   bool isLiteralImm(MVT type) const;
213 
214   bool isRegKind() const {
215     return Kind == Register;
216   }
217 
218   bool isReg() const override {
219     return isRegKind() && !hasModifiers();
220   }
221 
222   bool isRegOrImmWithInputMods(MVT type) const {
223     return isRegKind() || isInlinableImm(type);
224   }
225 
226   bool isRegOrImmWithInt16InputMods() const {
227     return isRegOrImmWithInputMods(MVT::i16);
228   }
229 
230   bool isRegOrImmWithInt32InputMods() const {
231     return isRegOrImmWithInputMods(MVT::i32);
232   }
233 
234   bool isRegOrImmWithInt64InputMods() const {
235     return isRegOrImmWithInputMods(MVT::i64);
236   }
237 
238   bool isRegOrImmWithFP16InputMods() const {
239     return isRegOrImmWithInputMods(MVT::f16);
240   }
241 
242   bool isRegOrImmWithFP32InputMods() const {
243     return isRegOrImmWithInputMods(MVT::f32);
244   }
245 
246   bool isRegOrImmWithFP64InputMods() const {
247     return isRegOrImmWithInputMods(MVT::f64);
248   }
249 
250   bool isVReg() const {
251     return isRegClass(AMDGPU::VGPR_32RegClassID) ||
252            isRegClass(AMDGPU::VReg_64RegClassID) ||
253            isRegClass(AMDGPU::VReg_96RegClassID) ||
254            isRegClass(AMDGPU::VReg_128RegClassID) ||
255            isRegClass(AMDGPU::VReg_256RegClassID) ||
256            isRegClass(AMDGPU::VReg_512RegClassID);
257   }
258 
259   bool isVReg32OrOff() const {
260     return isOff() || isRegClass(AMDGPU::VGPR_32RegClassID);
261   }
262 
263   bool isImmTy(ImmTy ImmT) const {
264     return isImm() && Imm.Type == ImmT;
265   }
266 
267   bool isImmModifier() const {
268     return isImm() && Imm.Type != ImmTyNone;
269   }
270 
271   bool isClampSI() const { return isImmTy(ImmTyClampSI); }
272   bool isOModSI() const { return isImmTy(ImmTyOModSI); }
273   bool isDMask() const { return isImmTy(ImmTyDMask); }
274   bool isUNorm() const { return isImmTy(ImmTyUNorm); }
275   bool isDA() const { return isImmTy(ImmTyDA); }
276   bool isR128() const { return isImmTy(ImmTyUNorm); }
277   bool isLWE() const { return isImmTy(ImmTyLWE); }
278   bool isOff() const { return isImmTy(ImmTyOff); }
279   bool isExpTgt() const { return isImmTy(ImmTyExpTgt); }
280   bool isExpVM() const { return isImmTy(ImmTyExpVM); }
281   bool isExpCompr() const { return isImmTy(ImmTyExpCompr); }
282   bool isOffen() const { return isImmTy(ImmTyOffen); }
283   bool isIdxen() const { return isImmTy(ImmTyIdxen); }
284   bool isAddr64() const { return isImmTy(ImmTyAddr64); }
285   bool isOffset() const { return isImmTy(ImmTyOffset) && isUInt<16>(getImm()); }
286   bool isOffset0() const { return isImmTy(ImmTyOffset0) && isUInt<16>(getImm()); }
287   bool isOffset1() const { return isImmTy(ImmTyOffset1) && isUInt<8>(getImm()); }
288 
289   bool isOffsetU12() const { return isImmTy(ImmTyOffset) && isUInt<12>(getImm()); }
290   bool isOffsetS13() const { return isImmTy(ImmTyOffset) && isInt<13>(getImm()); }
291   bool isGDS() const { return isImmTy(ImmTyGDS); }
292   bool isGLC() const { return isImmTy(ImmTyGLC); }
293   bool isSLC() const { return isImmTy(ImmTySLC); }
294   bool isTFE() const { return isImmTy(ImmTyTFE); }
295   bool isBankMask() const { return isImmTy(ImmTyDppBankMask); }
296   bool isRowMask() const { return isImmTy(ImmTyDppRowMask); }
297   bool isBoundCtrl() const { return isImmTy(ImmTyDppBoundCtrl); }
298   bool isSDWADstSel() const { return isImmTy(ImmTySdwaDstSel); }
299   bool isSDWASrc0Sel() const { return isImmTy(ImmTySdwaSrc0Sel); }
300   bool isSDWASrc1Sel() const { return isImmTy(ImmTySdwaSrc1Sel); }
301   bool isSDWADstUnused() const { return isImmTy(ImmTySdwaDstUnused); }
302   bool isInterpSlot() const { return isImmTy(ImmTyInterpSlot); }
303   bool isInterpAttr() const { return isImmTy(ImmTyInterpAttr); }
304   bool isAttrChan() const { return isImmTy(ImmTyAttrChan); }
305   bool isOpSel() const { return isImmTy(ImmTyOpSel); }
306   bool isOpSelHi() const { return isImmTy(ImmTyOpSelHi); }
307   bool isNegLo() const { return isImmTy(ImmTyNegLo); }
308   bool isNegHi() const { return isImmTy(ImmTyNegHi); }
309 
310   bool isMod() const {
311     return isClampSI() || isOModSI();
312   }
313 
314   bool isRegOrImm() const {
315     return isReg() || isImm();
316   }
317 
318   bool isRegClass(unsigned RCID) const;
319 
320   bool isRegOrInlineNoMods(unsigned RCID, MVT type) const {
321     return (isRegClass(RCID) || isInlinableImm(type)) && !hasModifiers();
322   }
323 
324   bool isSCSrcB16() const {
325     return isRegOrInlineNoMods(AMDGPU::SReg_32RegClassID, MVT::i16);
326   }
327 
328   bool isSCSrcV2B16() const {
329     return isSCSrcB16();
330   }
331 
332   bool isSCSrcB32() const {
333     return isRegOrInlineNoMods(AMDGPU::SReg_32RegClassID, MVT::i32);
334   }
335 
336   bool isSCSrcB64() const {
337     return isRegOrInlineNoMods(AMDGPU::SReg_64RegClassID, MVT::i64);
338   }
339 
340   bool isSCSrcF16() const {
341     return isRegOrInlineNoMods(AMDGPU::SReg_32RegClassID, MVT::f16);
342   }
343 
344   bool isSCSrcV2F16() const {
345     return isSCSrcF16();
346   }
347 
348   bool isSCSrcF32() const {
349     return isRegOrInlineNoMods(AMDGPU::SReg_32RegClassID, MVT::f32);
350   }
351 
352   bool isSCSrcF64() const {
353     return isRegOrInlineNoMods(AMDGPU::SReg_64RegClassID, MVT::f64);
354   }
355 
356   bool isSSrcB32() const {
357     return isSCSrcB32() || isLiteralImm(MVT::i32) || isExpr();
358   }
359 
360   bool isSSrcB16() const {
361     return isSCSrcB16() || isLiteralImm(MVT::i16);
362   }
363 
364   bool isSSrcV2B16() const {
365     llvm_unreachable("cannot happen");
366     return isSSrcB16();
367   }
368 
369   bool isSSrcB64() const {
370     // TODO: Find out how SALU supports extension of 32-bit literals to 64 bits.
371     // See isVSrc64().
372     return isSCSrcB64() || isLiteralImm(MVT::i64);
373   }
374 
375   bool isSSrcF32() const {
376     return isSCSrcB32() || isLiteralImm(MVT::f32) || isExpr();
377   }
378 
379   bool isSSrcF64() const {
380     return isSCSrcB64() || isLiteralImm(MVT::f64);
381   }
382 
383   bool isSSrcF16() const {
384     return isSCSrcB16() || isLiteralImm(MVT::f16);
385   }
386 
387   bool isSSrcV2F16() const {
388     llvm_unreachable("cannot happen");
389     return isSSrcF16();
390   }
391 
392   bool isVCSrcB32() const {
393     return isRegOrInlineNoMods(AMDGPU::VS_32RegClassID, MVT::i32);
394   }
395 
396   bool isVCSrcB64() const {
397     return isRegOrInlineNoMods(AMDGPU::VS_64RegClassID, MVT::i64);
398   }
399 
400   bool isVCSrcB16() const {
401     return isRegOrInlineNoMods(AMDGPU::VS_32RegClassID, MVT::i16);
402   }
403 
404   bool isVCSrcV2B16() const {
405     return isVCSrcB16();
406   }
407 
408   bool isVCSrcF32() const {
409     return isRegOrInlineNoMods(AMDGPU::VS_32RegClassID, MVT::f32);
410   }
411 
412   bool isVCSrcF64() const {
413     return isRegOrInlineNoMods(AMDGPU::VS_64RegClassID, MVT::f64);
414   }
415 
416   bool isVCSrcF16() const {
417     return isRegOrInlineNoMods(AMDGPU::VS_32RegClassID, MVT::f16);
418   }
419 
420   bool isVCSrcV2F16() const {
421     return isVCSrcF16();
422   }
423 
424   bool isVSrcB32() const {
425     return isVCSrcF32() || isLiteralImm(MVT::i32);
426   }
427 
428   bool isVSrcB64() const {
429     return isVCSrcF64() || isLiteralImm(MVT::i64);
430   }
431 
432   bool isVSrcB16() const {
433     return isVCSrcF16() || isLiteralImm(MVT::i16);
434   }
435 
436   bool isVSrcV2B16() const {
437     llvm_unreachable("cannot happen");
438     return isVSrcB16();
439   }
440 
441   bool isVSrcF32() const {
442     return isVCSrcF32() || isLiteralImm(MVT::f32);
443   }
444 
445   bool isVSrcF64() const {
446     return isVCSrcF64() || isLiteralImm(MVT::f64);
447   }
448 
449   bool isVSrcF16() const {
450     return isVCSrcF16() || isLiteralImm(MVT::f16);
451   }
452 
453   bool isVSrcV2F16() const {
454     llvm_unreachable("cannot happen");
455     return isVSrcF16();
456   }
457 
458   bool isKImmFP32() const {
459     return isLiteralImm(MVT::f32);
460   }
461 
462   bool isKImmFP16() const {
463     return isLiteralImm(MVT::f16);
464   }
465 
466   bool isMem() const override {
467     return false;
468   }
469 
470   bool isExpr() const {
471     return Kind == Expression;
472   }
473 
474   bool isSoppBrTarget() const {
475     return isExpr() || isImm();
476   }
477 
478   bool isSWaitCnt() const;
479   bool isHwreg() const;
480   bool isSendMsg() const;
481   bool isSwizzle() const;
482   bool isSMRDOffset8() const;
483   bool isSMRDOffset20() const;
484   bool isSMRDLiteralOffset() const;
485   bool isDPPCtrl() const;
486   bool isGPRIdxMode() const;
487   bool isS16Imm() const;
488   bool isU16Imm() const;
489 
490   StringRef getExpressionAsToken() const {
491     assert(isExpr());
492     const MCSymbolRefExpr *S = cast<MCSymbolRefExpr>(Expr);
493     return S->getSymbol().getName();
494   }
495 
496   StringRef getToken() const {
497     assert(isToken());
498 
499     if (Kind == Expression)
500       return getExpressionAsToken();
501 
502     return StringRef(Tok.Data, Tok.Length);
503   }
504 
505   int64_t getImm() const {
506     assert(isImm());
507     return Imm.Val;
508   }
509 
510   ImmTy getImmTy() const {
511     assert(isImm());
512     return Imm.Type;
513   }
514 
515   unsigned getReg() const override {
516     return Reg.RegNo;
517   }
518 
519   SMLoc getStartLoc() const override {
520     return StartLoc;
521   }
522 
523   SMLoc getEndLoc() const override {
524     return EndLoc;
525   }
526 
527   Modifiers getModifiers() const {
528     assert(isRegKind() || isImmTy(ImmTyNone));
529     return isRegKind() ? Reg.Mods : Imm.Mods;
530   }
531 
532   void setModifiers(Modifiers Mods) {
533     assert(isRegKind() || isImmTy(ImmTyNone));
534     if (isRegKind())
535       Reg.Mods = Mods;
536     else
537       Imm.Mods = Mods;
538   }
539 
540   bool hasModifiers() const {
541     return getModifiers().hasModifiers();
542   }
543 
544   bool hasFPModifiers() const {
545     return getModifiers().hasFPModifiers();
546   }
547 
548   bool hasIntModifiers() const {
549     return getModifiers().hasIntModifiers();
550   }
551 
552   uint64_t applyInputFPModifiers(uint64_t Val, unsigned Size) const;
553 
554   void addImmOperands(MCInst &Inst, unsigned N, bool ApplyModifiers = true) const;
555 
556   void addLiteralImmOperand(MCInst &Inst, int64_t Val, bool ApplyModifiers) const;
557 
558   template <unsigned Bitwidth>
559   void addKImmFPOperands(MCInst &Inst, unsigned N) const;
560 
561   void addKImmFP16Operands(MCInst &Inst, unsigned N) const {
562     addKImmFPOperands<16>(Inst, N);
563   }
564 
565   void addKImmFP32Operands(MCInst &Inst, unsigned N) const {
566     addKImmFPOperands<32>(Inst, N);
567   }
568 
569   void addRegOperands(MCInst &Inst, unsigned N) const;
570 
571   void addRegOrImmOperands(MCInst &Inst, unsigned N) const {
572     if (isRegKind())
573       addRegOperands(Inst, N);
574     else if (isExpr())
575       Inst.addOperand(MCOperand::createExpr(Expr));
576     else
577       addImmOperands(Inst, N);
578   }
579 
580   void addRegOrImmWithInputModsOperands(MCInst &Inst, unsigned N) const {
581     Modifiers Mods = getModifiers();
582     Inst.addOperand(MCOperand::createImm(Mods.getModifiersOperand()));
583     if (isRegKind()) {
584       addRegOperands(Inst, N);
585     } else {
586       addImmOperands(Inst, N, false);
587     }
588   }
589 
590   void addRegOrImmWithFPInputModsOperands(MCInst &Inst, unsigned N) const {
591     assert(!hasIntModifiers());
592     addRegOrImmWithInputModsOperands(Inst, N);
593   }
594 
595   void addRegOrImmWithIntInputModsOperands(MCInst &Inst, unsigned N) const {
596     assert(!hasFPModifiers());
597     addRegOrImmWithInputModsOperands(Inst, N);
598   }
599 
600   void addRegWithInputModsOperands(MCInst &Inst, unsigned N) const {
601     Modifiers Mods = getModifiers();
602     Inst.addOperand(MCOperand::createImm(Mods.getModifiersOperand()));
603     assert(isRegKind());
604     addRegOperands(Inst, N);
605   }
606 
607   void addRegWithFPInputModsOperands(MCInst &Inst, unsigned N) const {
608     assert(!hasIntModifiers());
609     addRegWithInputModsOperands(Inst, N);
610   }
611 
612   void addRegWithIntInputModsOperands(MCInst &Inst, unsigned N) const {
613     assert(!hasFPModifiers());
614     addRegWithInputModsOperands(Inst, N);
615   }
616 
617   void addSoppBrTargetOperands(MCInst &Inst, unsigned N) const {
618     if (isImm())
619       addImmOperands(Inst, N);
620     else {
621       assert(isExpr());
622       Inst.addOperand(MCOperand::createExpr(Expr));
623     }
624   }
625 
626   static void printImmTy(raw_ostream& OS, ImmTy Type) {
627     switch (Type) {
628     case ImmTyNone: OS << "None"; break;
629     case ImmTyGDS: OS << "GDS"; break;
630     case ImmTyOffen: OS << "Offen"; break;
631     case ImmTyIdxen: OS << "Idxen"; break;
632     case ImmTyAddr64: OS << "Addr64"; break;
633     case ImmTyOffset: OS << "Offset"; break;
634     case ImmTyOffset0: OS << "Offset0"; break;
635     case ImmTyOffset1: OS << "Offset1"; break;
636     case ImmTyGLC: OS << "GLC"; break;
637     case ImmTySLC: OS << "SLC"; break;
638     case ImmTyTFE: OS << "TFE"; break;
639     case ImmTyClampSI: OS << "ClampSI"; break;
640     case ImmTyOModSI: OS << "OModSI"; break;
641     case ImmTyDppCtrl: OS << "DppCtrl"; break;
642     case ImmTyDppRowMask: OS << "DppRowMask"; break;
643     case ImmTyDppBankMask: OS << "DppBankMask"; break;
644     case ImmTyDppBoundCtrl: OS << "DppBoundCtrl"; break;
645     case ImmTySdwaDstSel: OS << "SdwaDstSel"; break;
646     case ImmTySdwaSrc0Sel: OS << "SdwaSrc0Sel"; break;
647     case ImmTySdwaSrc1Sel: OS << "SdwaSrc1Sel"; break;
648     case ImmTySdwaDstUnused: OS << "SdwaDstUnused"; break;
649     case ImmTyDMask: OS << "DMask"; break;
650     case ImmTyUNorm: OS << "UNorm"; break;
651     case ImmTyDA: OS << "DA"; break;
652     case ImmTyR128: OS << "R128"; break;
653     case ImmTyLWE: OS << "LWE"; break;
654     case ImmTyOff: OS << "Off"; break;
655     case ImmTyExpTgt: OS << "ExpTgt"; break;
656     case ImmTyExpCompr: OS << "ExpCompr"; break;
657     case ImmTyExpVM: OS << "ExpVM"; break;
658     case ImmTyHwreg: OS << "Hwreg"; break;
659     case ImmTySendMsg: OS << "SendMsg"; break;
660     case ImmTyInterpSlot: OS << "InterpSlot"; break;
661     case ImmTyInterpAttr: OS << "InterpAttr"; break;
662     case ImmTyAttrChan: OS << "AttrChan"; break;
663     case ImmTyOpSel: OS << "OpSel"; break;
664     case ImmTyOpSelHi: OS << "OpSelHi"; break;
665     case ImmTyNegLo: OS << "NegLo"; break;
666     case ImmTyNegHi: OS << "NegHi"; break;
667     case ImmTySwizzle: OS << "Swizzle"; break;
668     }
669   }
670 
671   void print(raw_ostream &OS) const override {
672     switch (Kind) {
673     case Register:
674       OS << "<register " << getReg() << " mods: " << Reg.Mods << '>';
675       break;
676     case Immediate:
677       OS << '<' << getImm();
678       if (getImmTy() != ImmTyNone) {
679         OS << " type: "; printImmTy(OS, getImmTy());
680       }
681       OS << " mods: " << Imm.Mods << '>';
682       break;
683     case Token:
684       OS << '\'' << getToken() << '\'';
685       break;
686     case Expression:
687       OS << "<expr " << *Expr << '>';
688       break;
689     }
690   }
691 
692   static AMDGPUOperand::Ptr CreateImm(const AMDGPUAsmParser *AsmParser,
693                                       int64_t Val, SMLoc Loc,
694                                       ImmTy Type = ImmTyNone,
695                                       bool IsFPImm = false) {
696     auto Op = llvm::make_unique<AMDGPUOperand>(Immediate, AsmParser);
697     Op->Imm.Val = Val;
698     Op->Imm.IsFPImm = IsFPImm;
699     Op->Imm.Type = Type;
700     Op->Imm.Mods = Modifiers();
701     Op->StartLoc = Loc;
702     Op->EndLoc = Loc;
703     return Op;
704   }
705 
706   static AMDGPUOperand::Ptr CreateToken(const AMDGPUAsmParser *AsmParser,
707                                         StringRef Str, SMLoc Loc,
708                                         bool HasExplicitEncodingSize = true) {
709     auto Res = llvm::make_unique<AMDGPUOperand>(Token, AsmParser);
710     Res->Tok.Data = Str.data();
711     Res->Tok.Length = Str.size();
712     Res->StartLoc = Loc;
713     Res->EndLoc = Loc;
714     return Res;
715   }
716 
717   static AMDGPUOperand::Ptr CreateReg(const AMDGPUAsmParser *AsmParser,
718                                       unsigned RegNo, SMLoc S,
719                                       SMLoc E,
720                                       bool ForceVOP3) {
721     auto Op = llvm::make_unique<AMDGPUOperand>(Register, AsmParser);
722     Op->Reg.RegNo = RegNo;
723     Op->Reg.Mods = Modifiers();
724     Op->Reg.IsForcedVOP3 = ForceVOP3;
725     Op->StartLoc = S;
726     Op->EndLoc = E;
727     return Op;
728   }
729 
730   static AMDGPUOperand::Ptr CreateExpr(const AMDGPUAsmParser *AsmParser,
731                                        const class MCExpr *Expr, SMLoc S) {
732     auto Op = llvm::make_unique<AMDGPUOperand>(Expression, AsmParser);
733     Op->Expr = Expr;
734     Op->StartLoc = S;
735     Op->EndLoc = S;
736     return Op;
737   }
738 };
739 
740 raw_ostream &operator <<(raw_ostream &OS, AMDGPUOperand::Modifiers Mods) {
741   OS << "abs:" << Mods.Abs << " neg: " << Mods.Neg << " sext:" << Mods.Sext;
742   return OS;
743 }
744 
745 //===----------------------------------------------------------------------===//
746 // AsmParser
747 //===----------------------------------------------------------------------===//
748 
749 // Holds info related to the current kernel, e.g. count of SGPRs used.
750 // Kernel scope begins at .amdgpu_hsa_kernel directive, ends at next
751 // .amdgpu_hsa_kernel or at EOF.
752 class KernelScopeInfo {
753   int SgprIndexUnusedMin = -1;
754   int VgprIndexUnusedMin = -1;
755   MCContext *Ctx = nullptr;
756 
757   void usesSgprAt(int i) {
758     if (i >= SgprIndexUnusedMin) {
759       SgprIndexUnusedMin = ++i;
760       if (Ctx) {
761         MCSymbol * const Sym = Ctx->getOrCreateSymbol(Twine(".kernel.sgpr_count"));
762         Sym->setVariableValue(MCConstantExpr::create(SgprIndexUnusedMin, *Ctx));
763       }
764     }
765   }
766 
767   void usesVgprAt(int i) {
768     if (i >= VgprIndexUnusedMin) {
769       VgprIndexUnusedMin = ++i;
770       if (Ctx) {
771         MCSymbol * const Sym = Ctx->getOrCreateSymbol(Twine(".kernel.vgpr_count"));
772         Sym->setVariableValue(MCConstantExpr::create(VgprIndexUnusedMin, *Ctx));
773       }
774     }
775   }
776 
777 public:
778   KernelScopeInfo() = default;
779 
780   void initialize(MCContext &Context) {
781     Ctx = &Context;
782     usesSgprAt(SgprIndexUnusedMin = -1);
783     usesVgprAt(VgprIndexUnusedMin = -1);
784   }
785 
786   void usesRegister(RegisterKind RegKind, unsigned DwordRegIndex, unsigned RegWidth) {
787     switch (RegKind) {
788       case IS_SGPR: usesSgprAt(DwordRegIndex + RegWidth - 1); break;
789       case IS_VGPR: usesVgprAt(DwordRegIndex + RegWidth - 1); break;
790       default: break;
791     }
792   }
793 };
794 
795 class AMDGPUAsmParser : public MCTargetAsmParser {
796   const MCInstrInfo &MII;
797   MCAsmParser &Parser;
798 
799   unsigned ForcedEncodingSize = 0;
800   bool ForcedDPP = false;
801   bool ForcedSDWA = false;
802   KernelScopeInfo KernelScope;
803 
804   /// @name Auto-generated Match Functions
805   /// {
806 
807 #define GET_ASSEMBLER_HEADER
808 #include "AMDGPUGenAsmMatcher.inc"
809 
810   /// }
811 
812 private:
813   bool ParseAsAbsoluteExpression(uint32_t &Ret);
814   bool ParseDirectiveMajorMinor(uint32_t &Major, uint32_t &Minor);
815   bool ParseDirectiveHSACodeObjectVersion();
816   bool ParseDirectiveHSACodeObjectISA();
817   bool ParseDirectiveCodeObjectMetadata();
818   bool ParseAMDKernelCodeTValue(StringRef ID, amd_kernel_code_t &Header);
819   bool ParseDirectiveAMDKernelCodeT();
820   bool subtargetHasRegister(const MCRegisterInfo &MRI, unsigned RegNo) const;
821   bool ParseDirectiveAMDGPUHsaKernel();
822   bool AddNextRegisterToList(unsigned& Reg, unsigned& RegWidth,
823                              RegisterKind RegKind, unsigned Reg1,
824                              unsigned RegNum);
825   bool ParseAMDGPURegister(RegisterKind& RegKind, unsigned& Reg,
826                            unsigned& RegNum, unsigned& RegWidth,
827                            unsigned *DwordRegIndex);
828   void cvtMubufImpl(MCInst &Inst, const OperandVector &Operands,
829                     bool IsAtomic, bool IsAtomicReturn);
830   void cvtDSImpl(MCInst &Inst, const OperandVector &Operands,
831                  bool IsGdsHardcoded);
832 
833 public:
834   enum AMDGPUMatchResultTy {
835     Match_PreferE32 = FIRST_TARGET_MATCH_RESULT_TY
836   };
837 
838   typedef std::map<AMDGPUOperand::ImmTy, unsigned> OptionalImmIndexMap;
839 
840   AMDGPUAsmParser(const MCSubtargetInfo &STI, MCAsmParser &_Parser,
841                const MCInstrInfo &MII,
842                const MCTargetOptions &Options)
843       : MCTargetAsmParser(Options, STI), MII(MII), Parser(_Parser) {
844     MCAsmParserExtension::Initialize(Parser);
845 
846     if (getFeatureBits().none()) {
847       // Set default features.
848       copySTI().ToggleFeature("SOUTHERN_ISLANDS");
849     }
850 
851     setAvailableFeatures(ComputeAvailableFeatures(getFeatureBits()));
852 
853     {
854       // TODO: make those pre-defined variables read-only.
855       // Currently there is none suitable machinery in the core llvm-mc for this.
856       // MCSymbol::isRedefinable is intended for another purpose, and
857       // AsmParser::parseDirectiveSet() cannot be specialized for specific target.
858       AMDGPU::IsaInfo::IsaVersion ISA =
859           AMDGPU::IsaInfo::getIsaVersion(getFeatureBits());
860       MCContext &Ctx = getContext();
861       MCSymbol *Sym =
862           Ctx.getOrCreateSymbol(Twine(".option.machine_version_major"));
863       Sym->setVariableValue(MCConstantExpr::create(ISA.Major, Ctx));
864       Sym = Ctx.getOrCreateSymbol(Twine(".option.machine_version_minor"));
865       Sym->setVariableValue(MCConstantExpr::create(ISA.Minor, Ctx));
866       Sym = Ctx.getOrCreateSymbol(Twine(".option.machine_version_stepping"));
867       Sym->setVariableValue(MCConstantExpr::create(ISA.Stepping, Ctx));
868     }
869     KernelScope.initialize(getContext());
870   }
871 
872   bool isSI() const {
873     return AMDGPU::isSI(getSTI());
874   }
875 
876   bool isCI() const {
877     return AMDGPU::isCI(getSTI());
878   }
879 
880   bool isVI() const {
881     return AMDGPU::isVI(getSTI());
882   }
883 
884   bool isGFX9() const {
885     return AMDGPU::isGFX9(getSTI());
886   }
887 
888   bool hasInv2PiInlineImm() const {
889     return getFeatureBits()[AMDGPU::FeatureInv2PiInlineImm];
890   }
891 
892   bool hasFlatOffsets() const {
893     return getFeatureBits()[AMDGPU::FeatureFlatInstOffsets];
894   }
895 
896   bool hasSGPR102_SGPR103() const {
897     return !isVI();
898   }
899 
900   AMDGPUTargetStreamer &getTargetStreamer() {
901     MCTargetStreamer &TS = *getParser().getStreamer().getTargetStreamer();
902     return static_cast<AMDGPUTargetStreamer &>(TS);
903   }
904 
905   const MCRegisterInfo *getMRI() const {
906     // We need this const_cast because for some reason getContext() is not const
907     // in MCAsmParser.
908     return const_cast<AMDGPUAsmParser*>(this)->getContext().getRegisterInfo();
909   }
910 
911   const MCInstrInfo *getMII() const {
912     return &MII;
913   }
914 
915   const FeatureBitset &getFeatureBits() const {
916     return getSTI().getFeatureBits();
917   }
918 
919   void setForcedEncodingSize(unsigned Size) { ForcedEncodingSize = Size; }
920   void setForcedDPP(bool ForceDPP_) { ForcedDPP = ForceDPP_; }
921   void setForcedSDWA(bool ForceSDWA_) { ForcedSDWA = ForceSDWA_; }
922 
923   unsigned getForcedEncodingSize() const { return ForcedEncodingSize; }
924   bool isForcedVOP3() const { return ForcedEncodingSize == 64; }
925   bool isForcedDPP() const { return ForcedDPP; }
926   bool isForcedSDWA() const { return ForcedSDWA; }
927   ArrayRef<unsigned> getMatchedVariants() const;
928 
929   std::unique_ptr<AMDGPUOperand> parseRegister();
930   bool ParseRegister(unsigned &RegNo, SMLoc &StartLoc, SMLoc &EndLoc) override;
931   unsigned checkTargetMatchPredicate(MCInst &Inst) override;
932   unsigned validateTargetOperandClass(MCParsedAsmOperand &Op,
933                                       unsigned Kind) override;
934   bool MatchAndEmitInstruction(SMLoc IDLoc, unsigned &Opcode,
935                                OperandVector &Operands, MCStreamer &Out,
936                                uint64_t &ErrorInfo,
937                                bool MatchingInlineAsm) override;
938   bool ParseDirective(AsmToken DirectiveID) override;
939   OperandMatchResultTy parseOperand(OperandVector &Operands, StringRef Mnemonic);
940   StringRef parseMnemonicSuffix(StringRef Name);
941   bool ParseInstruction(ParseInstructionInfo &Info, StringRef Name,
942                         SMLoc NameLoc, OperandVector &Operands) override;
943   //bool ProcessInstruction(MCInst &Inst);
944 
945   OperandMatchResultTy parseIntWithPrefix(const char *Prefix, int64_t &Int);
946 
947   OperandMatchResultTy
948   parseIntWithPrefix(const char *Prefix, OperandVector &Operands,
949                      AMDGPUOperand::ImmTy ImmTy = AMDGPUOperand::ImmTyNone,
950                      bool (*ConvertResult)(int64_t &) = nullptr);
951 
952   OperandMatchResultTy parseOperandArrayWithPrefix(
953     const char *Prefix,
954     OperandVector &Operands,
955     AMDGPUOperand::ImmTy ImmTy = AMDGPUOperand::ImmTyNone,
956     bool (*ConvertResult)(int64_t&) = nullptr);
957 
958   OperandMatchResultTy
959   parseNamedBit(const char *Name, OperandVector &Operands,
960                 AMDGPUOperand::ImmTy ImmTy = AMDGPUOperand::ImmTyNone);
961   OperandMatchResultTy parseStringWithPrefix(StringRef Prefix,
962                                              StringRef &Value);
963 
964   bool parseAbsoluteExpr(int64_t &Val, bool AbsMod = false);
965   OperandMatchResultTy parseImm(OperandVector &Operands, bool AbsMod = false);
966   OperandMatchResultTy parseReg(OperandVector &Operands);
967   OperandMatchResultTy parseRegOrImm(OperandVector &Operands, bool AbsMod = false);
968   OperandMatchResultTy parseRegOrImmWithFPInputMods(OperandVector &Operands, bool AllowImm = true);
969   OperandMatchResultTy parseRegOrImmWithIntInputMods(OperandVector &Operands, bool AllowImm = true);
970   OperandMatchResultTy parseRegWithFPInputMods(OperandVector &Operands);
971   OperandMatchResultTy parseRegWithIntInputMods(OperandVector &Operands);
972   OperandMatchResultTy parseVReg32OrOff(OperandVector &Operands);
973 
974   void cvtDSOffset01(MCInst &Inst, const OperandVector &Operands);
975   void cvtDS(MCInst &Inst, const OperandVector &Operands) { cvtDSImpl(Inst, Operands, false); }
976   void cvtDSGds(MCInst &Inst, const OperandVector &Operands) { cvtDSImpl(Inst, Operands, true); }
977   void cvtExp(MCInst &Inst, const OperandVector &Operands);
978 
979   bool parseCnt(int64_t &IntVal);
980   OperandMatchResultTy parseSWaitCntOps(OperandVector &Operands);
981   OperandMatchResultTy parseHwreg(OperandVector &Operands);
982 
983 private:
984   struct OperandInfoTy {
985     int64_t Id;
986     bool IsSymbolic;
987     OperandInfoTy(int64_t Id_) : Id(Id_), IsSymbolic(false) { }
988   };
989 
990   bool parseSendMsgConstruct(OperandInfoTy &Msg, OperandInfoTy &Operation, int64_t &StreamId);
991   bool parseHwregConstruct(OperandInfoTy &HwReg, int64_t &Offset, int64_t &Width);
992 
993   void errorExpTgt();
994   OperandMatchResultTy parseExpTgtImpl(StringRef Str, uint8_t &Val);
995 
996   bool validateOperandLimitations(const MCInst &Inst);
997   bool usesConstantBus(const MCInst &Inst, unsigned OpIdx);
998   bool isInlineConstant(const MCInst &Inst, unsigned OpIdx) const;
999   unsigned findImplicitSGPRReadInVOP(const MCInst &Inst) const;
1000 
1001   bool trySkipId(const StringRef Id);
1002   bool trySkipToken(const AsmToken::TokenKind Kind);
1003   bool skipToken(const AsmToken::TokenKind Kind, const StringRef ErrMsg);
1004   bool parseString(StringRef &Val, const StringRef ErrMsg = "expected a string");
1005   bool parseExpr(int64_t &Imm);
1006 
1007 public:
1008   OperandMatchResultTy parseOptionalOperand(OperandVector &Operands);
1009 
1010   OperandMatchResultTy parseExpTgt(OperandVector &Operands);
1011   OperandMatchResultTy parseSendMsgOp(OperandVector &Operands);
1012   OperandMatchResultTy parseInterpSlot(OperandVector &Operands);
1013   OperandMatchResultTy parseInterpAttr(OperandVector &Operands);
1014   OperandMatchResultTy parseSOppBrTarget(OperandVector &Operands);
1015 
1016   bool parseSwizzleOperands(const unsigned OpNum, int64_t* Op,
1017                             const unsigned MinVal,
1018                             const unsigned MaxVal,
1019                             const StringRef ErrMsg);
1020   OperandMatchResultTy parseSwizzleOp(OperandVector &Operands);
1021   bool parseSwizzleOffset(int64_t &Imm);
1022   bool parseSwizzleMacro(int64_t &Imm);
1023   bool parseSwizzleQuadPerm(int64_t &Imm);
1024   bool parseSwizzleBitmaskPerm(int64_t &Imm);
1025   bool parseSwizzleBroadcast(int64_t &Imm);
1026   bool parseSwizzleSwap(int64_t &Imm);
1027   bool parseSwizzleReverse(int64_t &Imm);
1028 
1029   void cvtMubuf(MCInst &Inst, const OperandVector &Operands) { cvtMubufImpl(Inst, Operands, false, false); }
1030   void cvtMubufAtomic(MCInst &Inst, const OperandVector &Operands) { cvtMubufImpl(Inst, Operands, true, false); }
1031   void cvtMubufAtomicReturn(MCInst &Inst, const OperandVector &Operands) { cvtMubufImpl(Inst, Operands, true, true); }
1032   AMDGPUOperand::Ptr defaultGLC() const;
1033   AMDGPUOperand::Ptr defaultSLC() const;
1034   AMDGPUOperand::Ptr defaultTFE() const;
1035 
1036   AMDGPUOperand::Ptr defaultDMask() const;
1037   AMDGPUOperand::Ptr defaultUNorm() const;
1038   AMDGPUOperand::Ptr defaultDA() const;
1039   AMDGPUOperand::Ptr defaultR128() const;
1040   AMDGPUOperand::Ptr defaultLWE() const;
1041   AMDGPUOperand::Ptr defaultSMRDOffset8() const;
1042   AMDGPUOperand::Ptr defaultSMRDOffset20() const;
1043   AMDGPUOperand::Ptr defaultSMRDLiteralOffset() const;
1044   AMDGPUOperand::Ptr defaultOffsetU12() const;
1045 
1046   OperandMatchResultTy parseOModOperand(OperandVector &Operands);
1047 
1048   void cvtId(MCInst &Inst, const OperandVector &Operands);
1049   void cvtVOP3_2_mod(MCInst &Inst, const OperandVector &Operands);
1050 
1051   void cvtVOP3Impl(MCInst &Inst,
1052                    const OperandVector &Operands,
1053                    OptionalImmIndexMap &OptionalIdx);
1054   void cvtVOP3(MCInst &Inst, const OperandVector &Operands);
1055   void cvtVOP3OMod(MCInst &Inst, const OperandVector &Operands);
1056   void cvtVOP3P(MCInst &Inst, const OperandVector &Operands);
1057 
1058   void cvtMIMG(MCInst &Inst, const OperandVector &Operands);
1059   void cvtMIMGAtomic(MCInst &Inst, const OperandVector &Operands);
1060 
1061   OperandMatchResultTy parseDPPCtrl(OperandVector &Operands);
1062   AMDGPUOperand::Ptr defaultRowMask() const;
1063   AMDGPUOperand::Ptr defaultBankMask() const;
1064   AMDGPUOperand::Ptr defaultBoundCtrl() const;
1065   void cvtDPP(MCInst &Inst, const OperandVector &Operands);
1066 
1067   OperandMatchResultTy parseSDWASel(OperandVector &Operands, StringRef Prefix,
1068                                     AMDGPUOperand::ImmTy Type);
1069   OperandMatchResultTy parseSDWADstUnused(OperandVector &Operands);
1070   void cvtSdwaVOP1(MCInst &Inst, const OperandVector &Operands);
1071   void cvtSdwaVOP2(MCInst &Inst, const OperandVector &Operands);
1072   void cvtSdwaVOP2b(MCInst &Inst, const OperandVector &Operands);
1073   void cvtSdwaVOPC(MCInst &Inst, const OperandVector &Operands);
1074   void cvtSDWA(MCInst &Inst, const OperandVector &Operands,
1075                 uint64_t BasicInstType, bool skipVcc = false);
1076 };
1077 
1078 struct OptionalOperand {
1079   const char *Name;
1080   AMDGPUOperand::ImmTy Type;
1081   bool IsBit;
1082   bool (*ConvertResult)(int64_t&);
1083 };
1084 
1085 } // end anonymous namespace
1086 
1087 // May be called with integer type with equivalent bitwidth.
1088 static const fltSemantics *getFltSemantics(unsigned Size) {
1089   switch (Size) {
1090   case 4:
1091     return &APFloat::IEEEsingle();
1092   case 8:
1093     return &APFloat::IEEEdouble();
1094   case 2:
1095     return &APFloat::IEEEhalf();
1096   default:
1097     llvm_unreachable("unsupported fp type");
1098   }
1099 }
1100 
1101 static const fltSemantics *getFltSemantics(MVT VT) {
1102   return getFltSemantics(VT.getSizeInBits() / 8);
1103 }
1104 
1105 static const fltSemantics *getOpFltSemantics(uint8_t OperandType) {
1106   switch (OperandType) {
1107   case AMDGPU::OPERAND_REG_IMM_INT32:
1108   case AMDGPU::OPERAND_REG_IMM_FP32:
1109   case AMDGPU::OPERAND_REG_INLINE_C_INT32:
1110   case AMDGPU::OPERAND_REG_INLINE_C_FP32:
1111     return &APFloat::IEEEsingle();
1112   case AMDGPU::OPERAND_REG_IMM_INT64:
1113   case AMDGPU::OPERAND_REG_IMM_FP64:
1114   case AMDGPU::OPERAND_REG_INLINE_C_INT64:
1115   case AMDGPU::OPERAND_REG_INLINE_C_FP64:
1116     return &APFloat::IEEEdouble();
1117   case AMDGPU::OPERAND_REG_IMM_INT16:
1118   case AMDGPU::OPERAND_REG_IMM_FP16:
1119   case AMDGPU::OPERAND_REG_INLINE_C_INT16:
1120   case AMDGPU::OPERAND_REG_INLINE_C_FP16:
1121   case AMDGPU::OPERAND_REG_INLINE_C_V2INT16:
1122   case AMDGPU::OPERAND_REG_INLINE_C_V2FP16:
1123     return &APFloat::IEEEhalf();
1124   default:
1125     llvm_unreachable("unsupported fp type");
1126   }
1127 }
1128 
1129 //===----------------------------------------------------------------------===//
1130 // Operand
1131 //===----------------------------------------------------------------------===//
1132 
1133 static bool canLosslesslyConvertToFPType(APFloat &FPLiteral, MVT VT) {
1134   bool Lost;
1135 
1136   // Convert literal to single precision
1137   APFloat::opStatus Status = FPLiteral.convert(*getFltSemantics(VT),
1138                                                APFloat::rmNearestTiesToEven,
1139                                                &Lost);
1140   // We allow precision lost but not overflow or underflow
1141   if (Status != APFloat::opOK &&
1142       Lost &&
1143       ((Status & APFloat::opOverflow)  != 0 ||
1144        (Status & APFloat::opUnderflow) != 0)) {
1145     return false;
1146   }
1147 
1148   return true;
1149 }
1150 
1151 bool AMDGPUOperand::isInlinableImm(MVT type) const {
1152   if (!isImmTy(ImmTyNone)) {
1153     // Only plain immediates are inlinable (e.g. "clamp" attribute is not)
1154     return false;
1155   }
1156   // TODO: We should avoid using host float here. It would be better to
1157   // check the float bit values which is what a few other places do.
1158   // We've had bot failures before due to weird NaN support on mips hosts.
1159 
1160   APInt Literal(64, Imm.Val);
1161 
1162   if (Imm.IsFPImm) { // We got fp literal token
1163     if (type == MVT::f64 || type == MVT::i64) { // Expected 64-bit operand
1164       return AMDGPU::isInlinableLiteral64(Imm.Val,
1165                                           AsmParser->hasInv2PiInlineImm());
1166     }
1167 
1168     APFloat FPLiteral(APFloat::IEEEdouble(), APInt(64, Imm.Val));
1169     if (!canLosslesslyConvertToFPType(FPLiteral, type))
1170       return false;
1171 
1172     if (type.getScalarSizeInBits() == 16) {
1173       return AMDGPU::isInlinableLiteral16(
1174         static_cast<int16_t>(FPLiteral.bitcastToAPInt().getZExtValue()),
1175         AsmParser->hasInv2PiInlineImm());
1176     }
1177 
1178     // Check if single precision literal is inlinable
1179     return AMDGPU::isInlinableLiteral32(
1180       static_cast<int32_t>(FPLiteral.bitcastToAPInt().getZExtValue()),
1181       AsmParser->hasInv2PiInlineImm());
1182   }
1183 
1184   // We got int literal token.
1185   if (type == MVT::f64 || type == MVT::i64) { // Expected 64-bit operand
1186     return AMDGPU::isInlinableLiteral64(Imm.Val,
1187                                         AsmParser->hasInv2PiInlineImm());
1188   }
1189 
1190   if (type.getScalarSizeInBits() == 16) {
1191     return AMDGPU::isInlinableLiteral16(
1192       static_cast<int16_t>(Literal.getLoBits(16).getSExtValue()),
1193       AsmParser->hasInv2PiInlineImm());
1194   }
1195 
1196   return AMDGPU::isInlinableLiteral32(
1197     static_cast<int32_t>(Literal.getLoBits(32).getZExtValue()),
1198     AsmParser->hasInv2PiInlineImm());
1199 }
1200 
1201 bool AMDGPUOperand::isLiteralImm(MVT type) const {
1202   // Check that this imediate can be added as literal
1203   if (!isImmTy(ImmTyNone)) {
1204     return false;
1205   }
1206 
1207   if (!Imm.IsFPImm) {
1208     // We got int literal token.
1209 
1210     if (type == MVT::f64 && hasFPModifiers()) {
1211       // Cannot apply fp modifiers to int literals preserving the same semantics
1212       // for VOP1/2/C and VOP3 because of integer truncation. To avoid ambiguity,
1213       // disable these cases.
1214       return false;
1215     }
1216 
1217     unsigned Size = type.getSizeInBits();
1218     if (Size == 64)
1219       Size = 32;
1220 
1221     // FIXME: 64-bit operands can zero extend, sign extend, or pad zeroes for FP
1222     // types.
1223     return isUIntN(Size, Imm.Val) || isIntN(Size, Imm.Val);
1224   }
1225 
1226   // We got fp literal token
1227   if (type == MVT::f64) { // Expected 64-bit fp operand
1228     // We would set low 64-bits of literal to zeroes but we accept this literals
1229     return true;
1230   }
1231 
1232   if (type == MVT::i64) { // Expected 64-bit int operand
1233     // We don't allow fp literals in 64-bit integer instructions. It is
1234     // unclear how we should encode them.
1235     return false;
1236   }
1237 
1238   APFloat FPLiteral(APFloat::IEEEdouble(), APInt(64, Imm.Val));
1239   return canLosslesslyConvertToFPType(FPLiteral, type);
1240 }
1241 
1242 bool AMDGPUOperand::isRegClass(unsigned RCID) const {
1243   return isRegKind() && AsmParser->getMRI()->getRegClass(RCID).contains(getReg());
1244 }
1245 
1246 uint64_t AMDGPUOperand::applyInputFPModifiers(uint64_t Val, unsigned Size) const
1247 {
1248   assert(isImmTy(ImmTyNone) && Imm.Mods.hasFPModifiers());
1249   assert(Size == 2 || Size == 4 || Size == 8);
1250 
1251   const uint64_t FpSignMask = (1ULL << (Size * 8 - 1));
1252 
1253   if (Imm.Mods.Abs) {
1254     Val &= ~FpSignMask;
1255   }
1256   if (Imm.Mods.Neg) {
1257     Val ^= FpSignMask;
1258   }
1259 
1260   return Val;
1261 }
1262 
1263 void AMDGPUOperand::addImmOperands(MCInst &Inst, unsigned N, bool ApplyModifiers) const {
1264 
1265   if (AMDGPU::isSISrcOperand(AsmParser->getMII()->get(Inst.getOpcode()),
1266                              Inst.getNumOperands())) {
1267     addLiteralImmOperand(Inst, Imm.Val,
1268                          ApplyModifiers &
1269                          isImmTy(ImmTyNone) && Imm.Mods.hasFPModifiers());
1270   } else {
1271     assert(!isImmTy(ImmTyNone) || !hasModifiers());
1272     Inst.addOperand(MCOperand::createImm(Imm.Val));
1273   }
1274 }
1275 
1276 void AMDGPUOperand::addLiteralImmOperand(MCInst &Inst, int64_t Val, bool ApplyModifiers) const {
1277   const auto& InstDesc = AsmParser->getMII()->get(Inst.getOpcode());
1278   auto OpNum = Inst.getNumOperands();
1279   // Check that this operand accepts literals
1280   assert(AMDGPU::isSISrcOperand(InstDesc, OpNum));
1281 
1282   if (ApplyModifiers) {
1283     assert(AMDGPU::isSISrcFPOperand(InstDesc, OpNum));
1284     const unsigned Size = Imm.IsFPImm ? sizeof(double) : getOperandSize(InstDesc, OpNum);
1285     Val = applyInputFPModifiers(Val, Size);
1286   }
1287 
1288   APInt Literal(64, Val);
1289   uint8_t OpTy = InstDesc.OpInfo[OpNum].OperandType;
1290 
1291   if (Imm.IsFPImm) { // We got fp literal token
1292     switch (OpTy) {
1293     case AMDGPU::OPERAND_REG_IMM_INT64:
1294     case AMDGPU::OPERAND_REG_IMM_FP64:
1295     case AMDGPU::OPERAND_REG_INLINE_C_INT64:
1296     case AMDGPU::OPERAND_REG_INLINE_C_FP64: {
1297       if (AMDGPU::isInlinableLiteral64(Literal.getZExtValue(),
1298                                        AsmParser->hasInv2PiInlineImm())) {
1299         Inst.addOperand(MCOperand::createImm(Literal.getZExtValue()));
1300         return;
1301       }
1302 
1303       // Non-inlineable
1304       if (AMDGPU::isSISrcFPOperand(InstDesc, OpNum)) { // Expected 64-bit fp operand
1305         // For fp operands we check if low 32 bits are zeros
1306         if (Literal.getLoBits(32) != 0) {
1307           const_cast<AMDGPUAsmParser *>(AsmParser)->Warning(Inst.getLoc(),
1308           "Can't encode literal as exact 64-bit floating-point operand. "
1309           "Low 32-bits will be set to zero");
1310         }
1311 
1312         Inst.addOperand(MCOperand::createImm(Literal.lshr(32).getZExtValue()));
1313         return;
1314       }
1315 
1316       // We don't allow fp literals in 64-bit integer instructions. It is
1317       // unclear how we should encode them. This case should be checked earlier
1318       // in predicate methods (isLiteralImm())
1319       llvm_unreachable("fp literal in 64-bit integer instruction.");
1320     }
1321     case AMDGPU::OPERAND_REG_IMM_INT32:
1322     case AMDGPU::OPERAND_REG_IMM_FP32:
1323     case AMDGPU::OPERAND_REG_INLINE_C_INT32:
1324     case AMDGPU::OPERAND_REG_INLINE_C_FP32:
1325     case AMDGPU::OPERAND_REG_IMM_INT16:
1326     case AMDGPU::OPERAND_REG_IMM_FP16:
1327     case AMDGPU::OPERAND_REG_INLINE_C_INT16:
1328     case AMDGPU::OPERAND_REG_INLINE_C_FP16:
1329     case AMDGPU::OPERAND_REG_INLINE_C_V2INT16:
1330     case AMDGPU::OPERAND_REG_INLINE_C_V2FP16: {
1331       bool lost;
1332       APFloat FPLiteral(APFloat::IEEEdouble(), Literal);
1333       // Convert literal to single precision
1334       FPLiteral.convert(*getOpFltSemantics(OpTy),
1335                         APFloat::rmNearestTiesToEven, &lost);
1336       // We allow precision lost but not overflow or underflow. This should be
1337       // checked earlier in isLiteralImm()
1338 
1339       uint64_t ImmVal = FPLiteral.bitcastToAPInt().getZExtValue();
1340       if (OpTy == AMDGPU::OPERAND_REG_INLINE_C_V2INT16 ||
1341           OpTy == AMDGPU::OPERAND_REG_INLINE_C_V2FP16) {
1342         ImmVal |= (ImmVal << 16);
1343       }
1344 
1345       Inst.addOperand(MCOperand::createImm(ImmVal));
1346       return;
1347     }
1348     default:
1349       llvm_unreachable("invalid operand size");
1350     }
1351 
1352     return;
1353   }
1354 
1355    // We got int literal token.
1356   // Only sign extend inline immediates.
1357   // FIXME: No errors on truncation
1358   switch (OpTy) {
1359   case AMDGPU::OPERAND_REG_IMM_INT32:
1360   case AMDGPU::OPERAND_REG_IMM_FP32:
1361   case AMDGPU::OPERAND_REG_INLINE_C_INT32:
1362   case AMDGPU::OPERAND_REG_INLINE_C_FP32: {
1363     if (isInt<32>(Val) &&
1364         AMDGPU::isInlinableLiteral32(static_cast<int32_t>(Val),
1365                                      AsmParser->hasInv2PiInlineImm())) {
1366       Inst.addOperand(MCOperand::createImm(Val));
1367       return;
1368     }
1369 
1370     Inst.addOperand(MCOperand::createImm(Val & 0xffffffff));
1371     return;
1372   }
1373   case AMDGPU::OPERAND_REG_IMM_INT64:
1374   case AMDGPU::OPERAND_REG_IMM_FP64:
1375   case AMDGPU::OPERAND_REG_INLINE_C_INT64:
1376   case AMDGPU::OPERAND_REG_INLINE_C_FP64: {
1377     if (AMDGPU::isInlinableLiteral64(Val, AsmParser->hasInv2PiInlineImm())) {
1378       Inst.addOperand(MCOperand::createImm(Val));
1379       return;
1380     }
1381 
1382     Inst.addOperand(MCOperand::createImm(Lo_32(Val)));
1383     return;
1384   }
1385   case AMDGPU::OPERAND_REG_IMM_INT16:
1386   case AMDGPU::OPERAND_REG_IMM_FP16:
1387   case AMDGPU::OPERAND_REG_INLINE_C_INT16:
1388   case AMDGPU::OPERAND_REG_INLINE_C_FP16: {
1389     if (isInt<16>(Val) &&
1390         AMDGPU::isInlinableLiteral16(static_cast<int16_t>(Val),
1391                                      AsmParser->hasInv2PiInlineImm())) {
1392       Inst.addOperand(MCOperand::createImm(Val));
1393       return;
1394     }
1395 
1396     Inst.addOperand(MCOperand::createImm(Val & 0xffff));
1397     return;
1398   }
1399   case AMDGPU::OPERAND_REG_INLINE_C_V2INT16:
1400   case AMDGPU::OPERAND_REG_INLINE_C_V2FP16: {
1401     auto LiteralVal = static_cast<uint16_t>(Literal.getLoBits(16).getZExtValue());
1402     assert(AMDGPU::isInlinableLiteral16(LiteralVal,
1403                                         AsmParser->hasInv2PiInlineImm()));
1404 
1405     uint32_t ImmVal = static_cast<uint32_t>(LiteralVal) << 16 |
1406                       static_cast<uint32_t>(LiteralVal);
1407     Inst.addOperand(MCOperand::createImm(ImmVal));
1408     return;
1409   }
1410   default:
1411     llvm_unreachable("invalid operand size");
1412   }
1413 }
1414 
1415 template <unsigned Bitwidth>
1416 void AMDGPUOperand::addKImmFPOperands(MCInst &Inst, unsigned N) const {
1417   APInt Literal(64, Imm.Val);
1418 
1419   if (!Imm.IsFPImm) {
1420     // We got int literal token.
1421     Inst.addOperand(MCOperand::createImm(Literal.getLoBits(Bitwidth).getZExtValue()));
1422     return;
1423   }
1424 
1425   bool Lost;
1426   APFloat FPLiteral(APFloat::IEEEdouble(), Literal);
1427   FPLiteral.convert(*getFltSemantics(Bitwidth / 8),
1428                     APFloat::rmNearestTiesToEven, &Lost);
1429   Inst.addOperand(MCOperand::createImm(FPLiteral.bitcastToAPInt().getZExtValue()));
1430 }
1431 
1432 void AMDGPUOperand::addRegOperands(MCInst &Inst, unsigned N) const {
1433   Inst.addOperand(MCOperand::createReg(AMDGPU::getMCReg(getReg(), AsmParser->getSTI())));
1434 }
1435 
1436 //===----------------------------------------------------------------------===//
1437 // AsmParser
1438 //===----------------------------------------------------------------------===//
1439 
1440 static int getRegClass(RegisterKind Is, unsigned RegWidth) {
1441   if (Is == IS_VGPR) {
1442     switch (RegWidth) {
1443       default: return -1;
1444       case 1: return AMDGPU::VGPR_32RegClassID;
1445       case 2: return AMDGPU::VReg_64RegClassID;
1446       case 3: return AMDGPU::VReg_96RegClassID;
1447       case 4: return AMDGPU::VReg_128RegClassID;
1448       case 8: return AMDGPU::VReg_256RegClassID;
1449       case 16: return AMDGPU::VReg_512RegClassID;
1450     }
1451   } else if (Is == IS_TTMP) {
1452     switch (RegWidth) {
1453       default: return -1;
1454       case 1: return AMDGPU::TTMP_32RegClassID;
1455       case 2: return AMDGPU::TTMP_64RegClassID;
1456       case 4: return AMDGPU::TTMP_128RegClassID;
1457     }
1458   } else if (Is == IS_SGPR) {
1459     switch (RegWidth) {
1460       default: return -1;
1461       case 1: return AMDGPU::SGPR_32RegClassID;
1462       case 2: return AMDGPU::SGPR_64RegClassID;
1463       case 4: return AMDGPU::SGPR_128RegClassID;
1464       case 8: return AMDGPU::SReg_256RegClassID;
1465       case 16: return AMDGPU::SReg_512RegClassID;
1466     }
1467   }
1468   return -1;
1469 }
1470 
1471 static unsigned getSpecialRegForName(StringRef RegName) {
1472   return StringSwitch<unsigned>(RegName)
1473     .Case("exec", AMDGPU::EXEC)
1474     .Case("vcc", AMDGPU::VCC)
1475     .Case("flat_scratch", AMDGPU::FLAT_SCR)
1476     .Case("m0", AMDGPU::M0)
1477     .Case("scc", AMDGPU::SCC)
1478     .Case("tba", AMDGPU::TBA)
1479     .Case("tma", AMDGPU::TMA)
1480     .Case("flat_scratch_lo", AMDGPU::FLAT_SCR_LO)
1481     .Case("flat_scratch_hi", AMDGPU::FLAT_SCR_HI)
1482     .Case("vcc_lo", AMDGPU::VCC_LO)
1483     .Case("vcc_hi", AMDGPU::VCC_HI)
1484     .Case("exec_lo", AMDGPU::EXEC_LO)
1485     .Case("exec_hi", AMDGPU::EXEC_HI)
1486     .Case("tma_lo", AMDGPU::TMA_LO)
1487     .Case("tma_hi", AMDGPU::TMA_HI)
1488     .Case("tba_lo", AMDGPU::TBA_LO)
1489     .Case("tba_hi", AMDGPU::TBA_HI)
1490     .Default(0);
1491 }
1492 
1493 bool AMDGPUAsmParser::ParseRegister(unsigned &RegNo, SMLoc &StartLoc,
1494                                     SMLoc &EndLoc) {
1495   auto R = parseRegister();
1496   if (!R) return true;
1497   assert(R->isReg());
1498   RegNo = R->getReg();
1499   StartLoc = R->getStartLoc();
1500   EndLoc = R->getEndLoc();
1501   return false;
1502 }
1503 
1504 bool AMDGPUAsmParser::AddNextRegisterToList(unsigned &Reg, unsigned &RegWidth,
1505                                             RegisterKind RegKind, unsigned Reg1,
1506                                             unsigned RegNum) {
1507   switch (RegKind) {
1508   case IS_SPECIAL:
1509     if (Reg == AMDGPU::EXEC_LO && Reg1 == AMDGPU::EXEC_HI) {
1510       Reg = AMDGPU::EXEC;
1511       RegWidth = 2;
1512       return true;
1513     }
1514     if (Reg == AMDGPU::FLAT_SCR_LO && Reg1 == AMDGPU::FLAT_SCR_HI) {
1515       Reg = AMDGPU::FLAT_SCR;
1516       RegWidth = 2;
1517       return true;
1518     }
1519     if (Reg == AMDGPU::VCC_LO && Reg1 == AMDGPU::VCC_HI) {
1520       Reg = AMDGPU::VCC;
1521       RegWidth = 2;
1522       return true;
1523     }
1524     if (Reg == AMDGPU::TBA_LO && Reg1 == AMDGPU::TBA_HI) {
1525       Reg = AMDGPU::TBA;
1526       RegWidth = 2;
1527       return true;
1528     }
1529     if (Reg == AMDGPU::TMA_LO && Reg1 == AMDGPU::TMA_HI) {
1530       Reg = AMDGPU::TMA;
1531       RegWidth = 2;
1532       return true;
1533     }
1534     return false;
1535   case IS_VGPR:
1536   case IS_SGPR:
1537   case IS_TTMP:
1538     if (Reg1 != Reg + RegWidth) {
1539       return false;
1540     }
1541     RegWidth++;
1542     return true;
1543   default:
1544     llvm_unreachable("unexpected register kind");
1545   }
1546 }
1547 
1548 bool AMDGPUAsmParser::ParseAMDGPURegister(RegisterKind &RegKind, unsigned &Reg,
1549                                           unsigned &RegNum, unsigned &RegWidth,
1550                                           unsigned *DwordRegIndex) {
1551   if (DwordRegIndex) { *DwordRegIndex = 0; }
1552   const MCRegisterInfo *TRI = getContext().getRegisterInfo();
1553   if (getLexer().is(AsmToken::Identifier)) {
1554     StringRef RegName = Parser.getTok().getString();
1555     if ((Reg = getSpecialRegForName(RegName))) {
1556       Parser.Lex();
1557       RegKind = IS_SPECIAL;
1558     } else {
1559       unsigned RegNumIndex = 0;
1560       if (RegName[0] == 'v') {
1561         RegNumIndex = 1;
1562         RegKind = IS_VGPR;
1563       } else if (RegName[0] == 's') {
1564         RegNumIndex = 1;
1565         RegKind = IS_SGPR;
1566       } else if (RegName.startswith("ttmp")) {
1567         RegNumIndex = strlen("ttmp");
1568         RegKind = IS_TTMP;
1569       } else {
1570         return false;
1571       }
1572       if (RegName.size() > RegNumIndex) {
1573         // Single 32-bit register: vXX.
1574         if (RegName.substr(RegNumIndex).getAsInteger(10, RegNum))
1575           return false;
1576         Parser.Lex();
1577         RegWidth = 1;
1578       } else {
1579         // Range of registers: v[XX:YY]. ":YY" is optional.
1580         Parser.Lex();
1581         int64_t RegLo, RegHi;
1582         if (getLexer().isNot(AsmToken::LBrac))
1583           return false;
1584         Parser.Lex();
1585 
1586         if (getParser().parseAbsoluteExpression(RegLo))
1587           return false;
1588 
1589         const bool isRBrace = getLexer().is(AsmToken::RBrac);
1590         if (!isRBrace && getLexer().isNot(AsmToken::Colon))
1591           return false;
1592         Parser.Lex();
1593 
1594         if (isRBrace) {
1595           RegHi = RegLo;
1596         } else {
1597           if (getParser().parseAbsoluteExpression(RegHi))
1598             return false;
1599 
1600           if (getLexer().isNot(AsmToken::RBrac))
1601             return false;
1602           Parser.Lex();
1603         }
1604         RegNum = (unsigned) RegLo;
1605         RegWidth = (RegHi - RegLo) + 1;
1606       }
1607     }
1608   } else if (getLexer().is(AsmToken::LBrac)) {
1609     // List of consecutive registers: [s0,s1,s2,s3]
1610     Parser.Lex();
1611     if (!ParseAMDGPURegister(RegKind, Reg, RegNum, RegWidth, nullptr))
1612       return false;
1613     if (RegWidth != 1)
1614       return false;
1615     RegisterKind RegKind1;
1616     unsigned Reg1, RegNum1, RegWidth1;
1617     do {
1618       if (getLexer().is(AsmToken::Comma)) {
1619         Parser.Lex();
1620       } else if (getLexer().is(AsmToken::RBrac)) {
1621         Parser.Lex();
1622         break;
1623       } else if (ParseAMDGPURegister(RegKind1, Reg1, RegNum1, RegWidth1, nullptr)) {
1624         if (RegWidth1 != 1) {
1625           return false;
1626         }
1627         if (RegKind1 != RegKind) {
1628           return false;
1629         }
1630         if (!AddNextRegisterToList(Reg, RegWidth, RegKind1, Reg1, RegNum1)) {
1631           return false;
1632         }
1633       } else {
1634         return false;
1635       }
1636     } while (true);
1637   } else {
1638     return false;
1639   }
1640   switch (RegKind) {
1641   case IS_SPECIAL:
1642     RegNum = 0;
1643     RegWidth = 1;
1644     break;
1645   case IS_VGPR:
1646   case IS_SGPR:
1647   case IS_TTMP:
1648   {
1649     unsigned Size = 1;
1650     if (RegKind == IS_SGPR || RegKind == IS_TTMP) {
1651       // SGPR and TTMP registers must be aligned. Max required alignment is 4 dwords.
1652       Size = std::min(RegWidth, 4u);
1653     }
1654     if (RegNum % Size != 0)
1655       return false;
1656     if (DwordRegIndex) { *DwordRegIndex = RegNum; }
1657     RegNum = RegNum / Size;
1658     int RCID = getRegClass(RegKind, RegWidth);
1659     if (RCID == -1)
1660       return false;
1661     const MCRegisterClass RC = TRI->getRegClass(RCID);
1662     if (RegNum >= RC.getNumRegs())
1663       return false;
1664     Reg = RC.getRegister(RegNum);
1665     break;
1666   }
1667 
1668   default:
1669     llvm_unreachable("unexpected register kind");
1670   }
1671 
1672   if (!subtargetHasRegister(*TRI, Reg))
1673     return false;
1674   return true;
1675 }
1676 
1677 std::unique_ptr<AMDGPUOperand> AMDGPUAsmParser::parseRegister() {
1678   const auto &Tok = Parser.getTok();
1679   SMLoc StartLoc = Tok.getLoc();
1680   SMLoc EndLoc = Tok.getEndLoc();
1681   RegisterKind RegKind;
1682   unsigned Reg, RegNum, RegWidth, DwordRegIndex;
1683 
1684   if (!ParseAMDGPURegister(RegKind, Reg, RegNum, RegWidth, &DwordRegIndex)) {
1685     return nullptr;
1686   }
1687   KernelScope.usesRegister(RegKind, DwordRegIndex, RegWidth);
1688   return AMDGPUOperand::CreateReg(this, Reg, StartLoc, EndLoc, false);
1689 }
1690 
1691 bool
1692 AMDGPUAsmParser::parseAbsoluteExpr(int64_t &Val, bool AbsMod) {
1693   if (AbsMod && getLexer().peekTok().is(AsmToken::Pipe) &&
1694       (getLexer().getKind() == AsmToken::Integer ||
1695        getLexer().getKind() == AsmToken::Real)) {
1696 
1697     // This is a workaround for handling operands like these:
1698     //     |1.0|
1699     //     |-1|
1700     // This syntax is not compatible with syntax of standard
1701     // MC expressions (due to the trailing '|').
1702 
1703     SMLoc EndLoc;
1704     const MCExpr *Expr;
1705 
1706     if (getParser().parsePrimaryExpr(Expr, EndLoc)) {
1707       return true;
1708     }
1709 
1710     return !Expr->evaluateAsAbsolute(Val);
1711   }
1712 
1713   return getParser().parseAbsoluteExpression(Val);
1714 }
1715 
1716 OperandMatchResultTy
1717 AMDGPUAsmParser::parseImm(OperandVector &Operands, bool AbsMod) {
1718   // TODO: add syntactic sugar for 1/(2*PI)
1719   bool Minus = false;
1720   if (getLexer().getKind() == AsmToken::Minus) {
1721     Minus = true;
1722     Parser.Lex();
1723   }
1724 
1725   SMLoc S = Parser.getTok().getLoc();
1726   switch(getLexer().getKind()) {
1727   case AsmToken::Integer: {
1728     int64_t IntVal;
1729     if (parseAbsoluteExpr(IntVal, AbsMod))
1730       return MatchOperand_ParseFail;
1731     if (Minus)
1732       IntVal *= -1;
1733     Operands.push_back(AMDGPUOperand::CreateImm(this, IntVal, S));
1734     return MatchOperand_Success;
1735   }
1736   case AsmToken::Real: {
1737     int64_t IntVal;
1738     if (parseAbsoluteExpr(IntVal, AbsMod))
1739       return MatchOperand_ParseFail;
1740 
1741     APFloat F(BitsToDouble(IntVal));
1742     if (Minus)
1743       F.changeSign();
1744     Operands.push_back(
1745         AMDGPUOperand::CreateImm(this, F.bitcastToAPInt().getZExtValue(), S,
1746                                  AMDGPUOperand::ImmTyNone, true));
1747     return MatchOperand_Success;
1748   }
1749   default:
1750     return Minus ? MatchOperand_ParseFail : MatchOperand_NoMatch;
1751   }
1752 }
1753 
1754 OperandMatchResultTy
1755 AMDGPUAsmParser::parseReg(OperandVector &Operands) {
1756   if (auto R = parseRegister()) {
1757     assert(R->isReg());
1758     R->Reg.IsForcedVOP3 = isForcedVOP3();
1759     Operands.push_back(std::move(R));
1760     return MatchOperand_Success;
1761   }
1762   return MatchOperand_NoMatch;
1763 }
1764 
1765 OperandMatchResultTy
1766 AMDGPUAsmParser::parseRegOrImm(OperandVector &Operands, bool AbsMod) {
1767   auto res = parseImm(Operands, AbsMod);
1768   if (res != MatchOperand_NoMatch) {
1769     return res;
1770   }
1771 
1772   return parseReg(Operands);
1773 }
1774 
1775 OperandMatchResultTy
1776 AMDGPUAsmParser::parseRegOrImmWithFPInputMods(OperandVector &Operands,
1777                                               bool AllowImm) {
1778   bool Negate = false, Negate2 = false, Abs = false, Abs2 = false;
1779 
1780   if (getLexer().getKind()== AsmToken::Minus) {
1781     const AsmToken NextToken = getLexer().peekTok();
1782 
1783     // Disable ambiguous constructs like '--1' etc. Should use neg(-1) instead.
1784     if (NextToken.is(AsmToken::Minus)) {
1785       Error(Parser.getTok().getLoc(), "invalid syntax, expected 'neg' modifier");
1786       return MatchOperand_ParseFail;
1787     }
1788 
1789     // '-' followed by an integer literal N should be interpreted as integer
1790     // negation rather than a floating-point NEG modifier applied to N.
1791     // Beside being contr-intuitive, such use of floating-point NEG modifier
1792     // results in different meaning of integer literals used with VOP1/2/C
1793     // and VOP3, for example:
1794     //    v_exp_f32_e32 v5, -1 // VOP1: src0 = 0xFFFFFFFF
1795     //    v_exp_f32_e64 v5, -1 // VOP3: src0 = 0x80000001
1796     // Negative fp literals should be handled likewise for unifomtity
1797     if (!NextToken.is(AsmToken::Integer) && !NextToken.is(AsmToken::Real)) {
1798       Parser.Lex();
1799       Negate = true;
1800     }
1801   }
1802 
1803   if (getLexer().getKind() == AsmToken::Identifier &&
1804       Parser.getTok().getString() == "neg") {
1805     if (Negate) {
1806       Error(Parser.getTok().getLoc(), "expected register or immediate");
1807       return MatchOperand_ParseFail;
1808     }
1809     Parser.Lex();
1810     Negate2 = true;
1811     if (getLexer().isNot(AsmToken::LParen)) {
1812       Error(Parser.getTok().getLoc(), "expected left paren after neg");
1813       return MatchOperand_ParseFail;
1814     }
1815     Parser.Lex();
1816   }
1817 
1818   if (getLexer().getKind() == AsmToken::Identifier &&
1819       Parser.getTok().getString() == "abs") {
1820     Parser.Lex();
1821     Abs2 = true;
1822     if (getLexer().isNot(AsmToken::LParen)) {
1823       Error(Parser.getTok().getLoc(), "expected left paren after abs");
1824       return MatchOperand_ParseFail;
1825     }
1826     Parser.Lex();
1827   }
1828 
1829   if (getLexer().getKind() == AsmToken::Pipe) {
1830     if (Abs2) {
1831       Error(Parser.getTok().getLoc(), "expected register or immediate");
1832       return MatchOperand_ParseFail;
1833     }
1834     Parser.Lex();
1835     Abs = true;
1836   }
1837 
1838   OperandMatchResultTy Res;
1839   if (AllowImm) {
1840     Res = parseRegOrImm(Operands, Abs);
1841   } else {
1842     Res = parseReg(Operands);
1843   }
1844   if (Res != MatchOperand_Success) {
1845     return Res;
1846   }
1847 
1848   AMDGPUOperand::Modifiers Mods;
1849   if (Abs) {
1850     if (getLexer().getKind() != AsmToken::Pipe) {
1851       Error(Parser.getTok().getLoc(), "expected vertical bar");
1852       return MatchOperand_ParseFail;
1853     }
1854     Parser.Lex();
1855     Mods.Abs = true;
1856   }
1857   if (Abs2) {
1858     if (getLexer().isNot(AsmToken::RParen)) {
1859       Error(Parser.getTok().getLoc(), "expected closing parentheses");
1860       return MatchOperand_ParseFail;
1861     }
1862     Parser.Lex();
1863     Mods.Abs = true;
1864   }
1865 
1866   if (Negate) {
1867     Mods.Neg = true;
1868   } else if (Negate2) {
1869     if (getLexer().isNot(AsmToken::RParen)) {
1870       Error(Parser.getTok().getLoc(), "expected closing parentheses");
1871       return MatchOperand_ParseFail;
1872     }
1873     Parser.Lex();
1874     Mods.Neg = true;
1875   }
1876 
1877   if (Mods.hasFPModifiers()) {
1878     AMDGPUOperand &Op = static_cast<AMDGPUOperand &>(*Operands.back());
1879     Op.setModifiers(Mods);
1880   }
1881   return MatchOperand_Success;
1882 }
1883 
1884 OperandMatchResultTy
1885 AMDGPUAsmParser::parseRegOrImmWithIntInputMods(OperandVector &Operands,
1886                                                bool AllowImm) {
1887   bool Sext = false;
1888 
1889   if (getLexer().getKind() == AsmToken::Identifier &&
1890       Parser.getTok().getString() == "sext") {
1891     Parser.Lex();
1892     Sext = true;
1893     if (getLexer().isNot(AsmToken::LParen)) {
1894       Error(Parser.getTok().getLoc(), "expected left paren after sext");
1895       return MatchOperand_ParseFail;
1896     }
1897     Parser.Lex();
1898   }
1899 
1900   OperandMatchResultTy Res;
1901   if (AllowImm) {
1902     Res = parseRegOrImm(Operands);
1903   } else {
1904     Res = parseReg(Operands);
1905   }
1906   if (Res != MatchOperand_Success) {
1907     return Res;
1908   }
1909 
1910   AMDGPUOperand::Modifiers Mods;
1911   if (Sext) {
1912     if (getLexer().isNot(AsmToken::RParen)) {
1913       Error(Parser.getTok().getLoc(), "expected closing parentheses");
1914       return MatchOperand_ParseFail;
1915     }
1916     Parser.Lex();
1917     Mods.Sext = true;
1918   }
1919 
1920   if (Mods.hasIntModifiers()) {
1921     AMDGPUOperand &Op = static_cast<AMDGPUOperand &>(*Operands.back());
1922     Op.setModifiers(Mods);
1923   }
1924 
1925   return MatchOperand_Success;
1926 }
1927 
1928 OperandMatchResultTy
1929 AMDGPUAsmParser::parseRegWithFPInputMods(OperandVector &Operands) {
1930   return parseRegOrImmWithFPInputMods(Operands, false);
1931 }
1932 
1933 OperandMatchResultTy
1934 AMDGPUAsmParser::parseRegWithIntInputMods(OperandVector &Operands) {
1935   return parseRegOrImmWithIntInputMods(Operands, false);
1936 }
1937 
1938 OperandMatchResultTy AMDGPUAsmParser::parseVReg32OrOff(OperandVector &Operands) {
1939   std::unique_ptr<AMDGPUOperand> Reg = parseRegister();
1940   if (Reg) {
1941     Operands.push_back(std::move(Reg));
1942     return MatchOperand_Success;
1943   }
1944 
1945   const AsmToken &Tok = Parser.getTok();
1946   if (Tok.getString() == "off") {
1947     Operands.push_back(AMDGPUOperand::CreateImm(this, 0, Tok.getLoc(),
1948                                                 AMDGPUOperand::ImmTyOff, false));
1949     Parser.Lex();
1950     return MatchOperand_Success;
1951   }
1952 
1953   return MatchOperand_NoMatch;
1954 }
1955 
1956 unsigned AMDGPUAsmParser::checkTargetMatchPredicate(MCInst &Inst) {
1957   uint64_t TSFlags = MII.get(Inst.getOpcode()).TSFlags;
1958 
1959   if ((getForcedEncodingSize() == 32 && (TSFlags & SIInstrFlags::VOP3)) ||
1960       (getForcedEncodingSize() == 64 && !(TSFlags & SIInstrFlags::VOP3)) ||
1961       (isForcedDPP() && !(TSFlags & SIInstrFlags::DPP)) ||
1962       (isForcedSDWA() && !(TSFlags & SIInstrFlags::SDWA)) )
1963     return Match_InvalidOperand;
1964 
1965   if ((TSFlags & SIInstrFlags::VOP3) &&
1966       (TSFlags & SIInstrFlags::VOPAsmPrefer32Bit) &&
1967       getForcedEncodingSize() != 64)
1968     return Match_PreferE32;
1969 
1970   if (Inst.getOpcode() == AMDGPU::V_MAC_F32_sdwa_vi ||
1971       Inst.getOpcode() == AMDGPU::V_MAC_F16_sdwa_vi) {
1972     // v_mac_f32/16 allow only dst_sel == DWORD;
1973     auto OpNum =
1974         AMDGPU::getNamedOperandIdx(Inst.getOpcode(), AMDGPU::OpName::dst_sel);
1975     const auto &Op = Inst.getOperand(OpNum);
1976     if (!Op.isImm() || Op.getImm() != AMDGPU::SDWA::SdwaSel::DWORD) {
1977       return Match_InvalidOperand;
1978     }
1979   }
1980 
1981   if ((TSFlags & SIInstrFlags::FLAT) && !hasFlatOffsets()) {
1982     // FIXME: Produces error without correct column reported.
1983     auto OpNum =
1984         AMDGPU::getNamedOperandIdx(Inst.getOpcode(), AMDGPU::OpName::offset);
1985     const auto &Op = Inst.getOperand(OpNum);
1986     if (Op.getImm() != 0)
1987       return Match_InvalidOperand;
1988   }
1989 
1990   return Match_Success;
1991 }
1992 
1993 // What asm variants we should check
1994 ArrayRef<unsigned> AMDGPUAsmParser::getMatchedVariants() const {
1995   if (getForcedEncodingSize() == 32) {
1996     static const unsigned Variants[] = {AMDGPUAsmVariants::DEFAULT};
1997     return makeArrayRef(Variants);
1998   }
1999 
2000   if (isForcedVOP3()) {
2001     static const unsigned Variants[] = {AMDGPUAsmVariants::VOP3};
2002     return makeArrayRef(Variants);
2003   }
2004 
2005   if (isForcedSDWA()) {
2006     static const unsigned Variants[] = {AMDGPUAsmVariants::SDWA,
2007                                         AMDGPUAsmVariants::SDWA9};
2008     return makeArrayRef(Variants);
2009   }
2010 
2011   if (isForcedDPP()) {
2012     static const unsigned Variants[] = {AMDGPUAsmVariants::DPP};
2013     return makeArrayRef(Variants);
2014   }
2015 
2016   static const unsigned Variants[] = {
2017     AMDGPUAsmVariants::DEFAULT, AMDGPUAsmVariants::VOP3,
2018     AMDGPUAsmVariants::SDWA, AMDGPUAsmVariants::SDWA9, AMDGPUAsmVariants::DPP
2019   };
2020 
2021   return makeArrayRef(Variants);
2022 }
2023 
2024 unsigned AMDGPUAsmParser::findImplicitSGPRReadInVOP(const MCInst &Inst) const {
2025   const MCInstrDesc &Desc = MII.get(Inst.getOpcode());
2026   const unsigned Num = Desc.getNumImplicitUses();
2027   for (unsigned i = 0; i < Num; ++i) {
2028     unsigned Reg = Desc.ImplicitUses[i];
2029     switch (Reg) {
2030     case AMDGPU::FLAT_SCR:
2031     case AMDGPU::VCC:
2032     case AMDGPU::M0:
2033       return Reg;
2034     default:
2035       break;
2036     }
2037   }
2038   return AMDGPU::NoRegister;
2039 }
2040 
2041 // NB: This code is correct only when used to check constant
2042 // bus limitations because GFX7 support no f16 inline constants.
2043 // Note that there are no cases when a GFX7 opcode violates
2044 // constant bus limitations due to the use of an f16 constant.
2045 bool AMDGPUAsmParser::isInlineConstant(const MCInst &Inst,
2046                                        unsigned OpIdx) const {
2047   const MCInstrDesc &Desc = MII.get(Inst.getOpcode());
2048 
2049   if (!AMDGPU::isSISrcOperand(Desc, OpIdx)) {
2050     return false;
2051   }
2052 
2053   const MCOperand &MO = Inst.getOperand(OpIdx);
2054 
2055   int64_t Val = MO.getImm();
2056   auto OpSize = AMDGPU::getOperandSize(Desc, OpIdx);
2057 
2058   switch (OpSize) { // expected operand size
2059   case 8:
2060     return AMDGPU::isInlinableLiteral64(Val, hasInv2PiInlineImm());
2061   case 4:
2062     return AMDGPU::isInlinableLiteral32(Val, hasInv2PiInlineImm());
2063   case 2: {
2064     const unsigned OperandType = Desc.OpInfo[OpIdx].OperandType;
2065     if (OperandType == AMDGPU::OPERAND_REG_INLINE_C_V2INT16 ||
2066         OperandType == AMDGPU::OPERAND_REG_INLINE_C_V2FP16) {
2067       return AMDGPU::isInlinableLiteralV216(Val, hasInv2PiInlineImm());
2068     } else {
2069       return AMDGPU::isInlinableLiteral16(Val, hasInv2PiInlineImm());
2070     }
2071   }
2072   default:
2073     llvm_unreachable("invalid operand size");
2074   }
2075 }
2076 
2077 bool AMDGPUAsmParser::usesConstantBus(const MCInst &Inst, unsigned OpIdx) {
2078   const MCOperand &MO = Inst.getOperand(OpIdx);
2079   if (MO.isImm()) {
2080     return !isInlineConstant(Inst, OpIdx);
2081   }
2082   return !MO.isReg() ||
2083          isSGPR(mc2PseudoReg(MO.getReg()), getContext().getRegisterInfo());
2084 }
2085 
2086 bool AMDGPUAsmParser::validateOperandLimitations(const MCInst &Inst) {
2087   const unsigned Opcode = Inst.getOpcode();
2088   const MCInstrDesc &Desc = MII.get(Opcode);
2089   unsigned ConstantBusUseCount = 0;
2090 
2091   if (Desc.TSFlags &
2092       (SIInstrFlags::VOPC |
2093        SIInstrFlags::VOP1 | SIInstrFlags::VOP2 |
2094        SIInstrFlags::VOP3 | SIInstrFlags::VOP3P |
2095        SIInstrFlags::SDWA)) {
2096 
2097     // Check special imm operands (used by madmk, etc)
2098     if (AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::imm) != -1) {
2099       ++ConstantBusUseCount;
2100     }
2101 
2102     unsigned SGPRUsed = findImplicitSGPRReadInVOP(Inst);
2103     if (SGPRUsed != AMDGPU::NoRegister) {
2104       ++ConstantBusUseCount;
2105     }
2106 
2107     const int Src0Idx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::src0);
2108     const int Src1Idx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::src1);
2109     const int Src2Idx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::src2);
2110 
2111     const int OpIndices[] = { Src0Idx, Src1Idx, Src2Idx };
2112 
2113     for (int OpIdx : OpIndices) {
2114       if (OpIdx == -1) break;
2115 
2116       const MCOperand &MO = Inst.getOperand(OpIdx);
2117       if (usesConstantBus(Inst, OpIdx)) {
2118         if (MO.isReg()) {
2119           const unsigned Reg = mc2PseudoReg(MO.getReg());
2120           // Pairs of registers with a partial intersections like these
2121           //   s0, s[0:1]
2122           //   flat_scratch_lo, flat_scratch
2123           //   flat_scratch_lo, flat_scratch_hi
2124           // are theoretically valid but they are disabled anyway.
2125           // Note that this code mimics SIInstrInfo::verifyInstruction
2126           if (Reg != SGPRUsed) {
2127             ++ConstantBusUseCount;
2128           }
2129           SGPRUsed = Reg;
2130         } else { // Expression or a literal
2131           ++ConstantBusUseCount;
2132         }
2133       }
2134     }
2135   }
2136 
2137   return ConstantBusUseCount <= 1;
2138 }
2139 
2140 bool AMDGPUAsmParser::MatchAndEmitInstruction(SMLoc IDLoc, unsigned &Opcode,
2141                                               OperandVector &Operands,
2142                                               MCStreamer &Out,
2143                                               uint64_t &ErrorInfo,
2144                                               bool MatchingInlineAsm) {
2145   MCInst Inst;
2146   unsigned Result = Match_Success;
2147   for (auto Variant : getMatchedVariants()) {
2148     uint64_t EI;
2149     auto R = MatchInstructionImpl(Operands, Inst, EI, MatchingInlineAsm,
2150                                   Variant);
2151     // We order match statuses from least to most specific. We use most specific
2152     // status as resulting
2153     // Match_MnemonicFail < Match_InvalidOperand < Match_MissingFeature < Match_PreferE32
2154     if ((R == Match_Success) ||
2155         (R == Match_PreferE32) ||
2156         (R == Match_MissingFeature && Result != Match_PreferE32) ||
2157         (R == Match_InvalidOperand && Result != Match_MissingFeature
2158                                    && Result != Match_PreferE32) ||
2159         (R == Match_MnemonicFail   && Result != Match_InvalidOperand
2160                                    && Result != Match_MissingFeature
2161                                    && Result != Match_PreferE32)) {
2162       Result = R;
2163       ErrorInfo = EI;
2164     }
2165     if (R == Match_Success)
2166       break;
2167   }
2168 
2169   switch (Result) {
2170   default: break;
2171   case Match_Success:
2172     if (!validateOperandLimitations(Inst)) {
2173       return Error(IDLoc,
2174                    "invalid operand (violates constant bus restrictions)");
2175     }
2176     Inst.setLoc(IDLoc);
2177     Out.EmitInstruction(Inst, getSTI());
2178     return false;
2179 
2180   case Match_MissingFeature:
2181     return Error(IDLoc, "instruction not supported on this GPU");
2182 
2183   case Match_MnemonicFail:
2184     return Error(IDLoc, "unrecognized instruction mnemonic");
2185 
2186   case Match_InvalidOperand: {
2187     SMLoc ErrorLoc = IDLoc;
2188     if (ErrorInfo != ~0ULL) {
2189       if (ErrorInfo >= Operands.size()) {
2190         return Error(IDLoc, "too few operands for instruction");
2191       }
2192       ErrorLoc = ((AMDGPUOperand &)*Operands[ErrorInfo]).getStartLoc();
2193       if (ErrorLoc == SMLoc())
2194         ErrorLoc = IDLoc;
2195     }
2196     return Error(ErrorLoc, "invalid operand for instruction");
2197   }
2198 
2199   case Match_PreferE32:
2200     return Error(IDLoc, "internal error: instruction without _e64 suffix "
2201                         "should be encoded as e32");
2202   }
2203   llvm_unreachable("Implement any new match types added!");
2204 }
2205 
2206 bool AMDGPUAsmParser::ParseAsAbsoluteExpression(uint32_t &Ret) {
2207   int64_t Tmp = -1;
2208   if (getLexer().isNot(AsmToken::Integer) && getLexer().isNot(AsmToken::Identifier)) {
2209     return true;
2210   }
2211   if (getParser().parseAbsoluteExpression(Tmp)) {
2212     return true;
2213   }
2214   Ret = static_cast<uint32_t>(Tmp);
2215   return false;
2216 }
2217 
2218 bool AMDGPUAsmParser::ParseDirectiveMajorMinor(uint32_t &Major,
2219                                                uint32_t &Minor) {
2220   if (ParseAsAbsoluteExpression(Major))
2221     return TokError("invalid major version");
2222 
2223   if (getLexer().isNot(AsmToken::Comma))
2224     return TokError("minor version number required, comma expected");
2225   Lex();
2226 
2227   if (ParseAsAbsoluteExpression(Minor))
2228     return TokError("invalid minor version");
2229 
2230   return false;
2231 }
2232 
2233 bool AMDGPUAsmParser::ParseDirectiveHSACodeObjectVersion() {
2234   uint32_t Major;
2235   uint32_t Minor;
2236 
2237   if (ParseDirectiveMajorMinor(Major, Minor))
2238     return true;
2239 
2240   getTargetStreamer().EmitDirectiveHSACodeObjectVersion(Major, Minor);
2241   return false;
2242 }
2243 
2244 bool AMDGPUAsmParser::ParseDirectiveHSACodeObjectISA() {
2245   uint32_t Major;
2246   uint32_t Minor;
2247   uint32_t Stepping;
2248   StringRef VendorName;
2249   StringRef ArchName;
2250 
2251   // If this directive has no arguments, then use the ISA version for the
2252   // targeted GPU.
2253   if (getLexer().is(AsmToken::EndOfStatement)) {
2254     AMDGPU::IsaInfo::IsaVersion ISA =
2255         AMDGPU::IsaInfo::getIsaVersion(getFeatureBits());
2256     getTargetStreamer().EmitDirectiveHSACodeObjectISA(ISA.Major, ISA.Minor,
2257                                                       ISA.Stepping,
2258                                                       "AMD", "AMDGPU");
2259     return false;
2260   }
2261 
2262   if (ParseDirectiveMajorMinor(Major, Minor))
2263     return true;
2264 
2265   if (getLexer().isNot(AsmToken::Comma))
2266     return TokError("stepping version number required, comma expected");
2267   Lex();
2268 
2269   if (ParseAsAbsoluteExpression(Stepping))
2270     return TokError("invalid stepping version");
2271 
2272   if (getLexer().isNot(AsmToken::Comma))
2273     return TokError("vendor name required, comma expected");
2274   Lex();
2275 
2276   if (getLexer().isNot(AsmToken::String))
2277     return TokError("invalid vendor name");
2278 
2279   VendorName = getLexer().getTok().getStringContents();
2280   Lex();
2281 
2282   if (getLexer().isNot(AsmToken::Comma))
2283     return TokError("arch name required, comma expected");
2284   Lex();
2285 
2286   if (getLexer().isNot(AsmToken::String))
2287     return TokError("invalid arch name");
2288 
2289   ArchName = getLexer().getTok().getStringContents();
2290   Lex();
2291 
2292   getTargetStreamer().EmitDirectiveHSACodeObjectISA(Major, Minor, Stepping,
2293                                                     VendorName, ArchName);
2294   return false;
2295 }
2296 
2297 bool AMDGPUAsmParser::ParseDirectiveCodeObjectMetadata() {
2298   std::string YamlString;
2299   raw_string_ostream YamlStream(YamlString);
2300 
2301   getLexer().setSkipSpace(false);
2302 
2303   bool FoundEnd = false;
2304   while (!getLexer().is(AsmToken::Eof)) {
2305     while (getLexer().is(AsmToken::Space)) {
2306       YamlStream << getLexer().getTok().getString();
2307       Lex();
2308     }
2309 
2310     if (getLexer().is(AsmToken::Identifier)) {
2311       StringRef ID = getLexer().getTok().getIdentifier();
2312       if (ID == AMDGPU::CodeObject::MetadataAssemblerDirectiveEnd) {
2313         Lex();
2314         FoundEnd = true;
2315         break;
2316       }
2317     }
2318 
2319     YamlStream << Parser.parseStringToEndOfStatement()
2320                << getContext().getAsmInfo()->getSeparatorString();
2321 
2322     Parser.eatToEndOfStatement();
2323   }
2324 
2325   getLexer().setSkipSpace(true);
2326 
2327   if (getLexer().is(AsmToken::Eof) && !FoundEnd) {
2328     return TokError(
2329         "expected directive .end_amdgpu_code_object_metadata not found");
2330   }
2331 
2332   YamlStream.flush();
2333 
2334   if (!getTargetStreamer().EmitCodeObjectMetadata(YamlString))
2335     return Error(getParser().getTok().getLoc(), "invalid code object metadata");
2336 
2337   return false;
2338 }
2339 
2340 bool AMDGPUAsmParser::ParseAMDKernelCodeTValue(StringRef ID,
2341                                                amd_kernel_code_t &Header) {
2342   SmallString<40> ErrStr;
2343   raw_svector_ostream Err(ErrStr);
2344   if (!parseAmdKernelCodeField(ID, getParser(), Header, Err)) {
2345     return TokError(Err.str());
2346   }
2347   Lex();
2348   return false;
2349 }
2350 
2351 bool AMDGPUAsmParser::ParseDirectiveAMDKernelCodeT() {
2352   amd_kernel_code_t Header;
2353   AMDGPU::initDefaultAMDKernelCodeT(Header, getFeatureBits());
2354 
2355   while (true) {
2356     // Lex EndOfStatement.  This is in a while loop, because lexing a comment
2357     // will set the current token to EndOfStatement.
2358     while(getLexer().is(AsmToken::EndOfStatement))
2359       Lex();
2360 
2361     if (getLexer().isNot(AsmToken::Identifier))
2362       return TokError("expected value identifier or .end_amd_kernel_code_t");
2363 
2364     StringRef ID = getLexer().getTok().getIdentifier();
2365     Lex();
2366 
2367     if (ID == ".end_amd_kernel_code_t")
2368       break;
2369 
2370     if (ParseAMDKernelCodeTValue(ID, Header))
2371       return true;
2372   }
2373 
2374   getTargetStreamer().EmitAMDKernelCodeT(Header);
2375 
2376   return false;
2377 }
2378 
2379 bool AMDGPUAsmParser::ParseDirectiveAMDGPUHsaKernel() {
2380   if (getLexer().isNot(AsmToken::Identifier))
2381     return TokError("expected symbol name");
2382 
2383   StringRef KernelName = Parser.getTok().getString();
2384 
2385   getTargetStreamer().EmitAMDGPUSymbolType(KernelName,
2386                                            ELF::STT_AMDGPU_HSA_KERNEL);
2387   Lex();
2388   KernelScope.initialize(getContext());
2389   return false;
2390 }
2391 
2392 bool AMDGPUAsmParser::ParseDirective(AsmToken DirectiveID) {
2393   StringRef IDVal = DirectiveID.getString();
2394 
2395   if (IDVal == ".hsa_code_object_version")
2396     return ParseDirectiveHSACodeObjectVersion();
2397 
2398   if (IDVal == ".hsa_code_object_isa")
2399     return ParseDirectiveHSACodeObjectISA();
2400 
2401   if (IDVal == AMDGPU::CodeObject::MetadataAssemblerDirectiveBegin)
2402     return ParseDirectiveCodeObjectMetadata();
2403 
2404   if (IDVal == ".amd_kernel_code_t")
2405     return ParseDirectiveAMDKernelCodeT();
2406 
2407   if (IDVal == ".amdgpu_hsa_kernel")
2408     return ParseDirectiveAMDGPUHsaKernel();
2409 
2410   return true;
2411 }
2412 
2413 bool AMDGPUAsmParser::subtargetHasRegister(const MCRegisterInfo &MRI,
2414                                            unsigned RegNo) const {
2415   if (isCI())
2416     return true;
2417 
2418   if (isSI()) {
2419     // No flat_scr
2420     switch (RegNo) {
2421     case AMDGPU::FLAT_SCR:
2422     case AMDGPU::FLAT_SCR_LO:
2423     case AMDGPU::FLAT_SCR_HI:
2424       return false;
2425     default:
2426       return true;
2427     }
2428   }
2429 
2430   // VI only has 102 SGPRs, so make sure we aren't trying to use the 2 more that
2431   // SI/CI have.
2432   for (MCRegAliasIterator R(AMDGPU::SGPR102_SGPR103, &MRI, true);
2433        R.isValid(); ++R) {
2434     if (*R == RegNo)
2435       return false;
2436   }
2437 
2438   return true;
2439 }
2440 
2441 OperandMatchResultTy
2442 AMDGPUAsmParser::parseOperand(OperandVector &Operands, StringRef Mnemonic) {
2443   // Try to parse with a custom parser
2444   OperandMatchResultTy ResTy = MatchOperandParserImpl(Operands, Mnemonic);
2445 
2446   // If we successfully parsed the operand or if there as an error parsing,
2447   // we are done.
2448   //
2449   // If we are parsing after we reach EndOfStatement then this means we
2450   // are appending default values to the Operands list.  This is only done
2451   // by custom parser, so we shouldn't continue on to the generic parsing.
2452   if (ResTy == MatchOperand_Success || ResTy == MatchOperand_ParseFail ||
2453       getLexer().is(AsmToken::EndOfStatement))
2454     return ResTy;
2455 
2456   ResTy = parseRegOrImm(Operands);
2457 
2458   if (ResTy == MatchOperand_Success)
2459     return ResTy;
2460 
2461   if (getLexer().getKind() == AsmToken::Identifier) {
2462     // If this identifier is a symbol, we want to create an expression for it.
2463     // It is a little difficult to distinguish between a symbol name, and
2464     // an instruction flag like 'gds'.  In order to do this, we parse
2465     // all tokens as expressions and then treate the symbol name as the token
2466     // string when we want to interpret the operand as a token.
2467     const auto &Tok = Parser.getTok();
2468     SMLoc S = Tok.getLoc();
2469     const MCExpr *Expr = nullptr;
2470     if (!Parser.parseExpression(Expr)) {
2471       Operands.push_back(AMDGPUOperand::CreateExpr(this, Expr, S));
2472       return MatchOperand_Success;
2473     }
2474 
2475     Operands.push_back(AMDGPUOperand::CreateToken(this, Tok.getString(), Tok.getLoc()));
2476     Parser.Lex();
2477     return MatchOperand_Success;
2478   }
2479   return MatchOperand_NoMatch;
2480 }
2481 
2482 StringRef AMDGPUAsmParser::parseMnemonicSuffix(StringRef Name) {
2483   // Clear any forced encodings from the previous instruction.
2484   setForcedEncodingSize(0);
2485   setForcedDPP(false);
2486   setForcedSDWA(false);
2487 
2488   if (Name.endswith("_e64")) {
2489     setForcedEncodingSize(64);
2490     return Name.substr(0, Name.size() - 4);
2491   } else if (Name.endswith("_e32")) {
2492     setForcedEncodingSize(32);
2493     return Name.substr(0, Name.size() - 4);
2494   } else if (Name.endswith("_dpp")) {
2495     setForcedDPP(true);
2496     return Name.substr(0, Name.size() - 4);
2497   } else if (Name.endswith("_sdwa")) {
2498     setForcedSDWA(true);
2499     return Name.substr(0, Name.size() - 5);
2500   }
2501   return Name;
2502 }
2503 
2504 bool AMDGPUAsmParser::ParseInstruction(ParseInstructionInfo &Info,
2505                                        StringRef Name,
2506                                        SMLoc NameLoc, OperandVector &Operands) {
2507   // Add the instruction mnemonic
2508   Name = parseMnemonicSuffix(Name);
2509   Operands.push_back(AMDGPUOperand::CreateToken(this, Name, NameLoc));
2510 
2511   while (!getLexer().is(AsmToken::EndOfStatement)) {
2512     OperandMatchResultTy Res = parseOperand(Operands, Name);
2513 
2514     // Eat the comma or space if there is one.
2515     if (getLexer().is(AsmToken::Comma))
2516       Parser.Lex();
2517 
2518     switch (Res) {
2519       case MatchOperand_Success: break;
2520       case MatchOperand_ParseFail:
2521         Error(getLexer().getLoc(), "failed parsing operand.");
2522         while (!getLexer().is(AsmToken::EndOfStatement)) {
2523           Parser.Lex();
2524         }
2525         return true;
2526       case MatchOperand_NoMatch:
2527         Error(getLexer().getLoc(), "not a valid operand.");
2528         while (!getLexer().is(AsmToken::EndOfStatement)) {
2529           Parser.Lex();
2530         }
2531         return true;
2532     }
2533   }
2534 
2535   return false;
2536 }
2537 
2538 //===----------------------------------------------------------------------===//
2539 // Utility functions
2540 //===----------------------------------------------------------------------===//
2541 
2542 OperandMatchResultTy
2543 AMDGPUAsmParser::parseIntWithPrefix(const char *Prefix, int64_t &Int) {
2544   switch(getLexer().getKind()) {
2545     default: return MatchOperand_NoMatch;
2546     case AsmToken::Identifier: {
2547       StringRef Name = Parser.getTok().getString();
2548       if (!Name.equals(Prefix)) {
2549         return MatchOperand_NoMatch;
2550       }
2551 
2552       Parser.Lex();
2553       if (getLexer().isNot(AsmToken::Colon))
2554         return MatchOperand_ParseFail;
2555 
2556       Parser.Lex();
2557       if (getLexer().isNot(AsmToken::Integer))
2558         return MatchOperand_ParseFail;
2559 
2560       if (getParser().parseAbsoluteExpression(Int))
2561         return MatchOperand_ParseFail;
2562       break;
2563     }
2564   }
2565   return MatchOperand_Success;
2566 }
2567 
2568 OperandMatchResultTy
2569 AMDGPUAsmParser::parseIntWithPrefix(const char *Prefix, OperandVector &Operands,
2570                                     AMDGPUOperand::ImmTy ImmTy,
2571                                     bool (*ConvertResult)(int64_t&)) {
2572   SMLoc S = Parser.getTok().getLoc();
2573   int64_t Value = 0;
2574 
2575   OperandMatchResultTy Res = parseIntWithPrefix(Prefix, Value);
2576   if (Res != MatchOperand_Success)
2577     return Res;
2578 
2579   if (ConvertResult && !ConvertResult(Value)) {
2580     return MatchOperand_ParseFail;
2581   }
2582 
2583   Operands.push_back(AMDGPUOperand::CreateImm(this, Value, S, ImmTy));
2584   return MatchOperand_Success;
2585 }
2586 
2587 OperandMatchResultTy AMDGPUAsmParser::parseOperandArrayWithPrefix(
2588   const char *Prefix,
2589   OperandVector &Operands,
2590   AMDGPUOperand::ImmTy ImmTy,
2591   bool (*ConvertResult)(int64_t&)) {
2592   StringRef Name = Parser.getTok().getString();
2593   if (!Name.equals(Prefix))
2594     return MatchOperand_NoMatch;
2595 
2596   Parser.Lex();
2597   if (getLexer().isNot(AsmToken::Colon))
2598     return MatchOperand_ParseFail;
2599 
2600   Parser.Lex();
2601   if (getLexer().isNot(AsmToken::LBrac))
2602     return MatchOperand_ParseFail;
2603   Parser.Lex();
2604 
2605   unsigned Val = 0;
2606   SMLoc S = Parser.getTok().getLoc();
2607 
2608   // FIXME: How to verify the number of elements matches the number of src
2609   // operands?
2610   for (int I = 0; I < 3; ++I) {
2611     if (I != 0) {
2612       if (getLexer().is(AsmToken::RBrac))
2613         break;
2614 
2615       if (getLexer().isNot(AsmToken::Comma))
2616         return MatchOperand_ParseFail;
2617       Parser.Lex();
2618     }
2619 
2620     if (getLexer().isNot(AsmToken::Integer))
2621       return MatchOperand_ParseFail;
2622 
2623     int64_t Op;
2624     if (getParser().parseAbsoluteExpression(Op))
2625       return MatchOperand_ParseFail;
2626 
2627     if (Op != 0 && Op != 1)
2628       return MatchOperand_ParseFail;
2629     Val |= (Op << I);
2630   }
2631 
2632   Parser.Lex();
2633   Operands.push_back(AMDGPUOperand::CreateImm(this, Val, S, ImmTy));
2634   return MatchOperand_Success;
2635 }
2636 
2637 OperandMatchResultTy
2638 AMDGPUAsmParser::parseNamedBit(const char *Name, OperandVector &Operands,
2639                                AMDGPUOperand::ImmTy ImmTy) {
2640   int64_t Bit = 0;
2641   SMLoc S = Parser.getTok().getLoc();
2642 
2643   // We are at the end of the statement, and this is a default argument, so
2644   // use a default value.
2645   if (getLexer().isNot(AsmToken::EndOfStatement)) {
2646     switch(getLexer().getKind()) {
2647       case AsmToken::Identifier: {
2648         StringRef Tok = Parser.getTok().getString();
2649         if (Tok == Name) {
2650           Bit = 1;
2651           Parser.Lex();
2652         } else if (Tok.startswith("no") && Tok.endswith(Name)) {
2653           Bit = 0;
2654           Parser.Lex();
2655         } else {
2656           return MatchOperand_NoMatch;
2657         }
2658         break;
2659       }
2660       default:
2661         return MatchOperand_NoMatch;
2662     }
2663   }
2664 
2665   Operands.push_back(AMDGPUOperand::CreateImm(this, Bit, S, ImmTy));
2666   return MatchOperand_Success;
2667 }
2668 
2669 static void addOptionalImmOperand(
2670   MCInst& Inst, const OperandVector& Operands,
2671   AMDGPUAsmParser::OptionalImmIndexMap& OptionalIdx,
2672   AMDGPUOperand::ImmTy ImmT,
2673   int64_t Default = 0) {
2674   auto i = OptionalIdx.find(ImmT);
2675   if (i != OptionalIdx.end()) {
2676     unsigned Idx = i->second;
2677     ((AMDGPUOperand &)*Operands[Idx]).addImmOperands(Inst, 1);
2678   } else {
2679     Inst.addOperand(MCOperand::createImm(Default));
2680   }
2681 }
2682 
2683 OperandMatchResultTy
2684 AMDGPUAsmParser::parseStringWithPrefix(StringRef Prefix, StringRef &Value) {
2685   if (getLexer().isNot(AsmToken::Identifier)) {
2686     return MatchOperand_NoMatch;
2687   }
2688   StringRef Tok = Parser.getTok().getString();
2689   if (Tok != Prefix) {
2690     return MatchOperand_NoMatch;
2691   }
2692 
2693   Parser.Lex();
2694   if (getLexer().isNot(AsmToken::Colon)) {
2695     return MatchOperand_ParseFail;
2696   }
2697 
2698   Parser.Lex();
2699   if (getLexer().isNot(AsmToken::Identifier)) {
2700     return MatchOperand_ParseFail;
2701   }
2702 
2703   Value = Parser.getTok().getString();
2704   return MatchOperand_Success;
2705 }
2706 
2707 //===----------------------------------------------------------------------===//
2708 // ds
2709 //===----------------------------------------------------------------------===//
2710 
2711 void AMDGPUAsmParser::cvtDSOffset01(MCInst &Inst,
2712                                     const OperandVector &Operands) {
2713   OptionalImmIndexMap OptionalIdx;
2714 
2715   for (unsigned i = 1, e = Operands.size(); i != e; ++i) {
2716     AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[i]);
2717 
2718     // Add the register arguments
2719     if (Op.isReg()) {
2720       Op.addRegOperands(Inst, 1);
2721       continue;
2722     }
2723 
2724     // Handle optional arguments
2725     OptionalIdx[Op.getImmTy()] = i;
2726   }
2727 
2728   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyOffset0);
2729   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyOffset1);
2730   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyGDS);
2731 
2732   Inst.addOperand(MCOperand::createReg(AMDGPU::M0)); // m0
2733 }
2734 
2735 void AMDGPUAsmParser::cvtDSImpl(MCInst &Inst, const OperandVector &Operands,
2736                                 bool IsGdsHardcoded) {
2737   OptionalImmIndexMap OptionalIdx;
2738 
2739   for (unsigned i = 1, e = Operands.size(); i != e; ++i) {
2740     AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[i]);
2741 
2742     // Add the register arguments
2743     if (Op.isReg()) {
2744       Op.addRegOperands(Inst, 1);
2745       continue;
2746     }
2747 
2748     if (Op.isToken() && Op.getToken() == "gds") {
2749       IsGdsHardcoded = true;
2750       continue;
2751     }
2752 
2753     // Handle optional arguments
2754     OptionalIdx[Op.getImmTy()] = i;
2755   }
2756 
2757   AMDGPUOperand::ImmTy OffsetType =
2758     (Inst.getOpcode() == AMDGPU::DS_SWIZZLE_B32_si ||
2759      Inst.getOpcode() == AMDGPU::DS_SWIZZLE_B32_vi) ? AMDGPUOperand::ImmTySwizzle :
2760                                                       AMDGPUOperand::ImmTyOffset;
2761 
2762   addOptionalImmOperand(Inst, Operands, OptionalIdx, OffsetType);
2763 
2764   if (!IsGdsHardcoded) {
2765     addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyGDS);
2766   }
2767   Inst.addOperand(MCOperand::createReg(AMDGPU::M0)); // m0
2768 }
2769 
2770 void AMDGPUAsmParser::cvtExp(MCInst &Inst, const OperandVector &Operands) {
2771   OptionalImmIndexMap OptionalIdx;
2772 
2773   unsigned OperandIdx[4];
2774   unsigned EnMask = 0;
2775   int SrcIdx = 0;
2776 
2777   for (unsigned i = 1, e = Operands.size(); i != e; ++i) {
2778     AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[i]);
2779 
2780     // Add the register arguments
2781     if (Op.isReg()) {
2782       assert(SrcIdx < 4);
2783       OperandIdx[SrcIdx] = Inst.size();
2784       Op.addRegOperands(Inst, 1);
2785       ++SrcIdx;
2786       continue;
2787     }
2788 
2789     if (Op.isOff()) {
2790       assert(SrcIdx < 4);
2791       OperandIdx[SrcIdx] = Inst.size();
2792       Inst.addOperand(MCOperand::createReg(AMDGPU::NoRegister));
2793       ++SrcIdx;
2794       continue;
2795     }
2796 
2797     if (Op.isImm() && Op.getImmTy() == AMDGPUOperand::ImmTyExpTgt) {
2798       Op.addImmOperands(Inst, 1);
2799       continue;
2800     }
2801 
2802     if (Op.isToken() && Op.getToken() == "done")
2803       continue;
2804 
2805     // Handle optional arguments
2806     OptionalIdx[Op.getImmTy()] = i;
2807   }
2808 
2809   assert(SrcIdx == 4);
2810 
2811   bool Compr = false;
2812   if (OptionalIdx.find(AMDGPUOperand::ImmTyExpCompr) != OptionalIdx.end()) {
2813     Compr = true;
2814     Inst.getOperand(OperandIdx[1]) = Inst.getOperand(OperandIdx[2]);
2815     Inst.getOperand(OperandIdx[2]).setReg(AMDGPU::NoRegister);
2816     Inst.getOperand(OperandIdx[3]).setReg(AMDGPU::NoRegister);
2817   }
2818 
2819   for (auto i = 0; i < SrcIdx; ++i) {
2820     if (Inst.getOperand(OperandIdx[i]).getReg() != AMDGPU::NoRegister) {
2821       EnMask |= Compr? (0x3 << i * 2) : (0x1 << i);
2822     }
2823   }
2824 
2825   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyExpVM);
2826   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyExpCompr);
2827 
2828   Inst.addOperand(MCOperand::createImm(EnMask));
2829 }
2830 
2831 //===----------------------------------------------------------------------===//
2832 // s_waitcnt
2833 //===----------------------------------------------------------------------===//
2834 
2835 static bool
2836 encodeCnt(
2837   const AMDGPU::IsaInfo::IsaVersion ISA,
2838   int64_t &IntVal,
2839   int64_t CntVal,
2840   bool Saturate,
2841   unsigned (*encode)(const IsaInfo::IsaVersion &Version, unsigned, unsigned),
2842   unsigned (*decode)(const IsaInfo::IsaVersion &Version, unsigned))
2843 {
2844   bool Failed = false;
2845 
2846   IntVal = encode(ISA, IntVal, CntVal);
2847   if (CntVal != decode(ISA, IntVal)) {
2848     if (Saturate) {
2849       IntVal = encode(ISA, IntVal, -1);
2850     } else {
2851       Failed = true;
2852     }
2853   }
2854   return Failed;
2855 }
2856 
2857 bool AMDGPUAsmParser::parseCnt(int64_t &IntVal) {
2858   StringRef CntName = Parser.getTok().getString();
2859   int64_t CntVal;
2860 
2861   Parser.Lex();
2862   if (getLexer().isNot(AsmToken::LParen))
2863     return true;
2864 
2865   Parser.Lex();
2866   if (getLexer().isNot(AsmToken::Integer))
2867     return true;
2868 
2869   SMLoc ValLoc = Parser.getTok().getLoc();
2870   if (getParser().parseAbsoluteExpression(CntVal))
2871     return true;
2872 
2873   AMDGPU::IsaInfo::IsaVersion ISA =
2874       AMDGPU::IsaInfo::getIsaVersion(getFeatureBits());
2875 
2876   bool Failed = true;
2877   bool Sat = CntName.endswith("_sat");
2878 
2879   if (CntName == "vmcnt" || CntName == "vmcnt_sat") {
2880     Failed = encodeCnt(ISA, IntVal, CntVal, Sat, encodeVmcnt, decodeVmcnt);
2881   } else if (CntName == "expcnt" || CntName == "expcnt_sat") {
2882     Failed = encodeCnt(ISA, IntVal, CntVal, Sat, encodeExpcnt, decodeExpcnt);
2883   } else if (CntName == "lgkmcnt" || CntName == "lgkmcnt_sat") {
2884     Failed = encodeCnt(ISA, IntVal, CntVal, Sat, encodeLgkmcnt, decodeLgkmcnt);
2885   }
2886 
2887   if (Failed) {
2888     Error(ValLoc, "too large value for " + CntName);
2889     return true;
2890   }
2891 
2892   if (getLexer().isNot(AsmToken::RParen)) {
2893     return true;
2894   }
2895 
2896   Parser.Lex();
2897   if (getLexer().is(AsmToken::Amp) || getLexer().is(AsmToken::Comma)) {
2898     const AsmToken NextToken = getLexer().peekTok();
2899     if (NextToken.is(AsmToken::Identifier)) {
2900       Parser.Lex();
2901     }
2902   }
2903 
2904   return false;
2905 }
2906 
2907 OperandMatchResultTy
2908 AMDGPUAsmParser::parseSWaitCntOps(OperandVector &Operands) {
2909   AMDGPU::IsaInfo::IsaVersion ISA =
2910       AMDGPU::IsaInfo::getIsaVersion(getFeatureBits());
2911   int64_t Waitcnt = getWaitcntBitMask(ISA);
2912   SMLoc S = Parser.getTok().getLoc();
2913 
2914   switch(getLexer().getKind()) {
2915     default: return MatchOperand_ParseFail;
2916     case AsmToken::Integer:
2917       // The operand can be an integer value.
2918       if (getParser().parseAbsoluteExpression(Waitcnt))
2919         return MatchOperand_ParseFail;
2920       break;
2921 
2922     case AsmToken::Identifier:
2923       do {
2924         if (parseCnt(Waitcnt))
2925           return MatchOperand_ParseFail;
2926       } while(getLexer().isNot(AsmToken::EndOfStatement));
2927       break;
2928   }
2929   Operands.push_back(AMDGPUOperand::CreateImm(this, Waitcnt, S));
2930   return MatchOperand_Success;
2931 }
2932 
2933 bool AMDGPUAsmParser::parseHwregConstruct(OperandInfoTy &HwReg, int64_t &Offset,
2934                                           int64_t &Width) {
2935   using namespace llvm::AMDGPU::Hwreg;
2936 
2937   if (Parser.getTok().getString() != "hwreg")
2938     return true;
2939   Parser.Lex();
2940 
2941   if (getLexer().isNot(AsmToken::LParen))
2942     return true;
2943   Parser.Lex();
2944 
2945   if (getLexer().is(AsmToken::Identifier)) {
2946     HwReg.IsSymbolic = true;
2947     HwReg.Id = ID_UNKNOWN_;
2948     const StringRef tok = Parser.getTok().getString();
2949     for (int i = ID_SYMBOLIC_FIRST_; i < ID_SYMBOLIC_LAST_; ++i) {
2950       if (tok == IdSymbolic[i]) {
2951         HwReg.Id = i;
2952         break;
2953       }
2954     }
2955     Parser.Lex();
2956   } else {
2957     HwReg.IsSymbolic = false;
2958     if (getLexer().isNot(AsmToken::Integer))
2959       return true;
2960     if (getParser().parseAbsoluteExpression(HwReg.Id))
2961       return true;
2962   }
2963 
2964   if (getLexer().is(AsmToken::RParen)) {
2965     Parser.Lex();
2966     return false;
2967   }
2968 
2969   // optional params
2970   if (getLexer().isNot(AsmToken::Comma))
2971     return true;
2972   Parser.Lex();
2973 
2974   if (getLexer().isNot(AsmToken::Integer))
2975     return true;
2976   if (getParser().parseAbsoluteExpression(Offset))
2977     return true;
2978 
2979   if (getLexer().isNot(AsmToken::Comma))
2980     return true;
2981   Parser.Lex();
2982 
2983   if (getLexer().isNot(AsmToken::Integer))
2984     return true;
2985   if (getParser().parseAbsoluteExpression(Width))
2986     return true;
2987 
2988   if (getLexer().isNot(AsmToken::RParen))
2989     return true;
2990   Parser.Lex();
2991 
2992   return false;
2993 }
2994 
2995 OperandMatchResultTy AMDGPUAsmParser::parseHwreg(OperandVector &Operands) {
2996   using namespace llvm::AMDGPU::Hwreg;
2997 
2998   int64_t Imm16Val = 0;
2999   SMLoc S = Parser.getTok().getLoc();
3000 
3001   switch(getLexer().getKind()) {
3002     default: return MatchOperand_NoMatch;
3003     case AsmToken::Integer:
3004       // The operand can be an integer value.
3005       if (getParser().parseAbsoluteExpression(Imm16Val))
3006         return MatchOperand_NoMatch;
3007       if (Imm16Val < 0 || !isUInt<16>(Imm16Val)) {
3008         Error(S, "invalid immediate: only 16-bit values are legal");
3009         // Do not return error code, but create an imm operand anyway and proceed
3010         // to the next operand, if any. That avoids unneccessary error messages.
3011       }
3012       break;
3013 
3014     case AsmToken::Identifier: {
3015         OperandInfoTy HwReg(ID_UNKNOWN_);
3016         int64_t Offset = OFFSET_DEFAULT_;
3017         int64_t Width = WIDTH_M1_DEFAULT_ + 1;
3018         if (parseHwregConstruct(HwReg, Offset, Width))
3019           return MatchOperand_ParseFail;
3020         if (HwReg.Id < 0 || !isUInt<ID_WIDTH_>(HwReg.Id)) {
3021           if (HwReg.IsSymbolic)
3022             Error(S, "invalid symbolic name of hardware register");
3023           else
3024             Error(S, "invalid code of hardware register: only 6-bit values are legal");
3025         }
3026         if (Offset < 0 || !isUInt<OFFSET_WIDTH_>(Offset))
3027           Error(S, "invalid bit offset: only 5-bit values are legal");
3028         if ((Width-1) < 0 || !isUInt<WIDTH_M1_WIDTH_>(Width-1))
3029           Error(S, "invalid bitfield width: only values from 1 to 32 are legal");
3030         Imm16Val = (HwReg.Id << ID_SHIFT_) | (Offset << OFFSET_SHIFT_) | ((Width-1) << WIDTH_M1_SHIFT_);
3031       }
3032       break;
3033   }
3034   Operands.push_back(AMDGPUOperand::CreateImm(this, Imm16Val, S, AMDGPUOperand::ImmTyHwreg));
3035   return MatchOperand_Success;
3036 }
3037 
3038 bool AMDGPUOperand::isSWaitCnt() const {
3039   return isImm();
3040 }
3041 
3042 bool AMDGPUOperand::isHwreg() const {
3043   return isImmTy(ImmTyHwreg);
3044 }
3045 
3046 bool AMDGPUAsmParser::parseSendMsgConstruct(OperandInfoTy &Msg, OperandInfoTy &Operation, int64_t &StreamId) {
3047   using namespace llvm::AMDGPU::SendMsg;
3048 
3049   if (Parser.getTok().getString() != "sendmsg")
3050     return true;
3051   Parser.Lex();
3052 
3053   if (getLexer().isNot(AsmToken::LParen))
3054     return true;
3055   Parser.Lex();
3056 
3057   if (getLexer().is(AsmToken::Identifier)) {
3058     Msg.IsSymbolic = true;
3059     Msg.Id = ID_UNKNOWN_;
3060     const std::string tok = Parser.getTok().getString();
3061     for (int i = ID_GAPS_FIRST_; i < ID_GAPS_LAST_; ++i) {
3062       switch(i) {
3063         default: continue; // Omit gaps.
3064         case ID_INTERRUPT: case ID_GS: case ID_GS_DONE:  case ID_SYSMSG: break;
3065       }
3066       if (tok == IdSymbolic[i]) {
3067         Msg.Id = i;
3068         break;
3069       }
3070     }
3071     Parser.Lex();
3072   } else {
3073     Msg.IsSymbolic = false;
3074     if (getLexer().isNot(AsmToken::Integer))
3075       return true;
3076     if (getParser().parseAbsoluteExpression(Msg.Id))
3077       return true;
3078     if (getLexer().is(AsmToken::Integer))
3079       if (getParser().parseAbsoluteExpression(Msg.Id))
3080         Msg.Id = ID_UNKNOWN_;
3081   }
3082   if (Msg.Id == ID_UNKNOWN_) // Don't know how to parse the rest.
3083     return false;
3084 
3085   if (!(Msg.Id == ID_GS || Msg.Id == ID_GS_DONE || Msg.Id == ID_SYSMSG)) {
3086     if (getLexer().isNot(AsmToken::RParen))
3087       return true;
3088     Parser.Lex();
3089     return false;
3090   }
3091 
3092   if (getLexer().isNot(AsmToken::Comma))
3093     return true;
3094   Parser.Lex();
3095 
3096   assert(Msg.Id == ID_GS || Msg.Id == ID_GS_DONE || Msg.Id == ID_SYSMSG);
3097   Operation.Id = ID_UNKNOWN_;
3098   if (getLexer().is(AsmToken::Identifier)) {
3099     Operation.IsSymbolic = true;
3100     const char* const *S = (Msg.Id == ID_SYSMSG) ? OpSysSymbolic : OpGsSymbolic;
3101     const int F = (Msg.Id == ID_SYSMSG) ? OP_SYS_FIRST_ : OP_GS_FIRST_;
3102     const int L = (Msg.Id == ID_SYSMSG) ? OP_SYS_LAST_ : OP_GS_LAST_;
3103     const StringRef Tok = Parser.getTok().getString();
3104     for (int i = F; i < L; ++i) {
3105       if (Tok == S[i]) {
3106         Operation.Id = i;
3107         break;
3108       }
3109     }
3110     Parser.Lex();
3111   } else {
3112     Operation.IsSymbolic = false;
3113     if (getLexer().isNot(AsmToken::Integer))
3114       return true;
3115     if (getParser().parseAbsoluteExpression(Operation.Id))
3116       return true;
3117   }
3118 
3119   if ((Msg.Id == ID_GS || Msg.Id == ID_GS_DONE) && Operation.Id != OP_GS_NOP) {
3120     // Stream id is optional.
3121     if (getLexer().is(AsmToken::RParen)) {
3122       Parser.Lex();
3123       return false;
3124     }
3125 
3126     if (getLexer().isNot(AsmToken::Comma))
3127       return true;
3128     Parser.Lex();
3129 
3130     if (getLexer().isNot(AsmToken::Integer))
3131       return true;
3132     if (getParser().parseAbsoluteExpression(StreamId))
3133       return true;
3134   }
3135 
3136   if (getLexer().isNot(AsmToken::RParen))
3137     return true;
3138   Parser.Lex();
3139   return false;
3140 }
3141 
3142 OperandMatchResultTy AMDGPUAsmParser::parseInterpSlot(OperandVector &Operands) {
3143   if (getLexer().getKind() != AsmToken::Identifier)
3144     return MatchOperand_NoMatch;
3145 
3146   StringRef Str = Parser.getTok().getString();
3147   int Slot = StringSwitch<int>(Str)
3148     .Case("p10", 0)
3149     .Case("p20", 1)
3150     .Case("p0", 2)
3151     .Default(-1);
3152 
3153   SMLoc S = Parser.getTok().getLoc();
3154   if (Slot == -1)
3155     return MatchOperand_ParseFail;
3156 
3157   Parser.Lex();
3158   Operands.push_back(AMDGPUOperand::CreateImm(this, Slot, S,
3159                                               AMDGPUOperand::ImmTyInterpSlot));
3160   return MatchOperand_Success;
3161 }
3162 
3163 OperandMatchResultTy AMDGPUAsmParser::parseInterpAttr(OperandVector &Operands) {
3164   if (getLexer().getKind() != AsmToken::Identifier)
3165     return MatchOperand_NoMatch;
3166 
3167   StringRef Str = Parser.getTok().getString();
3168   if (!Str.startswith("attr"))
3169     return MatchOperand_NoMatch;
3170 
3171   StringRef Chan = Str.take_back(2);
3172   int AttrChan = StringSwitch<int>(Chan)
3173     .Case(".x", 0)
3174     .Case(".y", 1)
3175     .Case(".z", 2)
3176     .Case(".w", 3)
3177     .Default(-1);
3178   if (AttrChan == -1)
3179     return MatchOperand_ParseFail;
3180 
3181   Str = Str.drop_back(2).drop_front(4);
3182 
3183   uint8_t Attr;
3184   if (Str.getAsInteger(10, Attr))
3185     return MatchOperand_ParseFail;
3186 
3187   SMLoc S = Parser.getTok().getLoc();
3188   Parser.Lex();
3189   if (Attr > 63) {
3190     Error(S, "out of bounds attr");
3191     return MatchOperand_Success;
3192   }
3193 
3194   SMLoc SChan = SMLoc::getFromPointer(Chan.data());
3195 
3196   Operands.push_back(AMDGPUOperand::CreateImm(this, Attr, S,
3197                                               AMDGPUOperand::ImmTyInterpAttr));
3198   Operands.push_back(AMDGPUOperand::CreateImm(this, AttrChan, SChan,
3199                                               AMDGPUOperand::ImmTyAttrChan));
3200   return MatchOperand_Success;
3201 }
3202 
3203 void AMDGPUAsmParser::errorExpTgt() {
3204   Error(Parser.getTok().getLoc(), "invalid exp target");
3205 }
3206 
3207 OperandMatchResultTy AMDGPUAsmParser::parseExpTgtImpl(StringRef Str,
3208                                                       uint8_t &Val) {
3209   if (Str == "null") {
3210     Val = 9;
3211     return MatchOperand_Success;
3212   }
3213 
3214   if (Str.startswith("mrt")) {
3215     Str = Str.drop_front(3);
3216     if (Str == "z") { // == mrtz
3217       Val = 8;
3218       return MatchOperand_Success;
3219     }
3220 
3221     if (Str.getAsInteger(10, Val))
3222       return MatchOperand_ParseFail;
3223 
3224     if (Val > 7)
3225       errorExpTgt();
3226 
3227     return MatchOperand_Success;
3228   }
3229 
3230   if (Str.startswith("pos")) {
3231     Str = Str.drop_front(3);
3232     if (Str.getAsInteger(10, Val))
3233       return MatchOperand_ParseFail;
3234 
3235     if (Val > 3)
3236       errorExpTgt();
3237 
3238     Val += 12;
3239     return MatchOperand_Success;
3240   }
3241 
3242   if (Str.startswith("param")) {
3243     Str = Str.drop_front(5);
3244     if (Str.getAsInteger(10, Val))
3245       return MatchOperand_ParseFail;
3246 
3247     if (Val >= 32)
3248       errorExpTgt();
3249 
3250     Val += 32;
3251     return MatchOperand_Success;
3252   }
3253 
3254   if (Str.startswith("invalid_target_")) {
3255     Str = Str.drop_front(15);
3256     if (Str.getAsInteger(10, Val))
3257       return MatchOperand_ParseFail;
3258 
3259     errorExpTgt();
3260     return MatchOperand_Success;
3261   }
3262 
3263   return MatchOperand_NoMatch;
3264 }
3265 
3266 OperandMatchResultTy AMDGPUAsmParser::parseExpTgt(OperandVector &Operands) {
3267   uint8_t Val;
3268   StringRef Str = Parser.getTok().getString();
3269 
3270   auto Res = parseExpTgtImpl(Str, Val);
3271   if (Res != MatchOperand_Success)
3272     return Res;
3273 
3274   SMLoc S = Parser.getTok().getLoc();
3275   Parser.Lex();
3276 
3277   Operands.push_back(AMDGPUOperand::CreateImm(this, Val, S,
3278                                               AMDGPUOperand::ImmTyExpTgt));
3279   return MatchOperand_Success;
3280 }
3281 
3282 OperandMatchResultTy
3283 AMDGPUAsmParser::parseSendMsgOp(OperandVector &Operands) {
3284   using namespace llvm::AMDGPU::SendMsg;
3285 
3286   int64_t Imm16Val = 0;
3287   SMLoc S = Parser.getTok().getLoc();
3288 
3289   switch(getLexer().getKind()) {
3290   default:
3291     return MatchOperand_NoMatch;
3292   case AsmToken::Integer:
3293     // The operand can be an integer value.
3294     if (getParser().parseAbsoluteExpression(Imm16Val))
3295       return MatchOperand_NoMatch;
3296     if (Imm16Val < 0 || !isUInt<16>(Imm16Val)) {
3297       Error(S, "invalid immediate: only 16-bit values are legal");
3298       // Do not return error code, but create an imm operand anyway and proceed
3299       // to the next operand, if any. That avoids unneccessary error messages.
3300     }
3301     break;
3302   case AsmToken::Identifier: {
3303       OperandInfoTy Msg(ID_UNKNOWN_);
3304       OperandInfoTy Operation(OP_UNKNOWN_);
3305       int64_t StreamId = STREAM_ID_DEFAULT_;
3306       if (parseSendMsgConstruct(Msg, Operation, StreamId))
3307         return MatchOperand_ParseFail;
3308       do {
3309         // Validate and encode message ID.
3310         if (! ((ID_INTERRUPT <= Msg.Id && Msg.Id <= ID_GS_DONE)
3311                 || Msg.Id == ID_SYSMSG)) {
3312           if (Msg.IsSymbolic)
3313             Error(S, "invalid/unsupported symbolic name of message");
3314           else
3315             Error(S, "invalid/unsupported code of message");
3316           break;
3317         }
3318         Imm16Val = (Msg.Id << ID_SHIFT_);
3319         // Validate and encode operation ID.
3320         if (Msg.Id == ID_GS || Msg.Id == ID_GS_DONE) {
3321           if (! (OP_GS_FIRST_ <= Operation.Id && Operation.Id < OP_GS_LAST_)) {
3322             if (Operation.IsSymbolic)
3323               Error(S, "invalid symbolic name of GS_OP");
3324             else
3325               Error(S, "invalid code of GS_OP: only 2-bit values are legal");
3326             break;
3327           }
3328           if (Operation.Id == OP_GS_NOP
3329               && Msg.Id != ID_GS_DONE) {
3330             Error(S, "invalid GS_OP: NOP is for GS_DONE only");
3331             break;
3332           }
3333           Imm16Val |= (Operation.Id << OP_SHIFT_);
3334         }
3335         if (Msg.Id == ID_SYSMSG) {
3336           if (! (OP_SYS_FIRST_ <= Operation.Id && Operation.Id < OP_SYS_LAST_)) {
3337             if (Operation.IsSymbolic)
3338               Error(S, "invalid/unsupported symbolic name of SYSMSG_OP");
3339             else
3340               Error(S, "invalid/unsupported code of SYSMSG_OP");
3341             break;
3342           }
3343           Imm16Val |= (Operation.Id << OP_SHIFT_);
3344         }
3345         // Validate and encode stream ID.
3346         if ((Msg.Id == ID_GS || Msg.Id == ID_GS_DONE) && Operation.Id != OP_GS_NOP) {
3347           if (! (STREAM_ID_FIRST_ <= StreamId && StreamId < STREAM_ID_LAST_)) {
3348             Error(S, "invalid stream id: only 2-bit values are legal");
3349             break;
3350           }
3351           Imm16Val |= (StreamId << STREAM_ID_SHIFT_);
3352         }
3353       } while (false);
3354     }
3355     break;
3356   }
3357   Operands.push_back(AMDGPUOperand::CreateImm(this, Imm16Val, S, AMDGPUOperand::ImmTySendMsg));
3358   return MatchOperand_Success;
3359 }
3360 
3361 bool AMDGPUOperand::isSendMsg() const {
3362   return isImmTy(ImmTySendMsg);
3363 }
3364 
3365 //===----------------------------------------------------------------------===//
3366 // parser helpers
3367 //===----------------------------------------------------------------------===//
3368 
3369 bool
3370 AMDGPUAsmParser::trySkipId(const StringRef Id) {
3371   if (getLexer().getKind() == AsmToken::Identifier &&
3372       Parser.getTok().getString() == Id) {
3373     Parser.Lex();
3374     return true;
3375   }
3376   return false;
3377 }
3378 
3379 bool
3380 AMDGPUAsmParser::trySkipToken(const AsmToken::TokenKind Kind) {
3381   if (getLexer().getKind() == Kind) {
3382     Parser.Lex();
3383     return true;
3384   }
3385   return false;
3386 }
3387 
3388 bool
3389 AMDGPUAsmParser::skipToken(const AsmToken::TokenKind Kind,
3390                            const StringRef ErrMsg) {
3391   if (!trySkipToken(Kind)) {
3392     Error(Parser.getTok().getLoc(), ErrMsg);
3393     return false;
3394   }
3395   return true;
3396 }
3397 
3398 bool
3399 AMDGPUAsmParser::parseExpr(int64_t &Imm) {
3400   return !getParser().parseAbsoluteExpression(Imm);
3401 }
3402 
3403 bool
3404 AMDGPUAsmParser::parseString(StringRef &Val, const StringRef ErrMsg) {
3405   SMLoc S = Parser.getTok().getLoc();
3406   if (getLexer().getKind() == AsmToken::String) {
3407     Val = Parser.getTok().getStringContents();
3408     Parser.Lex();
3409     return true;
3410   } else {
3411     Error(S, ErrMsg);
3412     return false;
3413   }
3414 }
3415 
3416 //===----------------------------------------------------------------------===//
3417 // swizzle
3418 //===----------------------------------------------------------------------===//
3419 
3420 LLVM_READNONE
3421 static unsigned
3422 encodeBitmaskPerm(const unsigned AndMask,
3423                   const unsigned OrMask,
3424                   const unsigned XorMask) {
3425   using namespace llvm::AMDGPU::Swizzle;
3426 
3427   return BITMASK_PERM_ENC |
3428          (AndMask << BITMASK_AND_SHIFT) |
3429          (OrMask  << BITMASK_OR_SHIFT)  |
3430          (XorMask << BITMASK_XOR_SHIFT);
3431 }
3432 
3433 bool
3434 AMDGPUAsmParser::parseSwizzleOperands(const unsigned OpNum, int64_t* Op,
3435                                       const unsigned MinVal,
3436                                       const unsigned MaxVal,
3437                                       const StringRef ErrMsg) {
3438   for (unsigned i = 0; i < OpNum; ++i) {
3439     if (!skipToken(AsmToken::Comma, "expected a comma")){
3440       return false;
3441     }
3442     SMLoc ExprLoc = Parser.getTok().getLoc();
3443     if (!parseExpr(Op[i])) {
3444       return false;
3445     }
3446     if (Op[i] < MinVal || Op[i] > MaxVal) {
3447       Error(ExprLoc, ErrMsg);
3448       return false;
3449     }
3450   }
3451 
3452   return true;
3453 }
3454 
3455 bool
3456 AMDGPUAsmParser::parseSwizzleQuadPerm(int64_t &Imm) {
3457   using namespace llvm::AMDGPU::Swizzle;
3458 
3459   int64_t Lane[LANE_NUM];
3460   if (parseSwizzleOperands(LANE_NUM, Lane, 0, LANE_MAX,
3461                            "expected a 2-bit lane id")) {
3462     Imm = QUAD_PERM_ENC;
3463     for (auto i = 0; i < LANE_NUM; ++i) {
3464       Imm |= Lane[i] << (LANE_SHIFT * i);
3465     }
3466     return true;
3467   }
3468   return false;
3469 }
3470 
3471 bool
3472 AMDGPUAsmParser::parseSwizzleBroadcast(int64_t &Imm) {
3473   using namespace llvm::AMDGPU::Swizzle;
3474 
3475   SMLoc S = Parser.getTok().getLoc();
3476   int64_t GroupSize;
3477   int64_t LaneIdx;
3478 
3479   if (!parseSwizzleOperands(1, &GroupSize,
3480                             2, 32,
3481                             "group size must be in the interval [2,32]")) {
3482     return false;
3483   }
3484   if (!isPowerOf2_64(GroupSize)) {
3485     Error(S, "group size must be a power of two");
3486     return false;
3487   }
3488   if (parseSwizzleOperands(1, &LaneIdx,
3489                            0, GroupSize - 1,
3490                            "lane id must be in the interval [0,group size - 1]")) {
3491     Imm = encodeBitmaskPerm(BITMASK_MAX - GroupSize + 1, LaneIdx, 0);
3492     return true;
3493   }
3494   return false;
3495 }
3496 
3497 bool
3498 AMDGPUAsmParser::parseSwizzleReverse(int64_t &Imm) {
3499   using namespace llvm::AMDGPU::Swizzle;
3500 
3501   SMLoc S = Parser.getTok().getLoc();
3502   int64_t GroupSize;
3503 
3504   if (!parseSwizzleOperands(1, &GroupSize,
3505       2, 32, "group size must be in the interval [2,32]")) {
3506     return false;
3507   }
3508   if (!isPowerOf2_64(GroupSize)) {
3509     Error(S, "group size must be a power of two");
3510     return false;
3511   }
3512 
3513   Imm = encodeBitmaskPerm(BITMASK_MAX, 0, GroupSize - 1);
3514   return true;
3515 }
3516 
3517 bool
3518 AMDGPUAsmParser::parseSwizzleSwap(int64_t &Imm) {
3519   using namespace llvm::AMDGPU::Swizzle;
3520 
3521   SMLoc S = Parser.getTok().getLoc();
3522   int64_t GroupSize;
3523 
3524   if (!parseSwizzleOperands(1, &GroupSize,
3525       1, 16, "group size must be in the interval [1,16]")) {
3526     return false;
3527   }
3528   if (!isPowerOf2_64(GroupSize)) {
3529     Error(S, "group size must be a power of two");
3530     return false;
3531   }
3532 
3533   Imm = encodeBitmaskPerm(BITMASK_MAX, 0, GroupSize);
3534   return true;
3535 }
3536 
3537 bool
3538 AMDGPUAsmParser::parseSwizzleBitmaskPerm(int64_t &Imm) {
3539   using namespace llvm::AMDGPU::Swizzle;
3540 
3541   if (!skipToken(AsmToken::Comma, "expected a comma")) {
3542     return false;
3543   }
3544 
3545   StringRef Ctl;
3546   SMLoc StrLoc = Parser.getTok().getLoc();
3547   if (!parseString(Ctl)) {
3548     return false;
3549   }
3550   if (Ctl.size() != BITMASK_WIDTH) {
3551     Error(StrLoc, "expected a 5-character mask");
3552     return false;
3553   }
3554 
3555   unsigned AndMask = 0;
3556   unsigned OrMask = 0;
3557   unsigned XorMask = 0;
3558 
3559   for (size_t i = 0; i < Ctl.size(); ++i) {
3560     unsigned Mask = 1 << (BITMASK_WIDTH - 1 - i);
3561     switch(Ctl[i]) {
3562     default:
3563       Error(StrLoc, "invalid mask");
3564       return false;
3565     case '0':
3566       break;
3567     case '1':
3568       OrMask |= Mask;
3569       break;
3570     case 'p':
3571       AndMask |= Mask;
3572       break;
3573     case 'i':
3574       AndMask |= Mask;
3575       XorMask |= Mask;
3576       break;
3577     }
3578   }
3579 
3580   Imm = encodeBitmaskPerm(AndMask, OrMask, XorMask);
3581   return true;
3582 }
3583 
3584 bool
3585 AMDGPUAsmParser::parseSwizzleOffset(int64_t &Imm) {
3586 
3587   SMLoc OffsetLoc = Parser.getTok().getLoc();
3588 
3589   if (!parseExpr(Imm)) {
3590     return false;
3591   }
3592   if (!isUInt<16>(Imm)) {
3593     Error(OffsetLoc, "expected a 16-bit offset");
3594     return false;
3595   }
3596   return true;
3597 }
3598 
3599 bool
3600 AMDGPUAsmParser::parseSwizzleMacro(int64_t &Imm) {
3601   using namespace llvm::AMDGPU::Swizzle;
3602 
3603   if (skipToken(AsmToken::LParen, "expected a left parentheses")) {
3604 
3605     SMLoc ModeLoc = Parser.getTok().getLoc();
3606     bool Ok = false;
3607 
3608     if (trySkipId(IdSymbolic[ID_QUAD_PERM])) {
3609       Ok = parseSwizzleQuadPerm(Imm);
3610     } else if (trySkipId(IdSymbolic[ID_BITMASK_PERM])) {
3611       Ok = parseSwizzleBitmaskPerm(Imm);
3612     } else if (trySkipId(IdSymbolic[ID_BROADCAST])) {
3613       Ok = parseSwizzleBroadcast(Imm);
3614     } else if (trySkipId(IdSymbolic[ID_SWAP])) {
3615       Ok = parseSwizzleSwap(Imm);
3616     } else if (trySkipId(IdSymbolic[ID_REVERSE])) {
3617       Ok = parseSwizzleReverse(Imm);
3618     } else {
3619       Error(ModeLoc, "expected a swizzle mode");
3620     }
3621 
3622     return Ok && skipToken(AsmToken::RParen, "expected a closing parentheses");
3623   }
3624 
3625   return false;
3626 }
3627 
3628 OperandMatchResultTy
3629 AMDGPUAsmParser::parseSwizzleOp(OperandVector &Operands) {
3630   SMLoc S = Parser.getTok().getLoc();
3631   int64_t Imm = 0;
3632 
3633   if (trySkipId("offset")) {
3634 
3635     bool Ok = false;
3636     if (skipToken(AsmToken::Colon, "expected a colon")) {
3637       if (trySkipId("swizzle")) {
3638         Ok = parseSwizzleMacro(Imm);
3639       } else {
3640         Ok = parseSwizzleOffset(Imm);
3641       }
3642     }
3643 
3644     Operands.push_back(AMDGPUOperand::CreateImm(this, Imm, S, AMDGPUOperand::ImmTySwizzle));
3645 
3646     return Ok? MatchOperand_Success : MatchOperand_ParseFail;
3647   } else {
3648     return MatchOperand_NoMatch;
3649   }
3650 }
3651 
3652 bool
3653 AMDGPUOperand::isSwizzle() const {
3654   return isImmTy(ImmTySwizzle);
3655 }
3656 
3657 //===----------------------------------------------------------------------===//
3658 // sopp branch targets
3659 //===----------------------------------------------------------------------===//
3660 
3661 OperandMatchResultTy
3662 AMDGPUAsmParser::parseSOppBrTarget(OperandVector &Operands) {
3663   SMLoc S = Parser.getTok().getLoc();
3664 
3665   switch (getLexer().getKind()) {
3666     default: return MatchOperand_ParseFail;
3667     case AsmToken::Integer: {
3668       int64_t Imm;
3669       if (getParser().parseAbsoluteExpression(Imm))
3670         return MatchOperand_ParseFail;
3671       Operands.push_back(AMDGPUOperand::CreateImm(this, Imm, S));
3672       return MatchOperand_Success;
3673     }
3674 
3675     case AsmToken::Identifier:
3676       Operands.push_back(AMDGPUOperand::CreateExpr(this,
3677           MCSymbolRefExpr::create(getContext().getOrCreateSymbol(
3678                                   Parser.getTok().getString()), getContext()), S));
3679       Parser.Lex();
3680       return MatchOperand_Success;
3681   }
3682 }
3683 
3684 //===----------------------------------------------------------------------===//
3685 // mubuf
3686 //===----------------------------------------------------------------------===//
3687 
3688 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultGLC() const {
3689   return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTyGLC);
3690 }
3691 
3692 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultSLC() const {
3693   return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTySLC);
3694 }
3695 
3696 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultTFE() const {
3697   return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTyTFE);
3698 }
3699 
3700 void AMDGPUAsmParser::cvtMubufImpl(MCInst &Inst,
3701                                const OperandVector &Operands,
3702                                bool IsAtomic, bool IsAtomicReturn) {
3703   OptionalImmIndexMap OptionalIdx;
3704   assert(IsAtomicReturn ? IsAtomic : true);
3705 
3706   for (unsigned i = 1, e = Operands.size(); i != e; ++i) {
3707     AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[i]);
3708 
3709     // Add the register arguments
3710     if (Op.isReg()) {
3711       Op.addRegOperands(Inst, 1);
3712       continue;
3713     }
3714 
3715     // Handle the case where soffset is an immediate
3716     if (Op.isImm() && Op.getImmTy() == AMDGPUOperand::ImmTyNone) {
3717       Op.addImmOperands(Inst, 1);
3718       continue;
3719     }
3720 
3721     // Handle tokens like 'offen' which are sometimes hard-coded into the
3722     // asm string.  There are no MCInst operands for these.
3723     if (Op.isToken()) {
3724       continue;
3725     }
3726     assert(Op.isImm());
3727 
3728     // Handle optional arguments
3729     OptionalIdx[Op.getImmTy()] = i;
3730   }
3731 
3732   // Copy $vdata_in operand and insert as $vdata for MUBUF_Atomic RTN insns.
3733   if (IsAtomicReturn) {
3734     MCInst::iterator I = Inst.begin(); // $vdata_in is always at the beginning.
3735     Inst.insert(I, *I);
3736   }
3737 
3738   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyOffset);
3739   if (!IsAtomic) { // glc is hard-coded.
3740     addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyGLC);
3741   }
3742   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySLC);
3743   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyTFE);
3744 }
3745 
3746 //===----------------------------------------------------------------------===//
3747 // mimg
3748 //===----------------------------------------------------------------------===//
3749 
3750 void AMDGPUAsmParser::cvtMIMG(MCInst &Inst, const OperandVector &Operands) {
3751   unsigned I = 1;
3752   const MCInstrDesc &Desc = MII.get(Inst.getOpcode());
3753   for (unsigned J = 0; J < Desc.getNumDefs(); ++J) {
3754     ((AMDGPUOperand &)*Operands[I++]).addRegOperands(Inst, 1);
3755   }
3756 
3757   OptionalImmIndexMap OptionalIdx;
3758 
3759   for (unsigned E = Operands.size(); I != E; ++I) {
3760     AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[I]);
3761 
3762     // Add the register arguments
3763     if (Op.isRegOrImm()) {
3764       Op.addRegOrImmOperands(Inst, 1);
3765       continue;
3766     } else if (Op.isImmModifier()) {
3767       OptionalIdx[Op.getImmTy()] = I;
3768     } else {
3769       llvm_unreachable("unexpected operand type");
3770     }
3771   }
3772 
3773   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyDMask);
3774   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyUNorm);
3775   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyGLC);
3776   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyDA);
3777   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyR128);
3778   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyTFE);
3779   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyLWE);
3780   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySLC);
3781 }
3782 
3783 void AMDGPUAsmParser::cvtMIMGAtomic(MCInst &Inst, const OperandVector &Operands) {
3784   unsigned I = 1;
3785   const MCInstrDesc &Desc = MII.get(Inst.getOpcode());
3786   for (unsigned J = 0; J < Desc.getNumDefs(); ++J) {
3787     ((AMDGPUOperand &)*Operands[I++]).addRegOperands(Inst, 1);
3788   }
3789 
3790   // Add src, same as dst
3791   ((AMDGPUOperand &)*Operands[I]).addRegOperands(Inst, 1);
3792 
3793   OptionalImmIndexMap OptionalIdx;
3794 
3795   for (unsigned E = Operands.size(); I != E; ++I) {
3796     AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[I]);
3797 
3798     // Add the register arguments
3799     if (Op.isRegOrImm()) {
3800       Op.addRegOrImmOperands(Inst, 1);
3801       continue;
3802     } else if (Op.isImmModifier()) {
3803       OptionalIdx[Op.getImmTy()] = I;
3804     } else {
3805       llvm_unreachable("unexpected operand type");
3806     }
3807   }
3808 
3809   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyDMask);
3810   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyUNorm);
3811   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyGLC);
3812   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyDA);
3813   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyR128);
3814   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyTFE);
3815   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyLWE);
3816   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySLC);
3817 }
3818 
3819 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultDMask() const {
3820   return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTyDMask);
3821 }
3822 
3823 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultUNorm() const {
3824   return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTyUNorm);
3825 }
3826 
3827 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultDA() const {
3828   return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTyDA);
3829 }
3830 
3831 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultR128() const {
3832   return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTyR128);
3833 }
3834 
3835 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultLWE() const {
3836   return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTyLWE);
3837 }
3838 
3839 //===----------------------------------------------------------------------===//
3840 // smrd
3841 //===----------------------------------------------------------------------===//
3842 
3843 bool AMDGPUOperand::isSMRDOffset8() const {
3844   return isImm() && isUInt<8>(getImm());
3845 }
3846 
3847 bool AMDGPUOperand::isSMRDOffset20() const {
3848   return isImm() && isUInt<20>(getImm());
3849 }
3850 
3851 bool AMDGPUOperand::isSMRDLiteralOffset() const {
3852   // 32-bit literals are only supported on CI and we only want to use them
3853   // when the offset is > 8-bits.
3854   return isImm() && !isUInt<8>(getImm()) && isUInt<32>(getImm());
3855 }
3856 
3857 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultSMRDOffset8() const {
3858   return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTyOffset);
3859 }
3860 
3861 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultSMRDOffset20() const {
3862   return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTyOffset);
3863 }
3864 
3865 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultSMRDLiteralOffset() const {
3866   return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTyOffset);
3867 }
3868 
3869 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultOffsetU12() const {
3870   return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTyOffset);
3871 }
3872 
3873 //===----------------------------------------------------------------------===//
3874 // vop3
3875 //===----------------------------------------------------------------------===//
3876 
3877 static bool ConvertOmodMul(int64_t &Mul) {
3878   if (Mul != 1 && Mul != 2 && Mul != 4)
3879     return false;
3880 
3881   Mul >>= 1;
3882   return true;
3883 }
3884 
3885 static bool ConvertOmodDiv(int64_t &Div) {
3886   if (Div == 1) {
3887     Div = 0;
3888     return true;
3889   }
3890 
3891   if (Div == 2) {
3892     Div = 3;
3893     return true;
3894   }
3895 
3896   return false;
3897 }
3898 
3899 static bool ConvertBoundCtrl(int64_t &BoundCtrl) {
3900   if (BoundCtrl == 0) {
3901     BoundCtrl = 1;
3902     return true;
3903   }
3904 
3905   if (BoundCtrl == -1) {
3906     BoundCtrl = 0;
3907     return true;
3908   }
3909 
3910   return false;
3911 }
3912 
3913 // Note: the order in this table matches the order of operands in AsmString.
3914 static const OptionalOperand AMDGPUOptionalOperandTable[] = {
3915   {"offen",   AMDGPUOperand::ImmTyOffen, true, nullptr},
3916   {"idxen",   AMDGPUOperand::ImmTyIdxen, true, nullptr},
3917   {"addr64",  AMDGPUOperand::ImmTyAddr64, true, nullptr},
3918   {"offset0", AMDGPUOperand::ImmTyOffset0, false, nullptr},
3919   {"offset1", AMDGPUOperand::ImmTyOffset1, false, nullptr},
3920   {"gds",     AMDGPUOperand::ImmTyGDS, true, nullptr},
3921   {"offset",  AMDGPUOperand::ImmTyOffset, false, nullptr},
3922   {"glc",     AMDGPUOperand::ImmTyGLC, true, nullptr},
3923   {"slc",     AMDGPUOperand::ImmTySLC, true, nullptr},
3924   {"tfe",     AMDGPUOperand::ImmTyTFE, true, nullptr},
3925   {"clamp",   AMDGPUOperand::ImmTyClampSI, true, nullptr},
3926   {"omod",    AMDGPUOperand::ImmTyOModSI, false, ConvertOmodMul},
3927   {"unorm",   AMDGPUOperand::ImmTyUNorm, true, nullptr},
3928   {"da",      AMDGPUOperand::ImmTyDA,    true, nullptr},
3929   {"r128",    AMDGPUOperand::ImmTyR128,  true, nullptr},
3930   {"lwe",     AMDGPUOperand::ImmTyLWE,   true, nullptr},
3931   {"dmask",   AMDGPUOperand::ImmTyDMask, false, nullptr},
3932   {"row_mask",   AMDGPUOperand::ImmTyDppRowMask, false, nullptr},
3933   {"bank_mask",  AMDGPUOperand::ImmTyDppBankMask, false, nullptr},
3934   {"bound_ctrl", AMDGPUOperand::ImmTyDppBoundCtrl, false, ConvertBoundCtrl},
3935   {"dst_sel",    AMDGPUOperand::ImmTySdwaDstSel, false, nullptr},
3936   {"src0_sel",   AMDGPUOperand::ImmTySdwaSrc0Sel, false, nullptr},
3937   {"src1_sel",   AMDGPUOperand::ImmTySdwaSrc1Sel, false, nullptr},
3938   {"dst_unused", AMDGPUOperand::ImmTySdwaDstUnused, false, nullptr},
3939   {"compr", AMDGPUOperand::ImmTyExpCompr, true, nullptr },
3940   {"vm", AMDGPUOperand::ImmTyExpVM, true, nullptr},
3941   {"op_sel", AMDGPUOperand::ImmTyOpSel, false, nullptr},
3942   {"op_sel_hi", AMDGPUOperand::ImmTyOpSelHi, false, nullptr},
3943   {"neg_lo", AMDGPUOperand::ImmTyNegLo, false, nullptr},
3944   {"neg_hi", AMDGPUOperand::ImmTyNegHi, false, nullptr}
3945 };
3946 
3947 OperandMatchResultTy AMDGPUAsmParser::parseOptionalOperand(OperandVector &Operands) {
3948   OperandMatchResultTy res;
3949   for (const OptionalOperand &Op : AMDGPUOptionalOperandTable) {
3950     // try to parse any optional operand here
3951     if (Op.IsBit) {
3952       res = parseNamedBit(Op.Name, Operands, Op.Type);
3953     } else if (Op.Type == AMDGPUOperand::ImmTyOModSI) {
3954       res = parseOModOperand(Operands);
3955     } else if (Op.Type == AMDGPUOperand::ImmTySdwaDstSel ||
3956                Op.Type == AMDGPUOperand::ImmTySdwaSrc0Sel ||
3957                Op.Type == AMDGPUOperand::ImmTySdwaSrc1Sel) {
3958       res = parseSDWASel(Operands, Op.Name, Op.Type);
3959     } else if (Op.Type == AMDGPUOperand::ImmTySdwaDstUnused) {
3960       res = parseSDWADstUnused(Operands);
3961     } else if (Op.Type == AMDGPUOperand::ImmTyOpSel ||
3962                Op.Type == AMDGPUOperand::ImmTyOpSelHi ||
3963                Op.Type == AMDGPUOperand::ImmTyNegLo ||
3964                Op.Type == AMDGPUOperand::ImmTyNegHi) {
3965       res = parseOperandArrayWithPrefix(Op.Name, Operands, Op.Type,
3966                                         Op.ConvertResult);
3967     } else {
3968       res = parseIntWithPrefix(Op.Name, Operands, Op.Type, Op.ConvertResult);
3969     }
3970     if (res != MatchOperand_NoMatch) {
3971       return res;
3972     }
3973   }
3974   return MatchOperand_NoMatch;
3975 }
3976 
3977 OperandMatchResultTy AMDGPUAsmParser::parseOModOperand(OperandVector &Operands) {
3978   StringRef Name = Parser.getTok().getString();
3979   if (Name == "mul") {
3980     return parseIntWithPrefix("mul", Operands,
3981                               AMDGPUOperand::ImmTyOModSI, ConvertOmodMul);
3982   }
3983 
3984   if (Name == "div") {
3985     return parseIntWithPrefix("div", Operands,
3986                               AMDGPUOperand::ImmTyOModSI, ConvertOmodDiv);
3987   }
3988 
3989   return MatchOperand_NoMatch;
3990 }
3991 
3992 void AMDGPUAsmParser::cvtId(MCInst &Inst, const OperandVector &Operands) {
3993   unsigned I = 1;
3994   const MCInstrDesc &Desc = MII.get(Inst.getOpcode());
3995   for (unsigned J = 0; J < Desc.getNumDefs(); ++J) {
3996     ((AMDGPUOperand &)*Operands[I++]).addRegOperands(Inst, 1);
3997   }
3998   for (unsigned E = Operands.size(); I != E; ++I)
3999     ((AMDGPUOperand &)*Operands[I]).addRegOrImmOperands(Inst, 1);
4000 }
4001 
4002 void AMDGPUAsmParser::cvtVOP3_2_mod(MCInst &Inst, const OperandVector &Operands) {
4003   uint64_t TSFlags = MII.get(Inst.getOpcode()).TSFlags;
4004   if (TSFlags & SIInstrFlags::VOP3) {
4005     cvtVOP3(Inst, Operands);
4006   } else {
4007     cvtId(Inst, Operands);
4008   }
4009 }
4010 
4011 static bool isRegOrImmWithInputMods(const MCInstrDesc &Desc, unsigned OpNum) {
4012       // 1. This operand is input modifiers
4013   return Desc.OpInfo[OpNum].OperandType == AMDGPU::OPERAND_INPUT_MODS
4014       // 2. This is not last operand
4015       && Desc.NumOperands > (OpNum + 1)
4016       // 3. Next operand is register class
4017       && Desc.OpInfo[OpNum + 1].RegClass != -1
4018       // 4. Next register is not tied to any other operand
4019       && Desc.getOperandConstraint(OpNum + 1, MCOI::OperandConstraint::TIED_TO) == -1;
4020 }
4021 
4022 void AMDGPUAsmParser::cvtVOP3Impl(MCInst &Inst, const OperandVector &Operands,
4023                                   OptionalImmIndexMap &OptionalIdx) {
4024   unsigned I = 1;
4025   const MCInstrDesc &Desc = MII.get(Inst.getOpcode());
4026   for (unsigned J = 0; J < Desc.getNumDefs(); ++J) {
4027     ((AMDGPUOperand &)*Operands[I++]).addRegOperands(Inst, 1);
4028   }
4029 
4030   for (unsigned E = Operands.size(); I != E; ++I) {
4031     AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[I]);
4032     if (isRegOrImmWithInputMods(Desc, Inst.getNumOperands())) {
4033       Op.addRegOrImmWithFPInputModsOperands(Inst, 2);
4034     } else if (Op.isImmModifier()) {
4035       OptionalIdx[Op.getImmTy()] = I;
4036     } else if (Op.isRegOrImm()) {
4037       Op.addRegOrImmOperands(Inst, 1);
4038     } else {
4039       llvm_unreachable("unhandled operand type");
4040     }
4041   }
4042 }
4043 
4044 void AMDGPUAsmParser::cvtVOP3(MCInst &Inst, const OperandVector &Operands) {
4045   OptionalImmIndexMap OptionalIdx;
4046 
4047   cvtVOP3Impl(Inst, Operands, OptionalIdx);
4048 
4049   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyClampSI);
4050   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyOModSI);
4051 
4052   // special case v_mac_{f16, f32}:
4053   // it has src2 register operand that is tied to dst operand
4054   // we don't allow modifiers for this operand in assembler so src2_modifiers
4055   // should be 0
4056   if (Inst.getOpcode() == AMDGPU::V_MAC_F32_e64_si ||
4057       Inst.getOpcode() == AMDGPU::V_MAC_F32_e64_vi ||
4058       Inst.getOpcode() == AMDGPU::V_MAC_F16_e64_vi) {
4059     auto it = Inst.begin();
4060     std::advance(
4061       it,
4062       AMDGPU::getNamedOperandIdx(Inst.getOpcode() == AMDGPU::V_MAC_F16_e64_vi ?
4063                                      AMDGPU::V_MAC_F16_e64 :
4064                                      AMDGPU::V_MAC_F32_e64,
4065                                  AMDGPU::OpName::src2_modifiers));
4066     it = Inst.insert(it, MCOperand::createImm(0)); // no modifiers for src2
4067     ++it;
4068     Inst.insert(it, Inst.getOperand(0)); // src2 = dst
4069   }
4070 }
4071 
4072 void AMDGPUAsmParser::cvtVOP3OMod(MCInst &Inst, const OperandVector &Operands) {
4073   OptionalImmIndexMap OptionalIdx;
4074 
4075   unsigned I = 1;
4076   const MCInstrDesc &Desc = MII.get(Inst.getOpcode());
4077   for (unsigned J = 0; J < Desc.getNumDefs(); ++J) {
4078     ((AMDGPUOperand &)*Operands[I++]).addRegOperands(Inst, 1);
4079   }
4080 
4081   for (unsigned E = Operands.size(); I != E; ++I) {
4082     AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[I]);
4083     if (Op.isMod()) {
4084       OptionalIdx[Op.getImmTy()] = I;
4085     } else {
4086       Op.addRegOrImmOperands(Inst, 1);
4087     }
4088   }
4089 
4090   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyClampSI);
4091   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyOModSI);
4092 }
4093 
4094 void AMDGPUAsmParser::cvtVOP3P(MCInst &Inst, const OperandVector &Operands) {
4095   OptionalImmIndexMap OptIdx;
4096 
4097   cvtVOP3Impl(Inst, Operands, OptIdx);
4098 
4099   // FIXME: This is messy. Parse the modifiers as if it was a normal VOP3
4100   // instruction, and then figure out where to actually put the modifiers
4101   int Opc = Inst.getOpcode();
4102 
4103   if (AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::clamp) != -1) {
4104     addOptionalImmOperand(Inst, Operands, OptIdx, AMDGPUOperand::ImmTyClampSI);
4105   }
4106 
4107   addOptionalImmOperand(Inst, Operands, OptIdx, AMDGPUOperand::ImmTyOpSel);
4108   addOptionalImmOperand(Inst, Operands, OptIdx, AMDGPUOperand::ImmTyOpSelHi, -1);
4109 
4110   int NegLoIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::neg_lo);
4111   if (NegLoIdx != -1) {
4112     addOptionalImmOperand(Inst, Operands, OptIdx, AMDGPUOperand::ImmTyNegLo);
4113     addOptionalImmOperand(Inst, Operands, OptIdx, AMDGPUOperand::ImmTyNegHi);
4114   }
4115 
4116   const int Ops[] = { AMDGPU::OpName::src0,
4117                       AMDGPU::OpName::src1,
4118                       AMDGPU::OpName::src2 };
4119   const int ModOps[] = { AMDGPU::OpName::src0_modifiers,
4120                          AMDGPU::OpName::src1_modifiers,
4121                          AMDGPU::OpName::src2_modifiers };
4122 
4123   int OpSelIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::op_sel);
4124   int OpSelHiIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::op_sel_hi);
4125 
4126   unsigned OpSel = Inst.getOperand(OpSelIdx).getImm();
4127   unsigned OpSelHi = Inst.getOperand(OpSelHiIdx).getImm();
4128   unsigned NegLo = 0;
4129   unsigned NegHi = 0;
4130 
4131   if (NegLoIdx != -1) {
4132     int NegHiIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::neg_hi);
4133     NegLo = Inst.getOperand(NegLoIdx).getImm();
4134     NegHi = Inst.getOperand(NegHiIdx).getImm();
4135   }
4136 
4137   for (int J = 0; J < 3; ++J) {
4138     int OpIdx = AMDGPU::getNamedOperandIdx(Opc, Ops[J]);
4139     if (OpIdx == -1)
4140       break;
4141 
4142     uint32_t ModVal = 0;
4143 
4144     if ((OpSel & (1 << J)) != 0)
4145       ModVal |= SISrcMods::OP_SEL_0;
4146 
4147     if ((OpSelHi & (1 << J)) != 0)
4148       ModVal |= SISrcMods::OP_SEL_1;
4149 
4150     if ((NegLo & (1 << J)) != 0)
4151       ModVal |= SISrcMods::NEG;
4152 
4153     if ((NegHi & (1 << J)) != 0)
4154       ModVal |= SISrcMods::NEG_HI;
4155 
4156     int ModIdx = AMDGPU::getNamedOperandIdx(Opc, ModOps[J]);
4157 
4158     Inst.getOperand(ModIdx).setImm(ModVal);
4159   }
4160 }
4161 
4162 //===----------------------------------------------------------------------===//
4163 // dpp
4164 //===----------------------------------------------------------------------===//
4165 
4166 bool AMDGPUOperand::isDPPCtrl() const {
4167   bool result = isImm() && getImmTy() == ImmTyDppCtrl && isUInt<9>(getImm());
4168   if (result) {
4169     int64_t Imm = getImm();
4170     return ((Imm >= 0x000) && (Imm <= 0x0ff)) ||
4171            ((Imm >= 0x101) && (Imm <= 0x10f)) ||
4172            ((Imm >= 0x111) && (Imm <= 0x11f)) ||
4173            ((Imm >= 0x121) && (Imm <= 0x12f)) ||
4174            (Imm == 0x130) ||
4175            (Imm == 0x134) ||
4176            (Imm == 0x138) ||
4177            (Imm == 0x13c) ||
4178            (Imm == 0x140) ||
4179            (Imm == 0x141) ||
4180            (Imm == 0x142) ||
4181            (Imm == 0x143);
4182   }
4183   return false;
4184 }
4185 
4186 bool AMDGPUOperand::isGPRIdxMode() const {
4187   return isImm() && isUInt<4>(getImm());
4188 }
4189 
4190 bool AMDGPUOperand::isS16Imm() const {
4191   return isImm() && (isInt<16>(getImm()) || isUInt<16>(getImm()));
4192 }
4193 
4194 bool AMDGPUOperand::isU16Imm() const {
4195   return isImm() && isUInt<16>(getImm());
4196 }
4197 
4198 OperandMatchResultTy
4199 AMDGPUAsmParser::parseDPPCtrl(OperandVector &Operands) {
4200   SMLoc S = Parser.getTok().getLoc();
4201   StringRef Prefix;
4202   int64_t Int;
4203 
4204   if (getLexer().getKind() == AsmToken::Identifier) {
4205     Prefix = Parser.getTok().getString();
4206   } else {
4207     return MatchOperand_NoMatch;
4208   }
4209 
4210   if (Prefix == "row_mirror") {
4211     Int = 0x140;
4212     Parser.Lex();
4213   } else if (Prefix == "row_half_mirror") {
4214     Int = 0x141;
4215     Parser.Lex();
4216   } else {
4217     // Check to prevent parseDPPCtrlOps from eating invalid tokens
4218     if (Prefix != "quad_perm"
4219         && Prefix != "row_shl"
4220         && Prefix != "row_shr"
4221         && Prefix != "row_ror"
4222         && Prefix != "wave_shl"
4223         && Prefix != "wave_rol"
4224         && Prefix != "wave_shr"
4225         && Prefix != "wave_ror"
4226         && Prefix != "row_bcast") {
4227       return MatchOperand_NoMatch;
4228     }
4229 
4230     Parser.Lex();
4231     if (getLexer().isNot(AsmToken::Colon))
4232       return MatchOperand_ParseFail;
4233 
4234     if (Prefix == "quad_perm") {
4235       // quad_perm:[%d,%d,%d,%d]
4236       Parser.Lex();
4237       if (getLexer().isNot(AsmToken::LBrac))
4238         return MatchOperand_ParseFail;
4239       Parser.Lex();
4240 
4241       if (getParser().parseAbsoluteExpression(Int) || !(0 <= Int && Int <=3))
4242         return MatchOperand_ParseFail;
4243 
4244       for (int i = 0; i < 3; ++i) {
4245         if (getLexer().isNot(AsmToken::Comma))
4246           return MatchOperand_ParseFail;
4247         Parser.Lex();
4248 
4249         int64_t Temp;
4250         if (getParser().parseAbsoluteExpression(Temp) || !(0 <= Temp && Temp <=3))
4251           return MatchOperand_ParseFail;
4252         const int shift = i*2 + 2;
4253         Int += (Temp << shift);
4254       }
4255 
4256       if (getLexer().isNot(AsmToken::RBrac))
4257         return MatchOperand_ParseFail;
4258       Parser.Lex();
4259 
4260     } else {
4261       // sel:%d
4262       Parser.Lex();
4263       if (getParser().parseAbsoluteExpression(Int))
4264         return MatchOperand_ParseFail;
4265 
4266       if (Prefix == "row_shl" && 1 <= Int && Int <= 15) {
4267         Int |= 0x100;
4268       } else if (Prefix == "row_shr" && 1 <= Int && Int <= 15) {
4269         Int |= 0x110;
4270       } else if (Prefix == "row_ror" && 1 <= Int && Int <= 15) {
4271         Int |= 0x120;
4272       } else if (Prefix == "wave_shl" && 1 == Int) {
4273         Int = 0x130;
4274       } else if (Prefix == "wave_rol" && 1 == Int) {
4275         Int = 0x134;
4276       } else if (Prefix == "wave_shr" && 1 == Int) {
4277         Int = 0x138;
4278       } else if (Prefix == "wave_ror" && 1 == Int) {
4279         Int = 0x13C;
4280       } else if (Prefix == "row_bcast") {
4281         if (Int == 15) {
4282           Int = 0x142;
4283         } else if (Int == 31) {
4284           Int = 0x143;
4285         } else {
4286           return MatchOperand_ParseFail;
4287         }
4288       } else {
4289         return MatchOperand_ParseFail;
4290       }
4291     }
4292   }
4293 
4294   Operands.push_back(AMDGPUOperand::CreateImm(this, Int, S, AMDGPUOperand::ImmTyDppCtrl));
4295   return MatchOperand_Success;
4296 }
4297 
4298 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultRowMask() const {
4299   return AMDGPUOperand::CreateImm(this, 0xf, SMLoc(), AMDGPUOperand::ImmTyDppRowMask);
4300 }
4301 
4302 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultBankMask() const {
4303   return AMDGPUOperand::CreateImm(this, 0xf, SMLoc(), AMDGPUOperand::ImmTyDppBankMask);
4304 }
4305 
4306 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultBoundCtrl() const {
4307   return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTyDppBoundCtrl);
4308 }
4309 
4310 void AMDGPUAsmParser::cvtDPP(MCInst &Inst, const OperandVector &Operands) {
4311   OptionalImmIndexMap OptionalIdx;
4312 
4313   unsigned I = 1;
4314   const MCInstrDesc &Desc = MII.get(Inst.getOpcode());
4315   for (unsigned J = 0; J < Desc.getNumDefs(); ++J) {
4316     ((AMDGPUOperand &)*Operands[I++]).addRegOperands(Inst, 1);
4317   }
4318 
4319   for (unsigned E = Operands.size(); I != E; ++I) {
4320     AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[I]);
4321     // Add the register arguments
4322     if (Op.isReg() && Op.Reg.RegNo == AMDGPU::VCC) {
4323       // VOP2b (v_add_u32, v_sub_u32 ...) dpp use "vcc" token.
4324       // Skip it.
4325       continue;
4326     } if (isRegOrImmWithInputMods(Desc, Inst.getNumOperands())) {
4327       Op.addRegWithFPInputModsOperands(Inst, 2);
4328     } else if (Op.isDPPCtrl()) {
4329       Op.addImmOperands(Inst, 1);
4330     } else if (Op.isImm()) {
4331       // Handle optional arguments
4332       OptionalIdx[Op.getImmTy()] = I;
4333     } else {
4334       llvm_unreachable("Invalid operand type");
4335     }
4336   }
4337 
4338   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyDppRowMask, 0xf);
4339   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyDppBankMask, 0xf);
4340   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyDppBoundCtrl);
4341 
4342   // special case v_mac_{f16, f32}:
4343   // it has src2 register operand that is tied to dst operand
4344   if (Inst.getOpcode() == AMDGPU::V_MAC_F32_dpp ||
4345       Inst.getOpcode() == AMDGPU::V_MAC_F16_dpp) {
4346     auto it = Inst.begin();
4347     std::advance(
4348         it, AMDGPU::getNamedOperandIdx(Inst.getOpcode(), AMDGPU::OpName::src2));
4349     Inst.insert(it, Inst.getOperand(0)); // src2 = dst
4350   }
4351 }
4352 
4353 //===----------------------------------------------------------------------===//
4354 // sdwa
4355 //===----------------------------------------------------------------------===//
4356 
4357 OperandMatchResultTy
4358 AMDGPUAsmParser::parseSDWASel(OperandVector &Operands, StringRef Prefix,
4359                               AMDGPUOperand::ImmTy Type) {
4360   using namespace llvm::AMDGPU::SDWA;
4361 
4362   SMLoc S = Parser.getTok().getLoc();
4363   StringRef Value;
4364   OperandMatchResultTy res;
4365 
4366   res = parseStringWithPrefix(Prefix, Value);
4367   if (res != MatchOperand_Success) {
4368     return res;
4369   }
4370 
4371   int64_t Int;
4372   Int = StringSwitch<int64_t>(Value)
4373         .Case("BYTE_0", SdwaSel::BYTE_0)
4374         .Case("BYTE_1", SdwaSel::BYTE_1)
4375         .Case("BYTE_2", SdwaSel::BYTE_2)
4376         .Case("BYTE_3", SdwaSel::BYTE_3)
4377         .Case("WORD_0", SdwaSel::WORD_0)
4378         .Case("WORD_1", SdwaSel::WORD_1)
4379         .Case("DWORD", SdwaSel::DWORD)
4380         .Default(0xffffffff);
4381   Parser.Lex(); // eat last token
4382 
4383   if (Int == 0xffffffff) {
4384     return MatchOperand_ParseFail;
4385   }
4386 
4387   Operands.push_back(AMDGPUOperand::CreateImm(this, Int, S, Type));
4388   return MatchOperand_Success;
4389 }
4390 
4391 OperandMatchResultTy
4392 AMDGPUAsmParser::parseSDWADstUnused(OperandVector &Operands) {
4393   using namespace llvm::AMDGPU::SDWA;
4394 
4395   SMLoc S = Parser.getTok().getLoc();
4396   StringRef Value;
4397   OperandMatchResultTy res;
4398 
4399   res = parseStringWithPrefix("dst_unused", Value);
4400   if (res != MatchOperand_Success) {
4401     return res;
4402   }
4403 
4404   int64_t Int;
4405   Int = StringSwitch<int64_t>(Value)
4406         .Case("UNUSED_PAD", DstUnused::UNUSED_PAD)
4407         .Case("UNUSED_SEXT", DstUnused::UNUSED_SEXT)
4408         .Case("UNUSED_PRESERVE", DstUnused::UNUSED_PRESERVE)
4409         .Default(0xffffffff);
4410   Parser.Lex(); // eat last token
4411 
4412   if (Int == 0xffffffff) {
4413     return MatchOperand_ParseFail;
4414   }
4415 
4416   Operands.push_back(AMDGPUOperand::CreateImm(this, Int, S, AMDGPUOperand::ImmTySdwaDstUnused));
4417   return MatchOperand_Success;
4418 }
4419 
4420 void AMDGPUAsmParser::cvtSdwaVOP1(MCInst &Inst, const OperandVector &Operands) {
4421   cvtSDWA(Inst, Operands, SIInstrFlags::VOP1);
4422 }
4423 
4424 void AMDGPUAsmParser::cvtSdwaVOP2(MCInst &Inst, const OperandVector &Operands) {
4425   cvtSDWA(Inst, Operands, SIInstrFlags::VOP2);
4426 }
4427 
4428 void AMDGPUAsmParser::cvtSdwaVOP2b(MCInst &Inst, const OperandVector &Operands) {
4429   cvtSDWA(Inst, Operands, SIInstrFlags::VOP2, true);
4430 }
4431 
4432 void AMDGPUAsmParser::cvtSdwaVOPC(MCInst &Inst, const OperandVector &Operands) {
4433   cvtSDWA(Inst, Operands, SIInstrFlags::VOPC, isVI());
4434 }
4435 
4436 void AMDGPUAsmParser::cvtSDWA(MCInst &Inst, const OperandVector &Operands,
4437                               uint64_t BasicInstType, bool skipVcc) {
4438   using namespace llvm::AMDGPU::SDWA;
4439   OptionalImmIndexMap OptionalIdx;
4440   bool skippedVcc = false;
4441 
4442   unsigned I = 1;
4443   const MCInstrDesc &Desc = MII.get(Inst.getOpcode());
4444   for (unsigned J = 0; J < Desc.getNumDefs(); ++J) {
4445     ((AMDGPUOperand &)*Operands[I++]).addRegOperands(Inst, 1);
4446   }
4447 
4448   for (unsigned E = Operands.size(); I != E; ++I) {
4449     AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[I]);
4450     if (skipVcc && !skippedVcc && Op.isReg() && Op.Reg.RegNo == AMDGPU::VCC) {
4451       // VOP2b (v_add_u32, v_sub_u32 ...) sdwa use "vcc" token as dst.
4452       // Skip it if it's 2nd (e.g. v_add_i32_sdwa v1, vcc, v2, v3)
4453       // or 4th (v_addc_u32_sdwa v1, vcc, v2, v3, vcc) operand.
4454       // Skip VCC only if we didn't skip it on previous iteration.
4455       if (BasicInstType == SIInstrFlags::VOP2 &&
4456           (Inst.getNumOperands() == 1 || Inst.getNumOperands() == 5)) {
4457         skippedVcc = true;
4458         continue;
4459       } else if (BasicInstType == SIInstrFlags::VOPC &&
4460                  Inst.getNumOperands() == 0) {
4461         skippedVcc = true;
4462         continue;
4463       }
4464     }
4465     if (isRegOrImmWithInputMods(Desc, Inst.getNumOperands())) {
4466       Op.addRegWithInputModsOperands(Inst, 2);
4467     } else if (Op.isImm()) {
4468       // Handle optional arguments
4469       OptionalIdx[Op.getImmTy()] = I;
4470     } else {
4471       llvm_unreachable("Invalid operand type");
4472     }
4473     skippedVcc = false;
4474   }
4475 
4476   if (Inst.getOpcode() != AMDGPU::V_NOP_sdwa_gfx9 &&
4477       Inst.getOpcode() != AMDGPU::V_NOP_sdwa_vi) {
4478     // V_NOP_sdwa_vi has no optional sdwa arguments
4479     switch (BasicInstType) {
4480     case SIInstrFlags::VOP1:
4481       addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyClampSI, 0);
4482       if (isGFX9() &&
4483           AMDGPU::getNamedOperandIdx(Inst.getOpcode(), AMDGPU::OpName::omod) != -1) {
4484         addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyOModSI, 0);
4485       }
4486       addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySdwaDstSel, SdwaSel::DWORD);
4487       addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySdwaDstUnused, DstUnused::UNUSED_PRESERVE);
4488       addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySdwaSrc0Sel, SdwaSel::DWORD);
4489       break;
4490 
4491     case SIInstrFlags::VOP2:
4492       addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyClampSI, 0);
4493       if (isGFX9() &&
4494           AMDGPU::getNamedOperandIdx(Inst.getOpcode(), AMDGPU::OpName::omod) != -1) {
4495         addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyOModSI, 0);
4496       }
4497       addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySdwaDstSel, SdwaSel::DWORD);
4498       addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySdwaDstUnused, DstUnused::UNUSED_PRESERVE);
4499       addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySdwaSrc0Sel, SdwaSel::DWORD);
4500       addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySdwaSrc1Sel, SdwaSel::DWORD);
4501       break;
4502 
4503     case SIInstrFlags::VOPC:
4504       if (isVI()) {
4505         addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyClampSI, 0);
4506       }
4507       addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySdwaSrc0Sel, SdwaSel::DWORD);
4508       addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySdwaSrc1Sel, SdwaSel::DWORD);
4509       break;
4510 
4511     default:
4512       llvm_unreachable("Invalid instruction type. Only VOP1, VOP2 and VOPC allowed");
4513     }
4514   }
4515 
4516   // special case v_mac_{f16, f32}:
4517   // it has src2 register operand that is tied to dst operand
4518   if (Inst.getOpcode() == AMDGPU::V_MAC_F32_sdwa_vi ||
4519       Inst.getOpcode() == AMDGPU::V_MAC_F16_sdwa_vi)  {
4520     auto it = Inst.begin();
4521     std::advance(
4522       it, AMDGPU::getNamedOperandIdx(Inst.getOpcode(), AMDGPU::OpName::src2));
4523     Inst.insert(it, Inst.getOperand(0)); // src2 = dst
4524   }
4525 }
4526 
4527 /// Force static initialization.
4528 extern "C" void LLVMInitializeAMDGPUAsmParser() {
4529   RegisterMCAsmParser<AMDGPUAsmParser> A(getTheAMDGPUTarget());
4530   RegisterMCAsmParser<AMDGPUAsmParser> B(getTheGCNTarget());
4531 }
4532 
4533 #define GET_REGISTER_MATCHER
4534 #define GET_MATCHER_IMPLEMENTATION
4535 #include "AMDGPUGenAsmMatcher.inc"
4536 
4537 // This fuction should be defined after auto-generated include so that we have
4538 // MatchClassKind enum defined
4539 unsigned AMDGPUAsmParser::validateTargetOperandClass(MCParsedAsmOperand &Op,
4540                                                      unsigned Kind) {
4541   // Tokens like "glc" would be parsed as immediate operands in ParseOperand().
4542   // But MatchInstructionImpl() expects to meet token and fails to validate
4543   // operand. This method checks if we are given immediate operand but expect to
4544   // get corresponding token.
4545   AMDGPUOperand &Operand = (AMDGPUOperand&)Op;
4546   switch (Kind) {
4547   case MCK_addr64:
4548     return Operand.isAddr64() ? Match_Success : Match_InvalidOperand;
4549   case MCK_gds:
4550     return Operand.isGDS() ? Match_Success : Match_InvalidOperand;
4551   case MCK_glc:
4552     return Operand.isGLC() ? Match_Success : Match_InvalidOperand;
4553   case MCK_idxen:
4554     return Operand.isIdxen() ? Match_Success : Match_InvalidOperand;
4555   case MCK_offen:
4556     return Operand.isOffen() ? Match_Success : Match_InvalidOperand;
4557   case MCK_SSrcB32:
4558     // When operands have expression values, they will return true for isToken,
4559     // because it is not possible to distinguish between a token and an
4560     // expression at parse time. MatchInstructionImpl() will always try to
4561     // match an operand as a token, when isToken returns true, and when the
4562     // name of the expression is not a valid token, the match will fail,
4563     // so we need to handle it here.
4564     return Operand.isSSrcB32() ? Match_Success : Match_InvalidOperand;
4565   case MCK_SSrcF32:
4566     return Operand.isSSrcF32() ? Match_Success : Match_InvalidOperand;
4567   case MCK_SoppBrTarget:
4568     return Operand.isSoppBrTarget() ? Match_Success : Match_InvalidOperand;
4569   case MCK_VReg32OrOff:
4570     return Operand.isVReg32OrOff() ? Match_Success : Match_InvalidOperand;
4571   case MCK_InterpSlot:
4572     return Operand.isInterpSlot() ? Match_Success : Match_InvalidOperand;
4573   case MCK_Attr:
4574     return Operand.isInterpAttr() ? Match_Success : Match_InvalidOperand;
4575   case MCK_AttrChan:
4576     return Operand.isAttrChan() ? Match_Success : Match_InvalidOperand;
4577   default:
4578     return Match_InvalidOperand;
4579   }
4580 }
4581