1 //===- AMDGPUAsmParser.cpp - Parse SI asm to MCInst instructions ----------===//
2 //
3 //                     The LLVM Compiler Infrastructure
4 //
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
7 //
8 //===----------------------------------------------------------------------===//
9 
10 #include "AMDGPU.h"
11 #include "AMDKernelCodeT.h"
12 #include "MCTargetDesc/AMDGPUMCTargetDesc.h"
13 #include "MCTargetDesc/AMDGPUTargetStreamer.h"
14 #include "SIDefines.h"
15 #include "SIInstrInfo.h"
16 #include "Utils/AMDGPUAsmUtils.h"
17 #include "Utils/AMDGPUBaseInfo.h"
18 #include "Utils/AMDKernelCodeTUtils.h"
19 #include "llvm/ADT/APFloat.h"
20 #include "llvm/ADT/APInt.h"
21 #include "llvm/ADT/ArrayRef.h"
22 #include "llvm/ADT/STLExtras.h"
23 #include "llvm/ADT/SmallBitVector.h"
24 #include "llvm/ADT/SmallString.h"
25 #include "llvm/ADT/StringRef.h"
26 #include "llvm/ADT/StringSwitch.h"
27 #include "llvm/ADT/Twine.h"
28 #include "llvm/BinaryFormat/ELF.h"
29 #include "llvm/MC/MCAsmInfo.h"
30 #include "llvm/MC/MCContext.h"
31 #include "llvm/MC/MCExpr.h"
32 #include "llvm/MC/MCInst.h"
33 #include "llvm/MC/MCInstrDesc.h"
34 #include "llvm/MC/MCInstrInfo.h"
35 #include "llvm/MC/MCParser/MCAsmLexer.h"
36 #include "llvm/MC/MCParser/MCAsmParser.h"
37 #include "llvm/MC/MCParser/MCAsmParserExtension.h"
38 #include "llvm/MC/MCParser/MCParsedAsmOperand.h"
39 #include "llvm/MC/MCParser/MCTargetAsmParser.h"
40 #include "llvm/MC/MCRegisterInfo.h"
41 #include "llvm/MC/MCStreamer.h"
42 #include "llvm/MC/MCSubtargetInfo.h"
43 #include "llvm/MC/MCSymbol.h"
44 #include "llvm/Support/AMDGPUMetadata.h"
45 #include "llvm/Support/Casting.h"
46 #include "llvm/Support/Compiler.h"
47 #include "llvm/Support/ErrorHandling.h"
48 #include "llvm/Support/MachineValueType.h"
49 #include "llvm/Support/MathExtras.h"
50 #include "llvm/Support/SMLoc.h"
51 #include "llvm/Support/TargetRegistry.h"
52 #include "llvm/Support/raw_ostream.h"
53 #include <algorithm>
54 #include <cassert>
55 #include <cstdint>
56 #include <cstring>
57 #include <iterator>
58 #include <map>
59 #include <memory>
60 #include <string>
61 
62 using namespace llvm;
63 using namespace llvm::AMDGPU;
64 
65 namespace {
66 
67 class AMDGPUAsmParser;
68 
69 enum RegisterKind { IS_UNKNOWN, IS_VGPR, IS_SGPR, IS_TTMP, IS_SPECIAL };
70 
71 //===----------------------------------------------------------------------===//
72 // Operand
73 //===----------------------------------------------------------------------===//
74 
75 class AMDGPUOperand : public MCParsedAsmOperand {
76   enum KindTy {
77     Token,
78     Immediate,
79     Register,
80     Expression
81   } Kind;
82 
83   SMLoc StartLoc, EndLoc;
84   const AMDGPUAsmParser *AsmParser;
85 
86 public:
87   AMDGPUOperand(KindTy Kind_, const AMDGPUAsmParser *AsmParser_)
88     : MCParsedAsmOperand(), Kind(Kind_), AsmParser(AsmParser_) {}
89 
90   using Ptr = std::unique_ptr<AMDGPUOperand>;
91 
92   struct Modifiers {
93     bool Abs = false;
94     bool Neg = false;
95     bool Sext = false;
96 
97     bool hasFPModifiers() const { return Abs || Neg; }
98     bool hasIntModifiers() const { return Sext; }
99     bool hasModifiers() const { return hasFPModifiers() || hasIntModifiers(); }
100 
101     int64_t getFPModifiersOperand() const {
102       int64_t Operand = 0;
103       Operand |= Abs ? SISrcMods::ABS : 0;
104       Operand |= Neg ? SISrcMods::NEG : 0;
105       return Operand;
106     }
107 
108     int64_t getIntModifiersOperand() const {
109       int64_t Operand = 0;
110       Operand |= Sext ? SISrcMods::SEXT : 0;
111       return Operand;
112     }
113 
114     int64_t getModifiersOperand() const {
115       assert(!(hasFPModifiers() && hasIntModifiers())
116            && "fp and int modifiers should not be used simultaneously");
117       if (hasFPModifiers()) {
118         return getFPModifiersOperand();
119       } else if (hasIntModifiers()) {
120         return getIntModifiersOperand();
121       } else {
122         return 0;
123       }
124     }
125 
126     friend raw_ostream &operator <<(raw_ostream &OS, AMDGPUOperand::Modifiers Mods);
127   };
128 
129   enum ImmTy {
130     ImmTyNone,
131     ImmTyGDS,
132     ImmTyLDS,
133     ImmTyOffen,
134     ImmTyIdxen,
135     ImmTyAddr64,
136     ImmTyOffset,
137     ImmTyInstOffset,
138     ImmTyOffset0,
139     ImmTyOffset1,
140     ImmTyGLC,
141     ImmTySLC,
142     ImmTyTFE,
143     ImmTyD16,
144     ImmTyClampSI,
145     ImmTyOModSI,
146     ImmTyDppCtrl,
147     ImmTyDppRowMask,
148     ImmTyDppBankMask,
149     ImmTyDppBoundCtrl,
150     ImmTySdwaDstSel,
151     ImmTySdwaSrc0Sel,
152     ImmTySdwaSrc1Sel,
153     ImmTySdwaDstUnused,
154     ImmTyDMask,
155     ImmTyUNorm,
156     ImmTyDA,
157     ImmTyR128,
158     ImmTyLWE,
159     ImmTyExpTgt,
160     ImmTyExpCompr,
161     ImmTyExpVM,
162     ImmTyDFMT,
163     ImmTyNFMT,
164     ImmTyHwreg,
165     ImmTyOff,
166     ImmTySendMsg,
167     ImmTyInterpSlot,
168     ImmTyInterpAttr,
169     ImmTyAttrChan,
170     ImmTyOpSel,
171     ImmTyOpSelHi,
172     ImmTyNegLo,
173     ImmTyNegHi,
174     ImmTySwizzle,
175     ImmTyHigh
176   };
177 
178   struct TokOp {
179     const char *Data;
180     unsigned Length;
181   };
182 
183   struct ImmOp {
184     int64_t Val;
185     ImmTy Type;
186     bool IsFPImm;
187     Modifiers Mods;
188   };
189 
190   struct RegOp {
191     unsigned RegNo;
192     bool IsForcedVOP3;
193     Modifiers Mods;
194   };
195 
196   union {
197     TokOp Tok;
198     ImmOp Imm;
199     RegOp Reg;
200     const MCExpr *Expr;
201   };
202 
203   bool isToken() const override {
204     if (Kind == Token)
205       return true;
206 
207     if (Kind != Expression || !Expr)
208       return false;
209 
210     // When parsing operands, we can't always tell if something was meant to be
211     // a token, like 'gds', or an expression that references a global variable.
212     // In this case, we assume the string is an expression, and if we need to
213     // interpret is a token, then we treat the symbol name as the token.
214     return isa<MCSymbolRefExpr>(Expr);
215   }
216 
217   bool isImm() const override {
218     return Kind == Immediate;
219   }
220 
221   bool isInlinableImm(MVT type) const;
222   bool isLiteralImm(MVT type) const;
223 
224   bool isRegKind() const {
225     return Kind == Register;
226   }
227 
228   bool isReg() const override {
229     return isRegKind() && !hasModifiers();
230   }
231 
232   bool isRegOrImmWithInputMods(MVT type) const {
233     return isRegKind() || isInlinableImm(type);
234   }
235 
236   bool isRegOrImmWithInt16InputMods() const {
237     return isRegOrImmWithInputMods(MVT::i16);
238   }
239 
240   bool isRegOrImmWithInt32InputMods() const {
241     return isRegOrImmWithInputMods(MVT::i32);
242   }
243 
244   bool isRegOrImmWithInt64InputMods() const {
245     return isRegOrImmWithInputMods(MVT::i64);
246   }
247 
248   bool isRegOrImmWithFP16InputMods() const {
249     return isRegOrImmWithInputMods(MVT::f16);
250   }
251 
252   bool isRegOrImmWithFP32InputMods() const {
253     return isRegOrImmWithInputMods(MVT::f32);
254   }
255 
256   bool isRegOrImmWithFP64InputMods() const {
257     return isRegOrImmWithInputMods(MVT::f64);
258   }
259 
260   bool isVReg() const {
261     return isRegClass(AMDGPU::VGPR_32RegClassID) ||
262            isRegClass(AMDGPU::VReg_64RegClassID) ||
263            isRegClass(AMDGPU::VReg_96RegClassID) ||
264            isRegClass(AMDGPU::VReg_128RegClassID) ||
265            isRegClass(AMDGPU::VReg_256RegClassID) ||
266            isRegClass(AMDGPU::VReg_512RegClassID);
267   }
268 
269   bool isVReg32OrOff() const {
270     return isOff() || isRegClass(AMDGPU::VGPR_32RegClassID);
271   }
272 
273   bool isSDWAOperand(MVT type) const;
274   bool isSDWAFP16Operand() const;
275   bool isSDWAFP32Operand() const;
276   bool isSDWAInt16Operand() const;
277   bool isSDWAInt32Operand() const;
278 
279   bool isImmTy(ImmTy ImmT) const {
280     return isImm() && Imm.Type == ImmT;
281   }
282 
283   bool isImmModifier() const {
284     return isImm() && Imm.Type != ImmTyNone;
285   }
286 
287   bool isClampSI() const { return isImmTy(ImmTyClampSI); }
288   bool isOModSI() const { return isImmTy(ImmTyOModSI); }
289   bool isDMask() const { return isImmTy(ImmTyDMask); }
290   bool isUNorm() const { return isImmTy(ImmTyUNorm); }
291   bool isDA() const { return isImmTy(ImmTyDA); }
292   bool isR128() const { return isImmTy(ImmTyR128); }
293   bool isLWE() const { return isImmTy(ImmTyLWE); }
294   bool isOff() const { return isImmTy(ImmTyOff); }
295   bool isExpTgt() const { return isImmTy(ImmTyExpTgt); }
296   bool isExpVM() const { return isImmTy(ImmTyExpVM); }
297   bool isExpCompr() const { return isImmTy(ImmTyExpCompr); }
298   bool isOffen() const { return isImmTy(ImmTyOffen); }
299   bool isIdxen() const { return isImmTy(ImmTyIdxen); }
300   bool isAddr64() const { return isImmTy(ImmTyAddr64); }
301   bool isOffset() const { return isImmTy(ImmTyOffset) && isUInt<16>(getImm()); }
302   bool isOffset0() const { return isImmTy(ImmTyOffset0) && isUInt<16>(getImm()); }
303   bool isOffset1() const { return isImmTy(ImmTyOffset1) && isUInt<8>(getImm()); }
304 
305   bool isOffsetU12() const { return (isImmTy(ImmTyOffset) || isImmTy(ImmTyInstOffset)) && isUInt<12>(getImm()); }
306   bool isOffsetS13() const { return (isImmTy(ImmTyOffset) || isImmTy(ImmTyInstOffset)) && isInt<13>(getImm()); }
307   bool isGDS() const { return isImmTy(ImmTyGDS); }
308   bool isLDS() const { return isImmTy(ImmTyLDS); }
309   bool isGLC() const { return isImmTy(ImmTyGLC); }
310   bool isSLC() const { return isImmTy(ImmTySLC); }
311   bool isTFE() const { return isImmTy(ImmTyTFE); }
312   bool isD16() const { return isImmTy(ImmTyD16); }
313   bool isDFMT() const { return isImmTy(ImmTyDFMT) && isUInt<8>(getImm()); }
314   bool isNFMT() const { return isImmTy(ImmTyNFMT) && isUInt<8>(getImm()); }
315   bool isBankMask() const { return isImmTy(ImmTyDppBankMask); }
316   bool isRowMask() const { return isImmTy(ImmTyDppRowMask); }
317   bool isBoundCtrl() const { return isImmTy(ImmTyDppBoundCtrl); }
318   bool isSDWADstSel() const { return isImmTy(ImmTySdwaDstSel); }
319   bool isSDWASrc0Sel() const { return isImmTy(ImmTySdwaSrc0Sel); }
320   bool isSDWASrc1Sel() const { return isImmTy(ImmTySdwaSrc1Sel); }
321   bool isSDWADstUnused() const { return isImmTy(ImmTySdwaDstUnused); }
322   bool isInterpSlot() const { return isImmTy(ImmTyInterpSlot); }
323   bool isInterpAttr() const { return isImmTy(ImmTyInterpAttr); }
324   bool isAttrChan() const { return isImmTy(ImmTyAttrChan); }
325   bool isOpSel() const { return isImmTy(ImmTyOpSel); }
326   bool isOpSelHi() const { return isImmTy(ImmTyOpSelHi); }
327   bool isNegLo() const { return isImmTy(ImmTyNegLo); }
328   bool isNegHi() const { return isImmTy(ImmTyNegHi); }
329   bool isHigh() const { return isImmTy(ImmTyHigh); }
330 
331   bool isMod() const {
332     return isClampSI() || isOModSI();
333   }
334 
335   bool isRegOrImm() const {
336     return isReg() || isImm();
337   }
338 
339   bool isRegClass(unsigned RCID) const;
340 
341   bool isRegOrInlineNoMods(unsigned RCID, MVT type) const {
342     return (isRegClass(RCID) || isInlinableImm(type)) && !hasModifiers();
343   }
344 
345   bool isSCSrcB16() const {
346     return isRegOrInlineNoMods(AMDGPU::SReg_32RegClassID, MVT::i16);
347   }
348 
349   bool isSCSrcV2B16() const {
350     return isSCSrcB16();
351   }
352 
353   bool isSCSrcB32() const {
354     return isRegOrInlineNoMods(AMDGPU::SReg_32RegClassID, MVT::i32);
355   }
356 
357   bool isSCSrcB64() const {
358     return isRegOrInlineNoMods(AMDGPU::SReg_64RegClassID, MVT::i64);
359   }
360 
361   bool isSCSrcF16() const {
362     return isRegOrInlineNoMods(AMDGPU::SReg_32RegClassID, MVT::f16);
363   }
364 
365   bool isSCSrcV2F16() const {
366     return isSCSrcF16();
367   }
368 
369   bool isSCSrcF32() const {
370     return isRegOrInlineNoMods(AMDGPU::SReg_32RegClassID, MVT::f32);
371   }
372 
373   bool isSCSrcF64() const {
374     return isRegOrInlineNoMods(AMDGPU::SReg_64RegClassID, MVT::f64);
375   }
376 
377   bool isSSrcB32() const {
378     return isSCSrcB32() || isLiteralImm(MVT::i32) || isExpr();
379   }
380 
381   bool isSSrcB16() const {
382     return isSCSrcB16() || isLiteralImm(MVT::i16);
383   }
384 
385   bool isSSrcV2B16() const {
386     llvm_unreachable("cannot happen");
387     return isSSrcB16();
388   }
389 
390   bool isSSrcB64() const {
391     // TODO: Find out how SALU supports extension of 32-bit literals to 64 bits.
392     // See isVSrc64().
393     return isSCSrcB64() || isLiteralImm(MVT::i64);
394   }
395 
396   bool isSSrcF32() const {
397     return isSCSrcB32() || isLiteralImm(MVT::f32) || isExpr();
398   }
399 
400   bool isSSrcF64() const {
401     return isSCSrcB64() || isLiteralImm(MVT::f64);
402   }
403 
404   bool isSSrcF16() const {
405     return isSCSrcB16() || isLiteralImm(MVT::f16);
406   }
407 
408   bool isSSrcV2F16() const {
409     llvm_unreachable("cannot happen");
410     return isSSrcF16();
411   }
412 
413   bool isVCSrcB32() const {
414     return isRegOrInlineNoMods(AMDGPU::VS_32RegClassID, MVT::i32);
415   }
416 
417   bool isVCSrcB64() const {
418     return isRegOrInlineNoMods(AMDGPU::VS_64RegClassID, MVT::i64);
419   }
420 
421   bool isVCSrcB16() const {
422     return isRegOrInlineNoMods(AMDGPU::VS_32RegClassID, MVT::i16);
423   }
424 
425   bool isVCSrcV2B16() const {
426     return isVCSrcB16();
427   }
428 
429   bool isVCSrcF32() const {
430     return isRegOrInlineNoMods(AMDGPU::VS_32RegClassID, MVT::f32);
431   }
432 
433   bool isVCSrcF64() const {
434     return isRegOrInlineNoMods(AMDGPU::VS_64RegClassID, MVT::f64);
435   }
436 
437   bool isVCSrcF16() const {
438     return isRegOrInlineNoMods(AMDGPU::VS_32RegClassID, MVT::f16);
439   }
440 
441   bool isVCSrcV2F16() const {
442     return isVCSrcF16();
443   }
444 
445   bool isVSrcB32() const {
446     return isVCSrcF32() || isLiteralImm(MVT::i32);
447   }
448 
449   bool isVSrcB64() const {
450     return isVCSrcF64() || isLiteralImm(MVT::i64);
451   }
452 
453   bool isVSrcB16() const {
454     return isVCSrcF16() || isLiteralImm(MVT::i16);
455   }
456 
457   bool isVSrcV2B16() const {
458     llvm_unreachable("cannot happen");
459     return isVSrcB16();
460   }
461 
462   bool isVSrcF32() const {
463     return isVCSrcF32() || isLiteralImm(MVT::f32);
464   }
465 
466   bool isVSrcF64() const {
467     return isVCSrcF64() || isLiteralImm(MVT::f64);
468   }
469 
470   bool isVSrcF16() const {
471     return isVCSrcF16() || isLiteralImm(MVT::f16);
472   }
473 
474   bool isVSrcV2F16() const {
475     llvm_unreachable("cannot happen");
476     return isVSrcF16();
477   }
478 
479   bool isKImmFP32() const {
480     return isLiteralImm(MVT::f32);
481   }
482 
483   bool isKImmFP16() const {
484     return isLiteralImm(MVT::f16);
485   }
486 
487   bool isMem() const override {
488     return false;
489   }
490 
491   bool isExpr() const {
492     return Kind == Expression;
493   }
494 
495   bool isSoppBrTarget() const {
496     return isExpr() || isImm();
497   }
498 
499   bool isSWaitCnt() const;
500   bool isHwreg() const;
501   bool isSendMsg() const;
502   bool isSwizzle() const;
503   bool isSMRDOffset8() const;
504   bool isSMRDOffset20() const;
505   bool isSMRDLiteralOffset() const;
506   bool isDPPCtrl() const;
507   bool isGPRIdxMode() const;
508   bool isS16Imm() const;
509   bool isU16Imm() const;
510 
511   StringRef getExpressionAsToken() const {
512     assert(isExpr());
513     const MCSymbolRefExpr *S = cast<MCSymbolRefExpr>(Expr);
514     return S->getSymbol().getName();
515   }
516 
517   StringRef getToken() const {
518     assert(isToken());
519 
520     if (Kind == Expression)
521       return getExpressionAsToken();
522 
523     return StringRef(Tok.Data, Tok.Length);
524   }
525 
526   int64_t getImm() const {
527     assert(isImm());
528     return Imm.Val;
529   }
530 
531   ImmTy getImmTy() const {
532     assert(isImm());
533     return Imm.Type;
534   }
535 
536   unsigned getReg() const override {
537     return Reg.RegNo;
538   }
539 
540   SMLoc getStartLoc() const override {
541     return StartLoc;
542   }
543 
544   SMLoc getEndLoc() const override {
545     return EndLoc;
546   }
547 
548   SMRange getLocRange() const {
549     return SMRange(StartLoc, EndLoc);
550   }
551 
552   Modifiers getModifiers() const {
553     assert(isRegKind() || isImmTy(ImmTyNone));
554     return isRegKind() ? Reg.Mods : Imm.Mods;
555   }
556 
557   void setModifiers(Modifiers Mods) {
558     assert(isRegKind() || isImmTy(ImmTyNone));
559     if (isRegKind())
560       Reg.Mods = Mods;
561     else
562       Imm.Mods = Mods;
563   }
564 
565   bool hasModifiers() const {
566     return getModifiers().hasModifiers();
567   }
568 
569   bool hasFPModifiers() const {
570     return getModifiers().hasFPModifiers();
571   }
572 
573   bool hasIntModifiers() const {
574     return getModifiers().hasIntModifiers();
575   }
576 
577   uint64_t applyInputFPModifiers(uint64_t Val, unsigned Size) const;
578 
579   void addImmOperands(MCInst &Inst, unsigned N, bool ApplyModifiers = true) const;
580 
581   void addLiteralImmOperand(MCInst &Inst, int64_t Val, bool ApplyModifiers) const;
582 
583   template <unsigned Bitwidth>
584   void addKImmFPOperands(MCInst &Inst, unsigned N) const;
585 
586   void addKImmFP16Operands(MCInst &Inst, unsigned N) const {
587     addKImmFPOperands<16>(Inst, N);
588   }
589 
590   void addKImmFP32Operands(MCInst &Inst, unsigned N) const {
591     addKImmFPOperands<32>(Inst, N);
592   }
593 
594   void addRegOperands(MCInst &Inst, unsigned N) const;
595 
596   void addRegOrImmOperands(MCInst &Inst, unsigned N) const {
597     if (isRegKind())
598       addRegOperands(Inst, N);
599     else if (isExpr())
600       Inst.addOperand(MCOperand::createExpr(Expr));
601     else
602       addImmOperands(Inst, N);
603   }
604 
605   void addRegOrImmWithInputModsOperands(MCInst &Inst, unsigned N) const {
606     Modifiers Mods = getModifiers();
607     Inst.addOperand(MCOperand::createImm(Mods.getModifiersOperand()));
608     if (isRegKind()) {
609       addRegOperands(Inst, N);
610     } else {
611       addImmOperands(Inst, N, false);
612     }
613   }
614 
615   void addRegOrImmWithFPInputModsOperands(MCInst &Inst, unsigned N) const {
616     assert(!hasIntModifiers());
617     addRegOrImmWithInputModsOperands(Inst, N);
618   }
619 
620   void addRegOrImmWithIntInputModsOperands(MCInst &Inst, unsigned N) const {
621     assert(!hasFPModifiers());
622     addRegOrImmWithInputModsOperands(Inst, N);
623   }
624 
625   void addRegWithInputModsOperands(MCInst &Inst, unsigned N) const {
626     Modifiers Mods = getModifiers();
627     Inst.addOperand(MCOperand::createImm(Mods.getModifiersOperand()));
628     assert(isRegKind());
629     addRegOperands(Inst, N);
630   }
631 
632   void addRegWithFPInputModsOperands(MCInst &Inst, unsigned N) const {
633     assert(!hasIntModifiers());
634     addRegWithInputModsOperands(Inst, N);
635   }
636 
637   void addRegWithIntInputModsOperands(MCInst &Inst, unsigned N) const {
638     assert(!hasFPModifiers());
639     addRegWithInputModsOperands(Inst, N);
640   }
641 
642   void addSoppBrTargetOperands(MCInst &Inst, unsigned N) const {
643     if (isImm())
644       addImmOperands(Inst, N);
645     else {
646       assert(isExpr());
647       Inst.addOperand(MCOperand::createExpr(Expr));
648     }
649   }
650 
651   static void printImmTy(raw_ostream& OS, ImmTy Type) {
652     switch (Type) {
653     case ImmTyNone: OS << "None"; break;
654     case ImmTyGDS: OS << "GDS"; break;
655     case ImmTyLDS: OS << "LDS"; break;
656     case ImmTyOffen: OS << "Offen"; break;
657     case ImmTyIdxen: OS << "Idxen"; break;
658     case ImmTyAddr64: OS << "Addr64"; break;
659     case ImmTyOffset: OS << "Offset"; break;
660     case ImmTyInstOffset: OS << "InstOffset"; break;
661     case ImmTyOffset0: OS << "Offset0"; break;
662     case ImmTyOffset1: OS << "Offset1"; break;
663     case ImmTyGLC: OS << "GLC"; break;
664     case ImmTySLC: OS << "SLC"; break;
665     case ImmTyTFE: OS << "TFE"; break;
666     case ImmTyD16: OS << "D16"; break;
667     case ImmTyDFMT: OS << "DFMT"; break;
668     case ImmTyNFMT: OS << "NFMT"; break;
669     case ImmTyClampSI: OS << "ClampSI"; break;
670     case ImmTyOModSI: OS << "OModSI"; break;
671     case ImmTyDppCtrl: OS << "DppCtrl"; break;
672     case ImmTyDppRowMask: OS << "DppRowMask"; break;
673     case ImmTyDppBankMask: OS << "DppBankMask"; break;
674     case ImmTyDppBoundCtrl: OS << "DppBoundCtrl"; break;
675     case ImmTySdwaDstSel: OS << "SdwaDstSel"; break;
676     case ImmTySdwaSrc0Sel: OS << "SdwaSrc0Sel"; break;
677     case ImmTySdwaSrc1Sel: OS << "SdwaSrc1Sel"; break;
678     case ImmTySdwaDstUnused: OS << "SdwaDstUnused"; break;
679     case ImmTyDMask: OS << "DMask"; break;
680     case ImmTyUNorm: OS << "UNorm"; break;
681     case ImmTyDA: OS << "DA"; break;
682     case ImmTyR128: OS << "R128"; break;
683     case ImmTyLWE: OS << "LWE"; break;
684     case ImmTyOff: OS << "Off"; break;
685     case ImmTyExpTgt: OS << "ExpTgt"; break;
686     case ImmTyExpCompr: OS << "ExpCompr"; break;
687     case ImmTyExpVM: OS << "ExpVM"; break;
688     case ImmTyHwreg: OS << "Hwreg"; break;
689     case ImmTySendMsg: OS << "SendMsg"; break;
690     case ImmTyInterpSlot: OS << "InterpSlot"; break;
691     case ImmTyInterpAttr: OS << "InterpAttr"; break;
692     case ImmTyAttrChan: OS << "AttrChan"; break;
693     case ImmTyOpSel: OS << "OpSel"; break;
694     case ImmTyOpSelHi: OS << "OpSelHi"; break;
695     case ImmTyNegLo: OS << "NegLo"; break;
696     case ImmTyNegHi: OS << "NegHi"; break;
697     case ImmTySwizzle: OS << "Swizzle"; break;
698     case ImmTyHigh: OS << "High"; break;
699     }
700   }
701 
702   void print(raw_ostream &OS) const override {
703     switch (Kind) {
704     case Register:
705       OS << "<register " << getReg() << " mods: " << Reg.Mods << '>';
706       break;
707     case Immediate:
708       OS << '<' << getImm();
709       if (getImmTy() != ImmTyNone) {
710         OS << " type: "; printImmTy(OS, getImmTy());
711       }
712       OS << " mods: " << Imm.Mods << '>';
713       break;
714     case Token:
715       OS << '\'' << getToken() << '\'';
716       break;
717     case Expression:
718       OS << "<expr " << *Expr << '>';
719       break;
720     }
721   }
722 
723   static AMDGPUOperand::Ptr CreateImm(const AMDGPUAsmParser *AsmParser,
724                                       int64_t Val, SMLoc Loc,
725                                       ImmTy Type = ImmTyNone,
726                                       bool IsFPImm = false) {
727     auto Op = llvm::make_unique<AMDGPUOperand>(Immediate, AsmParser);
728     Op->Imm.Val = Val;
729     Op->Imm.IsFPImm = IsFPImm;
730     Op->Imm.Type = Type;
731     Op->Imm.Mods = Modifiers();
732     Op->StartLoc = Loc;
733     Op->EndLoc = Loc;
734     return Op;
735   }
736 
737   static AMDGPUOperand::Ptr CreateToken(const AMDGPUAsmParser *AsmParser,
738                                         StringRef Str, SMLoc Loc,
739                                         bool HasExplicitEncodingSize = true) {
740     auto Res = llvm::make_unique<AMDGPUOperand>(Token, AsmParser);
741     Res->Tok.Data = Str.data();
742     Res->Tok.Length = Str.size();
743     Res->StartLoc = Loc;
744     Res->EndLoc = Loc;
745     return Res;
746   }
747 
748   static AMDGPUOperand::Ptr CreateReg(const AMDGPUAsmParser *AsmParser,
749                                       unsigned RegNo, SMLoc S,
750                                       SMLoc E,
751                                       bool ForceVOP3) {
752     auto Op = llvm::make_unique<AMDGPUOperand>(Register, AsmParser);
753     Op->Reg.RegNo = RegNo;
754     Op->Reg.Mods = Modifiers();
755     Op->Reg.IsForcedVOP3 = ForceVOP3;
756     Op->StartLoc = S;
757     Op->EndLoc = E;
758     return Op;
759   }
760 
761   static AMDGPUOperand::Ptr CreateExpr(const AMDGPUAsmParser *AsmParser,
762                                        const class MCExpr *Expr, SMLoc S) {
763     auto Op = llvm::make_unique<AMDGPUOperand>(Expression, AsmParser);
764     Op->Expr = Expr;
765     Op->StartLoc = S;
766     Op->EndLoc = S;
767     return Op;
768   }
769 };
770 
771 raw_ostream &operator <<(raw_ostream &OS, AMDGPUOperand::Modifiers Mods) {
772   OS << "abs:" << Mods.Abs << " neg: " << Mods.Neg << " sext:" << Mods.Sext;
773   return OS;
774 }
775 
776 //===----------------------------------------------------------------------===//
777 // AsmParser
778 //===----------------------------------------------------------------------===//
779 
780 // Holds info related to the current kernel, e.g. count of SGPRs used.
781 // Kernel scope begins at .amdgpu_hsa_kernel directive, ends at next
782 // .amdgpu_hsa_kernel or at EOF.
783 class KernelScopeInfo {
784   int SgprIndexUnusedMin = -1;
785   int VgprIndexUnusedMin = -1;
786   MCContext *Ctx = nullptr;
787 
788   void usesSgprAt(int i) {
789     if (i >= SgprIndexUnusedMin) {
790       SgprIndexUnusedMin = ++i;
791       if (Ctx) {
792         MCSymbol * const Sym = Ctx->getOrCreateSymbol(Twine(".kernel.sgpr_count"));
793         Sym->setVariableValue(MCConstantExpr::create(SgprIndexUnusedMin, *Ctx));
794       }
795     }
796   }
797 
798   void usesVgprAt(int i) {
799     if (i >= VgprIndexUnusedMin) {
800       VgprIndexUnusedMin = ++i;
801       if (Ctx) {
802         MCSymbol * const Sym = Ctx->getOrCreateSymbol(Twine(".kernel.vgpr_count"));
803         Sym->setVariableValue(MCConstantExpr::create(VgprIndexUnusedMin, *Ctx));
804       }
805     }
806   }
807 
808 public:
809   KernelScopeInfo() = default;
810 
811   void initialize(MCContext &Context) {
812     Ctx = &Context;
813     usesSgprAt(SgprIndexUnusedMin = -1);
814     usesVgprAt(VgprIndexUnusedMin = -1);
815   }
816 
817   void usesRegister(RegisterKind RegKind, unsigned DwordRegIndex, unsigned RegWidth) {
818     switch (RegKind) {
819       case IS_SGPR: usesSgprAt(DwordRegIndex + RegWidth - 1); break;
820       case IS_VGPR: usesVgprAt(DwordRegIndex + RegWidth - 1); break;
821       default: break;
822     }
823   }
824 };
825 
826 class AMDGPUAsmParser : public MCTargetAsmParser {
827   MCAsmParser &Parser;
828 
829   // Number of extra operands parsed after the first optional operand.
830   // This may be necessary to skip hardcoded mandatory operands.
831   static const unsigned MAX_OPR_LOOKAHEAD = 8;
832 
833   unsigned ForcedEncodingSize = 0;
834   bool ForcedDPP = false;
835   bool ForcedSDWA = false;
836   KernelScopeInfo KernelScope;
837 
838   /// @name Auto-generated Match Functions
839   /// {
840 
841 #define GET_ASSEMBLER_HEADER
842 #include "AMDGPUGenAsmMatcher.inc"
843 
844   /// }
845 
846 private:
847   bool ParseAsAbsoluteExpression(uint32_t &Ret);
848   bool ParseDirectiveMajorMinor(uint32_t &Major, uint32_t &Minor);
849   bool ParseDirectiveHSACodeObjectVersion();
850   bool ParseDirectiveHSACodeObjectISA();
851   bool ParseAMDKernelCodeTValue(StringRef ID, amd_kernel_code_t &Header);
852   bool ParseDirectiveAMDKernelCodeT();
853   bool subtargetHasRegister(const MCRegisterInfo &MRI, unsigned RegNo) const;
854   bool ParseDirectiveAMDGPUHsaKernel();
855 
856   bool ParseDirectiveISAVersion();
857   bool ParseDirectiveHSAMetadata();
858   bool ParseDirectivePALMetadata();
859 
860   bool AddNextRegisterToList(unsigned& Reg, unsigned& RegWidth,
861                              RegisterKind RegKind, unsigned Reg1,
862                              unsigned RegNum);
863   bool ParseAMDGPURegister(RegisterKind& RegKind, unsigned& Reg,
864                            unsigned& RegNum, unsigned& RegWidth,
865                            unsigned *DwordRegIndex);
866   void cvtMubufImpl(MCInst &Inst, const OperandVector &Operands,
867                     bool IsAtomic, bool IsAtomicReturn, bool IsLds = false);
868   void cvtDSImpl(MCInst &Inst, const OperandVector &Operands,
869                  bool IsGdsHardcoded);
870 
871 public:
872   enum AMDGPUMatchResultTy {
873     Match_PreferE32 = FIRST_TARGET_MATCH_RESULT_TY
874   };
875 
876   using OptionalImmIndexMap = std::map<AMDGPUOperand::ImmTy, unsigned>;
877 
878   AMDGPUAsmParser(const MCSubtargetInfo &STI, MCAsmParser &_Parser,
879                const MCInstrInfo &MII,
880                const MCTargetOptions &Options)
881       : MCTargetAsmParser(Options, STI, MII), Parser(_Parser) {
882     MCAsmParserExtension::Initialize(Parser);
883 
884     if (getFeatureBits().none()) {
885       // Set default features.
886       copySTI().ToggleFeature("SOUTHERN_ISLANDS");
887     }
888 
889     setAvailableFeatures(ComputeAvailableFeatures(getFeatureBits()));
890 
891     {
892       // TODO: make those pre-defined variables read-only.
893       // Currently there is none suitable machinery in the core llvm-mc for this.
894       // MCSymbol::isRedefinable is intended for another purpose, and
895       // AsmParser::parseDirectiveSet() cannot be specialized for specific target.
896       AMDGPU::IsaInfo::IsaVersion ISA =
897           AMDGPU::IsaInfo::getIsaVersion(getFeatureBits());
898       MCContext &Ctx = getContext();
899       MCSymbol *Sym =
900           Ctx.getOrCreateSymbol(Twine(".option.machine_version_major"));
901       Sym->setVariableValue(MCConstantExpr::create(ISA.Major, Ctx));
902       Sym = Ctx.getOrCreateSymbol(Twine(".option.machine_version_minor"));
903       Sym->setVariableValue(MCConstantExpr::create(ISA.Minor, Ctx));
904       Sym = Ctx.getOrCreateSymbol(Twine(".option.machine_version_stepping"));
905       Sym->setVariableValue(MCConstantExpr::create(ISA.Stepping, Ctx));
906     }
907     KernelScope.initialize(getContext());
908   }
909 
910   bool hasXNACK() const {
911     return AMDGPU::hasXNACK(getSTI());
912   }
913 
914   bool hasMIMG_R128() const {
915     return AMDGPU::hasMIMG_R128(getSTI());
916   }
917 
918   bool hasPackedD16() const {
919     return AMDGPU::hasPackedD16(getSTI());
920   }
921 
922   bool isSI() const {
923     return AMDGPU::isSI(getSTI());
924   }
925 
926   bool isCI() const {
927     return AMDGPU::isCI(getSTI());
928   }
929 
930   bool isVI() const {
931     return AMDGPU::isVI(getSTI());
932   }
933 
934   bool isGFX9() const {
935     return AMDGPU::isGFX9(getSTI());
936   }
937 
938   bool hasInv2PiInlineImm() const {
939     return getFeatureBits()[AMDGPU::FeatureInv2PiInlineImm];
940   }
941 
942   bool hasFlatOffsets() const {
943     return getFeatureBits()[AMDGPU::FeatureFlatInstOffsets];
944   }
945 
946   bool hasSGPR102_SGPR103() const {
947     return !isVI();
948   }
949 
950   bool hasIntClamp() const {
951     return getFeatureBits()[AMDGPU::FeatureIntClamp];
952   }
953 
954   AMDGPUTargetStreamer &getTargetStreamer() {
955     MCTargetStreamer &TS = *getParser().getStreamer().getTargetStreamer();
956     return static_cast<AMDGPUTargetStreamer &>(TS);
957   }
958 
959   const MCRegisterInfo *getMRI() const {
960     // We need this const_cast because for some reason getContext() is not const
961     // in MCAsmParser.
962     return const_cast<AMDGPUAsmParser*>(this)->getContext().getRegisterInfo();
963   }
964 
965   const MCInstrInfo *getMII() const {
966     return &MII;
967   }
968 
969   const FeatureBitset &getFeatureBits() const {
970     return getSTI().getFeatureBits();
971   }
972 
973   void setForcedEncodingSize(unsigned Size) { ForcedEncodingSize = Size; }
974   void setForcedDPP(bool ForceDPP_) { ForcedDPP = ForceDPP_; }
975   void setForcedSDWA(bool ForceSDWA_) { ForcedSDWA = ForceSDWA_; }
976 
977   unsigned getForcedEncodingSize() const { return ForcedEncodingSize; }
978   bool isForcedVOP3() const { return ForcedEncodingSize == 64; }
979   bool isForcedDPP() const { return ForcedDPP; }
980   bool isForcedSDWA() const { return ForcedSDWA; }
981   ArrayRef<unsigned> getMatchedVariants() const;
982 
983   std::unique_ptr<AMDGPUOperand> parseRegister();
984   bool ParseRegister(unsigned &RegNo, SMLoc &StartLoc, SMLoc &EndLoc) override;
985   unsigned checkTargetMatchPredicate(MCInst &Inst) override;
986   unsigned validateTargetOperandClass(MCParsedAsmOperand &Op,
987                                       unsigned Kind) override;
988   bool MatchAndEmitInstruction(SMLoc IDLoc, unsigned &Opcode,
989                                OperandVector &Operands, MCStreamer &Out,
990                                uint64_t &ErrorInfo,
991                                bool MatchingInlineAsm) override;
992   bool ParseDirective(AsmToken DirectiveID) override;
993   OperandMatchResultTy parseOperand(OperandVector &Operands, StringRef Mnemonic);
994   StringRef parseMnemonicSuffix(StringRef Name);
995   bool ParseInstruction(ParseInstructionInfo &Info, StringRef Name,
996                         SMLoc NameLoc, OperandVector &Operands) override;
997   //bool ProcessInstruction(MCInst &Inst);
998 
999   OperandMatchResultTy parseIntWithPrefix(const char *Prefix, int64_t &Int);
1000 
1001   OperandMatchResultTy
1002   parseIntWithPrefix(const char *Prefix, OperandVector &Operands,
1003                      AMDGPUOperand::ImmTy ImmTy = AMDGPUOperand::ImmTyNone,
1004                      bool (*ConvertResult)(int64_t &) = nullptr);
1005 
1006   OperandMatchResultTy parseOperandArrayWithPrefix(
1007     const char *Prefix,
1008     OperandVector &Operands,
1009     AMDGPUOperand::ImmTy ImmTy = AMDGPUOperand::ImmTyNone,
1010     bool (*ConvertResult)(int64_t&) = nullptr);
1011 
1012   OperandMatchResultTy
1013   parseNamedBit(const char *Name, OperandVector &Operands,
1014                 AMDGPUOperand::ImmTy ImmTy = AMDGPUOperand::ImmTyNone);
1015   OperandMatchResultTy parseStringWithPrefix(StringRef Prefix,
1016                                              StringRef &Value);
1017 
1018   bool parseAbsoluteExpr(int64_t &Val, bool AbsMod = false);
1019   OperandMatchResultTy parseImm(OperandVector &Operands, bool AbsMod = false);
1020   OperandMatchResultTy parseReg(OperandVector &Operands);
1021   OperandMatchResultTy parseRegOrImm(OperandVector &Operands, bool AbsMod = false);
1022   OperandMatchResultTy parseRegOrImmWithFPInputMods(OperandVector &Operands, bool AllowImm = true);
1023   OperandMatchResultTy parseRegOrImmWithIntInputMods(OperandVector &Operands, bool AllowImm = true);
1024   OperandMatchResultTy parseRegWithFPInputMods(OperandVector &Operands);
1025   OperandMatchResultTy parseRegWithIntInputMods(OperandVector &Operands);
1026   OperandMatchResultTy parseVReg32OrOff(OperandVector &Operands);
1027 
1028   void cvtDSOffset01(MCInst &Inst, const OperandVector &Operands);
1029   void cvtDS(MCInst &Inst, const OperandVector &Operands) { cvtDSImpl(Inst, Operands, false); }
1030   void cvtDSGds(MCInst &Inst, const OperandVector &Operands) { cvtDSImpl(Inst, Operands, true); }
1031   void cvtExp(MCInst &Inst, const OperandVector &Operands);
1032 
1033   bool parseCnt(int64_t &IntVal);
1034   OperandMatchResultTy parseSWaitCntOps(OperandVector &Operands);
1035   OperandMatchResultTy parseHwreg(OperandVector &Operands);
1036 
1037 private:
1038   struct OperandInfoTy {
1039     int64_t Id;
1040     bool IsSymbolic = false;
1041 
1042     OperandInfoTy(int64_t Id_) : Id(Id_) {}
1043   };
1044 
1045   bool parseSendMsgConstruct(OperandInfoTy &Msg, OperandInfoTy &Operation, int64_t &StreamId);
1046   bool parseHwregConstruct(OperandInfoTy &HwReg, int64_t &Offset, int64_t &Width);
1047 
1048   void errorExpTgt();
1049   OperandMatchResultTy parseExpTgtImpl(StringRef Str, uint8_t &Val);
1050 
1051   bool validateInstruction(const MCInst &Inst, const SMLoc &IDLoc);
1052   bool validateConstantBusLimitations(const MCInst &Inst);
1053   bool validateEarlyClobberLimitations(const MCInst &Inst);
1054   bool validateIntClampSupported(const MCInst &Inst);
1055   bool validateMIMGAtomicDMask(const MCInst &Inst);
1056   bool validateMIMGGatherDMask(const MCInst &Inst);
1057   bool validateMIMGDataSize(const MCInst &Inst);
1058   bool validateMIMGR128(const MCInst &Inst);
1059   bool validateMIMGD16(const MCInst &Inst);
1060   bool usesConstantBus(const MCInst &Inst, unsigned OpIdx);
1061   bool isInlineConstant(const MCInst &Inst, unsigned OpIdx) const;
1062   unsigned findImplicitSGPRReadInVOP(const MCInst &Inst) const;
1063 
1064   bool trySkipId(const StringRef Id);
1065   bool trySkipToken(const AsmToken::TokenKind Kind);
1066   bool skipToken(const AsmToken::TokenKind Kind, const StringRef ErrMsg);
1067   bool parseString(StringRef &Val, const StringRef ErrMsg = "expected a string");
1068   bool parseExpr(int64_t &Imm);
1069 
1070 public:
1071   OperandMatchResultTy parseOptionalOperand(OperandVector &Operands);
1072   OperandMatchResultTy parseOptionalOpr(OperandVector &Operands);
1073 
1074   OperandMatchResultTy parseExpTgt(OperandVector &Operands);
1075   OperandMatchResultTy parseSendMsgOp(OperandVector &Operands);
1076   OperandMatchResultTy parseInterpSlot(OperandVector &Operands);
1077   OperandMatchResultTy parseInterpAttr(OperandVector &Operands);
1078   OperandMatchResultTy parseSOppBrTarget(OperandVector &Operands);
1079 
1080   bool parseSwizzleOperands(const unsigned OpNum, int64_t* Op,
1081                             const unsigned MinVal,
1082                             const unsigned MaxVal,
1083                             const StringRef ErrMsg);
1084   OperandMatchResultTy parseSwizzleOp(OperandVector &Operands);
1085   bool parseSwizzleOffset(int64_t &Imm);
1086   bool parseSwizzleMacro(int64_t &Imm);
1087   bool parseSwizzleQuadPerm(int64_t &Imm);
1088   bool parseSwizzleBitmaskPerm(int64_t &Imm);
1089   bool parseSwizzleBroadcast(int64_t &Imm);
1090   bool parseSwizzleSwap(int64_t &Imm);
1091   bool parseSwizzleReverse(int64_t &Imm);
1092 
1093   void cvtMubuf(MCInst &Inst, const OperandVector &Operands) { cvtMubufImpl(Inst, Operands, false, false); }
1094   void cvtMubufAtomic(MCInst &Inst, const OperandVector &Operands) { cvtMubufImpl(Inst, Operands, true, false); }
1095   void cvtMubufAtomicReturn(MCInst &Inst, const OperandVector &Operands) { cvtMubufImpl(Inst, Operands, true, true); }
1096   void cvtMubufLds(MCInst &Inst, const OperandVector &Operands) { cvtMubufImpl(Inst, Operands, false, false, true); }
1097   void cvtMtbuf(MCInst &Inst, const OperandVector &Operands);
1098 
1099   AMDGPUOperand::Ptr defaultGLC() const;
1100   AMDGPUOperand::Ptr defaultSLC() const;
1101   AMDGPUOperand::Ptr defaultTFE() const;
1102 
1103   AMDGPUOperand::Ptr defaultD16() const;
1104   AMDGPUOperand::Ptr defaultDMask() const;
1105   AMDGPUOperand::Ptr defaultUNorm() const;
1106   AMDGPUOperand::Ptr defaultDA() const;
1107   AMDGPUOperand::Ptr defaultR128() const;
1108   AMDGPUOperand::Ptr defaultLWE() const;
1109   AMDGPUOperand::Ptr defaultSMRDOffset8() const;
1110   AMDGPUOperand::Ptr defaultSMRDOffset20() const;
1111   AMDGPUOperand::Ptr defaultSMRDLiteralOffset() const;
1112   AMDGPUOperand::Ptr defaultOffsetU12() const;
1113   AMDGPUOperand::Ptr defaultOffsetS13() const;
1114 
1115   OperandMatchResultTy parseOModOperand(OperandVector &Operands);
1116 
1117   void cvtVOP3(MCInst &Inst, const OperandVector &Operands,
1118                OptionalImmIndexMap &OptionalIdx);
1119   void cvtVOP3OpSel(MCInst &Inst, const OperandVector &Operands);
1120   void cvtVOP3(MCInst &Inst, const OperandVector &Operands);
1121   void cvtVOP3P(MCInst &Inst, const OperandVector &Operands);
1122 
1123   void cvtVOP3Interp(MCInst &Inst, const OperandVector &Operands);
1124 
1125   void cvtMIMG(MCInst &Inst, const OperandVector &Operands,
1126                bool IsAtomic = false);
1127   void cvtMIMGAtomic(MCInst &Inst, const OperandVector &Operands);
1128 
1129   OperandMatchResultTy parseDPPCtrl(OperandVector &Operands);
1130   AMDGPUOperand::Ptr defaultRowMask() const;
1131   AMDGPUOperand::Ptr defaultBankMask() const;
1132   AMDGPUOperand::Ptr defaultBoundCtrl() const;
1133   void cvtDPP(MCInst &Inst, const OperandVector &Operands);
1134 
1135   OperandMatchResultTy parseSDWASel(OperandVector &Operands, StringRef Prefix,
1136                                     AMDGPUOperand::ImmTy Type);
1137   OperandMatchResultTy parseSDWADstUnused(OperandVector &Operands);
1138   void cvtSdwaVOP1(MCInst &Inst, const OperandVector &Operands);
1139   void cvtSdwaVOP2(MCInst &Inst, const OperandVector &Operands);
1140   void cvtSdwaVOP2b(MCInst &Inst, const OperandVector &Operands);
1141   void cvtSdwaVOPC(MCInst &Inst, const OperandVector &Operands);
1142   void cvtSDWA(MCInst &Inst, const OperandVector &Operands,
1143                 uint64_t BasicInstType, bool skipVcc = false);
1144 };
1145 
1146 struct OptionalOperand {
1147   const char *Name;
1148   AMDGPUOperand::ImmTy Type;
1149   bool IsBit;
1150   bool (*ConvertResult)(int64_t&);
1151 };
1152 
1153 } // end anonymous namespace
1154 
1155 // May be called with integer type with equivalent bitwidth.
1156 static const fltSemantics *getFltSemantics(unsigned Size) {
1157   switch (Size) {
1158   case 4:
1159     return &APFloat::IEEEsingle();
1160   case 8:
1161     return &APFloat::IEEEdouble();
1162   case 2:
1163     return &APFloat::IEEEhalf();
1164   default:
1165     llvm_unreachable("unsupported fp type");
1166   }
1167 }
1168 
1169 static const fltSemantics *getFltSemantics(MVT VT) {
1170   return getFltSemantics(VT.getSizeInBits() / 8);
1171 }
1172 
1173 static const fltSemantics *getOpFltSemantics(uint8_t OperandType) {
1174   switch (OperandType) {
1175   case AMDGPU::OPERAND_REG_IMM_INT32:
1176   case AMDGPU::OPERAND_REG_IMM_FP32:
1177   case AMDGPU::OPERAND_REG_INLINE_C_INT32:
1178   case AMDGPU::OPERAND_REG_INLINE_C_FP32:
1179     return &APFloat::IEEEsingle();
1180   case AMDGPU::OPERAND_REG_IMM_INT64:
1181   case AMDGPU::OPERAND_REG_IMM_FP64:
1182   case AMDGPU::OPERAND_REG_INLINE_C_INT64:
1183   case AMDGPU::OPERAND_REG_INLINE_C_FP64:
1184     return &APFloat::IEEEdouble();
1185   case AMDGPU::OPERAND_REG_IMM_INT16:
1186   case AMDGPU::OPERAND_REG_IMM_FP16:
1187   case AMDGPU::OPERAND_REG_INLINE_C_INT16:
1188   case AMDGPU::OPERAND_REG_INLINE_C_FP16:
1189   case AMDGPU::OPERAND_REG_INLINE_C_V2INT16:
1190   case AMDGPU::OPERAND_REG_INLINE_C_V2FP16:
1191     return &APFloat::IEEEhalf();
1192   default:
1193     llvm_unreachable("unsupported fp type");
1194   }
1195 }
1196 
1197 //===----------------------------------------------------------------------===//
1198 // Operand
1199 //===----------------------------------------------------------------------===//
1200 
1201 static bool canLosslesslyConvertToFPType(APFloat &FPLiteral, MVT VT) {
1202   bool Lost;
1203 
1204   // Convert literal to single precision
1205   APFloat::opStatus Status = FPLiteral.convert(*getFltSemantics(VT),
1206                                                APFloat::rmNearestTiesToEven,
1207                                                &Lost);
1208   // We allow precision lost but not overflow or underflow
1209   if (Status != APFloat::opOK &&
1210       Lost &&
1211       ((Status & APFloat::opOverflow)  != 0 ||
1212        (Status & APFloat::opUnderflow) != 0)) {
1213     return false;
1214   }
1215 
1216   return true;
1217 }
1218 
1219 bool AMDGPUOperand::isInlinableImm(MVT type) const {
1220   if (!isImmTy(ImmTyNone)) {
1221     // Only plain immediates are inlinable (e.g. "clamp" attribute is not)
1222     return false;
1223   }
1224   // TODO: We should avoid using host float here. It would be better to
1225   // check the float bit values which is what a few other places do.
1226   // We've had bot failures before due to weird NaN support on mips hosts.
1227 
1228   APInt Literal(64, Imm.Val);
1229 
1230   if (Imm.IsFPImm) { // We got fp literal token
1231     if (type == MVT::f64 || type == MVT::i64) { // Expected 64-bit operand
1232       return AMDGPU::isInlinableLiteral64(Imm.Val,
1233                                           AsmParser->hasInv2PiInlineImm());
1234     }
1235 
1236     APFloat FPLiteral(APFloat::IEEEdouble(), APInt(64, Imm.Val));
1237     if (!canLosslesslyConvertToFPType(FPLiteral, type))
1238       return false;
1239 
1240     if (type.getScalarSizeInBits() == 16) {
1241       return AMDGPU::isInlinableLiteral16(
1242         static_cast<int16_t>(FPLiteral.bitcastToAPInt().getZExtValue()),
1243         AsmParser->hasInv2PiInlineImm());
1244     }
1245 
1246     // Check if single precision literal is inlinable
1247     return AMDGPU::isInlinableLiteral32(
1248       static_cast<int32_t>(FPLiteral.bitcastToAPInt().getZExtValue()),
1249       AsmParser->hasInv2PiInlineImm());
1250   }
1251 
1252   // We got int literal token.
1253   if (type == MVT::f64 || type == MVT::i64) { // Expected 64-bit operand
1254     return AMDGPU::isInlinableLiteral64(Imm.Val,
1255                                         AsmParser->hasInv2PiInlineImm());
1256   }
1257 
1258   if (type.getScalarSizeInBits() == 16) {
1259     return AMDGPU::isInlinableLiteral16(
1260       static_cast<int16_t>(Literal.getLoBits(16).getSExtValue()),
1261       AsmParser->hasInv2PiInlineImm());
1262   }
1263 
1264   return AMDGPU::isInlinableLiteral32(
1265     static_cast<int32_t>(Literal.getLoBits(32).getZExtValue()),
1266     AsmParser->hasInv2PiInlineImm());
1267 }
1268 
1269 bool AMDGPUOperand::isLiteralImm(MVT type) const {
1270   // Check that this immediate can be added as literal
1271   if (!isImmTy(ImmTyNone)) {
1272     return false;
1273   }
1274 
1275   if (!Imm.IsFPImm) {
1276     // We got int literal token.
1277 
1278     if (type == MVT::f64 && hasFPModifiers()) {
1279       // Cannot apply fp modifiers to int literals preserving the same semantics
1280       // for VOP1/2/C and VOP3 because of integer truncation. To avoid ambiguity,
1281       // disable these cases.
1282       return false;
1283     }
1284 
1285     unsigned Size = type.getSizeInBits();
1286     if (Size == 64)
1287       Size = 32;
1288 
1289     // FIXME: 64-bit operands can zero extend, sign extend, or pad zeroes for FP
1290     // types.
1291     return isUIntN(Size, Imm.Val) || isIntN(Size, Imm.Val);
1292   }
1293 
1294   // We got fp literal token
1295   if (type == MVT::f64) { // Expected 64-bit fp operand
1296     // We would set low 64-bits of literal to zeroes but we accept this literals
1297     return true;
1298   }
1299 
1300   if (type == MVT::i64) { // Expected 64-bit int operand
1301     // We don't allow fp literals in 64-bit integer instructions. It is
1302     // unclear how we should encode them.
1303     return false;
1304   }
1305 
1306   APFloat FPLiteral(APFloat::IEEEdouble(), APInt(64, Imm.Val));
1307   return canLosslesslyConvertToFPType(FPLiteral, type);
1308 }
1309 
1310 bool AMDGPUOperand::isRegClass(unsigned RCID) const {
1311   return isRegKind() && AsmParser->getMRI()->getRegClass(RCID).contains(getReg());
1312 }
1313 
1314 bool AMDGPUOperand::isSDWAOperand(MVT type) const {
1315   if (AsmParser->isVI())
1316     return isVReg();
1317   else if (AsmParser->isGFX9())
1318     return isRegKind() || isInlinableImm(type);
1319   else
1320     return false;
1321 }
1322 
1323 bool AMDGPUOperand::isSDWAFP16Operand() const {
1324   return isSDWAOperand(MVT::f16);
1325 }
1326 
1327 bool AMDGPUOperand::isSDWAFP32Operand() const {
1328   return isSDWAOperand(MVT::f32);
1329 }
1330 
1331 bool AMDGPUOperand::isSDWAInt16Operand() const {
1332   return isSDWAOperand(MVT::i16);
1333 }
1334 
1335 bool AMDGPUOperand::isSDWAInt32Operand() const {
1336   return isSDWAOperand(MVT::i32);
1337 }
1338 
1339 uint64_t AMDGPUOperand::applyInputFPModifiers(uint64_t Val, unsigned Size) const
1340 {
1341   assert(isImmTy(ImmTyNone) && Imm.Mods.hasFPModifiers());
1342   assert(Size == 2 || Size == 4 || Size == 8);
1343 
1344   const uint64_t FpSignMask = (1ULL << (Size * 8 - 1));
1345 
1346   if (Imm.Mods.Abs) {
1347     Val &= ~FpSignMask;
1348   }
1349   if (Imm.Mods.Neg) {
1350     Val ^= FpSignMask;
1351   }
1352 
1353   return Val;
1354 }
1355 
1356 void AMDGPUOperand::addImmOperands(MCInst &Inst, unsigned N, bool ApplyModifiers) const {
1357   if (AMDGPU::isSISrcOperand(AsmParser->getMII()->get(Inst.getOpcode()),
1358                              Inst.getNumOperands())) {
1359     addLiteralImmOperand(Inst, Imm.Val,
1360                          ApplyModifiers &
1361                          isImmTy(ImmTyNone) && Imm.Mods.hasFPModifiers());
1362   } else {
1363     assert(!isImmTy(ImmTyNone) || !hasModifiers());
1364     Inst.addOperand(MCOperand::createImm(Imm.Val));
1365   }
1366 }
1367 
1368 void AMDGPUOperand::addLiteralImmOperand(MCInst &Inst, int64_t Val, bool ApplyModifiers) const {
1369   const auto& InstDesc = AsmParser->getMII()->get(Inst.getOpcode());
1370   auto OpNum = Inst.getNumOperands();
1371   // Check that this operand accepts literals
1372   assert(AMDGPU::isSISrcOperand(InstDesc, OpNum));
1373 
1374   if (ApplyModifiers) {
1375     assert(AMDGPU::isSISrcFPOperand(InstDesc, OpNum));
1376     const unsigned Size = Imm.IsFPImm ? sizeof(double) : getOperandSize(InstDesc, OpNum);
1377     Val = applyInputFPModifiers(Val, Size);
1378   }
1379 
1380   APInt Literal(64, Val);
1381   uint8_t OpTy = InstDesc.OpInfo[OpNum].OperandType;
1382 
1383   if (Imm.IsFPImm) { // We got fp literal token
1384     switch (OpTy) {
1385     case AMDGPU::OPERAND_REG_IMM_INT64:
1386     case AMDGPU::OPERAND_REG_IMM_FP64:
1387     case AMDGPU::OPERAND_REG_INLINE_C_INT64:
1388     case AMDGPU::OPERAND_REG_INLINE_C_FP64:
1389       if (AMDGPU::isInlinableLiteral64(Literal.getZExtValue(),
1390                                        AsmParser->hasInv2PiInlineImm())) {
1391         Inst.addOperand(MCOperand::createImm(Literal.getZExtValue()));
1392         return;
1393       }
1394 
1395       // Non-inlineable
1396       if (AMDGPU::isSISrcFPOperand(InstDesc, OpNum)) { // Expected 64-bit fp operand
1397         // For fp operands we check if low 32 bits are zeros
1398         if (Literal.getLoBits(32) != 0) {
1399           const_cast<AMDGPUAsmParser *>(AsmParser)->Warning(Inst.getLoc(),
1400           "Can't encode literal as exact 64-bit floating-point operand. "
1401           "Low 32-bits will be set to zero");
1402         }
1403 
1404         Inst.addOperand(MCOperand::createImm(Literal.lshr(32).getZExtValue()));
1405         return;
1406       }
1407 
1408       // We don't allow fp literals in 64-bit integer instructions. It is
1409       // unclear how we should encode them. This case should be checked earlier
1410       // in predicate methods (isLiteralImm())
1411       llvm_unreachable("fp literal in 64-bit integer instruction.");
1412 
1413     case AMDGPU::OPERAND_REG_IMM_INT32:
1414     case AMDGPU::OPERAND_REG_IMM_FP32:
1415     case AMDGPU::OPERAND_REG_INLINE_C_INT32:
1416     case AMDGPU::OPERAND_REG_INLINE_C_FP32:
1417     case AMDGPU::OPERAND_REG_IMM_INT16:
1418     case AMDGPU::OPERAND_REG_IMM_FP16:
1419     case AMDGPU::OPERAND_REG_INLINE_C_INT16:
1420     case AMDGPU::OPERAND_REG_INLINE_C_FP16:
1421     case AMDGPU::OPERAND_REG_INLINE_C_V2INT16:
1422     case AMDGPU::OPERAND_REG_INLINE_C_V2FP16: {
1423       bool lost;
1424       APFloat FPLiteral(APFloat::IEEEdouble(), Literal);
1425       // Convert literal to single precision
1426       FPLiteral.convert(*getOpFltSemantics(OpTy),
1427                         APFloat::rmNearestTiesToEven, &lost);
1428       // We allow precision lost but not overflow or underflow. This should be
1429       // checked earlier in isLiteralImm()
1430 
1431       uint64_t ImmVal = FPLiteral.bitcastToAPInt().getZExtValue();
1432       if (OpTy == AMDGPU::OPERAND_REG_INLINE_C_V2INT16 ||
1433           OpTy == AMDGPU::OPERAND_REG_INLINE_C_V2FP16) {
1434         ImmVal |= (ImmVal << 16);
1435       }
1436 
1437       Inst.addOperand(MCOperand::createImm(ImmVal));
1438       return;
1439     }
1440     default:
1441       llvm_unreachable("invalid operand size");
1442     }
1443 
1444     return;
1445   }
1446 
1447    // We got int literal token.
1448   // Only sign extend inline immediates.
1449   // FIXME: No errors on truncation
1450   switch (OpTy) {
1451   case AMDGPU::OPERAND_REG_IMM_INT32:
1452   case AMDGPU::OPERAND_REG_IMM_FP32:
1453   case AMDGPU::OPERAND_REG_INLINE_C_INT32:
1454   case AMDGPU::OPERAND_REG_INLINE_C_FP32:
1455     if (isInt<32>(Val) &&
1456         AMDGPU::isInlinableLiteral32(static_cast<int32_t>(Val),
1457                                      AsmParser->hasInv2PiInlineImm())) {
1458       Inst.addOperand(MCOperand::createImm(Val));
1459       return;
1460     }
1461 
1462     Inst.addOperand(MCOperand::createImm(Val & 0xffffffff));
1463     return;
1464 
1465   case AMDGPU::OPERAND_REG_IMM_INT64:
1466   case AMDGPU::OPERAND_REG_IMM_FP64:
1467   case AMDGPU::OPERAND_REG_INLINE_C_INT64:
1468   case AMDGPU::OPERAND_REG_INLINE_C_FP64:
1469     if (AMDGPU::isInlinableLiteral64(Val, AsmParser->hasInv2PiInlineImm())) {
1470       Inst.addOperand(MCOperand::createImm(Val));
1471       return;
1472     }
1473 
1474     Inst.addOperand(MCOperand::createImm(Lo_32(Val)));
1475     return;
1476 
1477   case AMDGPU::OPERAND_REG_IMM_INT16:
1478   case AMDGPU::OPERAND_REG_IMM_FP16:
1479   case AMDGPU::OPERAND_REG_INLINE_C_INT16:
1480   case AMDGPU::OPERAND_REG_INLINE_C_FP16:
1481     if (isInt<16>(Val) &&
1482         AMDGPU::isInlinableLiteral16(static_cast<int16_t>(Val),
1483                                      AsmParser->hasInv2PiInlineImm())) {
1484       Inst.addOperand(MCOperand::createImm(Val));
1485       return;
1486     }
1487 
1488     Inst.addOperand(MCOperand::createImm(Val & 0xffff));
1489     return;
1490 
1491   case AMDGPU::OPERAND_REG_INLINE_C_V2INT16:
1492   case AMDGPU::OPERAND_REG_INLINE_C_V2FP16: {
1493     auto LiteralVal = static_cast<uint16_t>(Literal.getLoBits(16).getZExtValue());
1494     assert(AMDGPU::isInlinableLiteral16(LiteralVal,
1495                                         AsmParser->hasInv2PiInlineImm()));
1496 
1497     uint32_t ImmVal = static_cast<uint32_t>(LiteralVal) << 16 |
1498                       static_cast<uint32_t>(LiteralVal);
1499     Inst.addOperand(MCOperand::createImm(ImmVal));
1500     return;
1501   }
1502   default:
1503     llvm_unreachable("invalid operand size");
1504   }
1505 }
1506 
1507 template <unsigned Bitwidth>
1508 void AMDGPUOperand::addKImmFPOperands(MCInst &Inst, unsigned N) const {
1509   APInt Literal(64, Imm.Val);
1510 
1511   if (!Imm.IsFPImm) {
1512     // We got int literal token.
1513     Inst.addOperand(MCOperand::createImm(Literal.getLoBits(Bitwidth).getZExtValue()));
1514     return;
1515   }
1516 
1517   bool Lost;
1518   APFloat FPLiteral(APFloat::IEEEdouble(), Literal);
1519   FPLiteral.convert(*getFltSemantics(Bitwidth / 8),
1520                     APFloat::rmNearestTiesToEven, &Lost);
1521   Inst.addOperand(MCOperand::createImm(FPLiteral.bitcastToAPInt().getZExtValue()));
1522 }
1523 
1524 void AMDGPUOperand::addRegOperands(MCInst &Inst, unsigned N) const {
1525   Inst.addOperand(MCOperand::createReg(AMDGPU::getMCReg(getReg(), AsmParser->getSTI())));
1526 }
1527 
1528 //===----------------------------------------------------------------------===//
1529 // AsmParser
1530 //===----------------------------------------------------------------------===//
1531 
1532 static int getRegClass(RegisterKind Is, unsigned RegWidth) {
1533   if (Is == IS_VGPR) {
1534     switch (RegWidth) {
1535       default: return -1;
1536       case 1: return AMDGPU::VGPR_32RegClassID;
1537       case 2: return AMDGPU::VReg_64RegClassID;
1538       case 3: return AMDGPU::VReg_96RegClassID;
1539       case 4: return AMDGPU::VReg_128RegClassID;
1540       case 8: return AMDGPU::VReg_256RegClassID;
1541       case 16: return AMDGPU::VReg_512RegClassID;
1542     }
1543   } else if (Is == IS_TTMP) {
1544     switch (RegWidth) {
1545       default: return -1;
1546       case 1: return AMDGPU::TTMP_32RegClassID;
1547       case 2: return AMDGPU::TTMP_64RegClassID;
1548       case 4: return AMDGPU::TTMP_128RegClassID;
1549       case 8: return AMDGPU::TTMP_256RegClassID;
1550       case 16: return AMDGPU::TTMP_512RegClassID;
1551     }
1552   } else if (Is == IS_SGPR) {
1553     switch (RegWidth) {
1554       default: return -1;
1555       case 1: return AMDGPU::SGPR_32RegClassID;
1556       case 2: return AMDGPU::SGPR_64RegClassID;
1557       case 4: return AMDGPU::SGPR_128RegClassID;
1558       case 8: return AMDGPU::SGPR_256RegClassID;
1559       case 16: return AMDGPU::SGPR_512RegClassID;
1560     }
1561   }
1562   return -1;
1563 }
1564 
1565 static unsigned getSpecialRegForName(StringRef RegName) {
1566   return StringSwitch<unsigned>(RegName)
1567     .Case("exec", AMDGPU::EXEC)
1568     .Case("vcc", AMDGPU::VCC)
1569     .Case("flat_scratch", AMDGPU::FLAT_SCR)
1570     .Case("xnack_mask", AMDGPU::XNACK_MASK)
1571     .Case("m0", AMDGPU::M0)
1572     .Case("scc", AMDGPU::SCC)
1573     .Case("tba", AMDGPU::TBA)
1574     .Case("tma", AMDGPU::TMA)
1575     .Case("flat_scratch_lo", AMDGPU::FLAT_SCR_LO)
1576     .Case("flat_scratch_hi", AMDGPU::FLAT_SCR_HI)
1577     .Case("xnack_mask_lo", AMDGPU::XNACK_MASK_LO)
1578     .Case("xnack_mask_hi", AMDGPU::XNACK_MASK_HI)
1579     .Case("vcc_lo", AMDGPU::VCC_LO)
1580     .Case("vcc_hi", AMDGPU::VCC_HI)
1581     .Case("exec_lo", AMDGPU::EXEC_LO)
1582     .Case("exec_hi", AMDGPU::EXEC_HI)
1583     .Case("tma_lo", AMDGPU::TMA_LO)
1584     .Case("tma_hi", AMDGPU::TMA_HI)
1585     .Case("tba_lo", AMDGPU::TBA_LO)
1586     .Case("tba_hi", AMDGPU::TBA_HI)
1587     .Default(0);
1588 }
1589 
1590 bool AMDGPUAsmParser::ParseRegister(unsigned &RegNo, SMLoc &StartLoc,
1591                                     SMLoc &EndLoc) {
1592   auto R = parseRegister();
1593   if (!R) return true;
1594   assert(R->isReg());
1595   RegNo = R->getReg();
1596   StartLoc = R->getStartLoc();
1597   EndLoc = R->getEndLoc();
1598   return false;
1599 }
1600 
1601 bool AMDGPUAsmParser::AddNextRegisterToList(unsigned &Reg, unsigned &RegWidth,
1602                                             RegisterKind RegKind, unsigned Reg1,
1603                                             unsigned RegNum) {
1604   switch (RegKind) {
1605   case IS_SPECIAL:
1606     if (Reg == AMDGPU::EXEC_LO && Reg1 == AMDGPU::EXEC_HI) {
1607       Reg = AMDGPU::EXEC;
1608       RegWidth = 2;
1609       return true;
1610     }
1611     if (Reg == AMDGPU::FLAT_SCR_LO && Reg1 == AMDGPU::FLAT_SCR_HI) {
1612       Reg = AMDGPU::FLAT_SCR;
1613       RegWidth = 2;
1614       return true;
1615     }
1616     if (Reg == AMDGPU::XNACK_MASK_LO && Reg1 == AMDGPU::XNACK_MASK_HI) {
1617       Reg = AMDGPU::XNACK_MASK;
1618       RegWidth = 2;
1619       return true;
1620     }
1621     if (Reg == AMDGPU::VCC_LO && Reg1 == AMDGPU::VCC_HI) {
1622       Reg = AMDGPU::VCC;
1623       RegWidth = 2;
1624       return true;
1625     }
1626     if (Reg == AMDGPU::TBA_LO && Reg1 == AMDGPU::TBA_HI) {
1627       Reg = AMDGPU::TBA;
1628       RegWidth = 2;
1629       return true;
1630     }
1631     if (Reg == AMDGPU::TMA_LO && Reg1 == AMDGPU::TMA_HI) {
1632       Reg = AMDGPU::TMA;
1633       RegWidth = 2;
1634       return true;
1635     }
1636     return false;
1637   case IS_VGPR:
1638   case IS_SGPR:
1639   case IS_TTMP:
1640     if (Reg1 != Reg + RegWidth) {
1641       return false;
1642     }
1643     RegWidth++;
1644     return true;
1645   default:
1646     llvm_unreachable("unexpected register kind");
1647   }
1648 }
1649 
1650 bool AMDGPUAsmParser::ParseAMDGPURegister(RegisterKind &RegKind, unsigned &Reg,
1651                                           unsigned &RegNum, unsigned &RegWidth,
1652                                           unsigned *DwordRegIndex) {
1653   if (DwordRegIndex) { *DwordRegIndex = 0; }
1654   const MCRegisterInfo *TRI = getContext().getRegisterInfo();
1655   if (getLexer().is(AsmToken::Identifier)) {
1656     StringRef RegName = Parser.getTok().getString();
1657     if ((Reg = getSpecialRegForName(RegName))) {
1658       Parser.Lex();
1659       RegKind = IS_SPECIAL;
1660     } else {
1661       unsigned RegNumIndex = 0;
1662       if (RegName[0] == 'v') {
1663         RegNumIndex = 1;
1664         RegKind = IS_VGPR;
1665       } else if (RegName[0] == 's') {
1666         RegNumIndex = 1;
1667         RegKind = IS_SGPR;
1668       } else if (RegName.startswith("ttmp")) {
1669         RegNumIndex = strlen("ttmp");
1670         RegKind = IS_TTMP;
1671       } else {
1672         return false;
1673       }
1674       if (RegName.size() > RegNumIndex) {
1675         // Single 32-bit register: vXX.
1676         if (RegName.substr(RegNumIndex).getAsInteger(10, RegNum))
1677           return false;
1678         Parser.Lex();
1679         RegWidth = 1;
1680       } else {
1681         // Range of registers: v[XX:YY]. ":YY" is optional.
1682         Parser.Lex();
1683         int64_t RegLo, RegHi;
1684         if (getLexer().isNot(AsmToken::LBrac))
1685           return false;
1686         Parser.Lex();
1687 
1688         if (getParser().parseAbsoluteExpression(RegLo))
1689           return false;
1690 
1691         const bool isRBrace = getLexer().is(AsmToken::RBrac);
1692         if (!isRBrace && getLexer().isNot(AsmToken::Colon))
1693           return false;
1694         Parser.Lex();
1695 
1696         if (isRBrace) {
1697           RegHi = RegLo;
1698         } else {
1699           if (getParser().parseAbsoluteExpression(RegHi))
1700             return false;
1701 
1702           if (getLexer().isNot(AsmToken::RBrac))
1703             return false;
1704           Parser.Lex();
1705         }
1706         RegNum = (unsigned) RegLo;
1707         RegWidth = (RegHi - RegLo) + 1;
1708       }
1709     }
1710   } else if (getLexer().is(AsmToken::LBrac)) {
1711     // List of consecutive registers: [s0,s1,s2,s3]
1712     Parser.Lex();
1713     if (!ParseAMDGPURegister(RegKind, Reg, RegNum, RegWidth, nullptr))
1714       return false;
1715     if (RegWidth != 1)
1716       return false;
1717     RegisterKind RegKind1;
1718     unsigned Reg1, RegNum1, RegWidth1;
1719     do {
1720       if (getLexer().is(AsmToken::Comma)) {
1721         Parser.Lex();
1722       } else if (getLexer().is(AsmToken::RBrac)) {
1723         Parser.Lex();
1724         break;
1725       } else if (ParseAMDGPURegister(RegKind1, Reg1, RegNum1, RegWidth1, nullptr)) {
1726         if (RegWidth1 != 1) {
1727           return false;
1728         }
1729         if (RegKind1 != RegKind) {
1730           return false;
1731         }
1732         if (!AddNextRegisterToList(Reg, RegWidth, RegKind1, Reg1, RegNum1)) {
1733           return false;
1734         }
1735       } else {
1736         return false;
1737       }
1738     } while (true);
1739   } else {
1740     return false;
1741   }
1742   switch (RegKind) {
1743   case IS_SPECIAL:
1744     RegNum = 0;
1745     RegWidth = 1;
1746     break;
1747   case IS_VGPR:
1748   case IS_SGPR:
1749   case IS_TTMP:
1750   {
1751     unsigned Size = 1;
1752     if (RegKind == IS_SGPR || RegKind == IS_TTMP) {
1753       // SGPR and TTMP registers must be aligned. Max required alignment is 4 dwords.
1754       Size = std::min(RegWidth, 4u);
1755     }
1756     if (RegNum % Size != 0)
1757       return false;
1758     if (DwordRegIndex) { *DwordRegIndex = RegNum; }
1759     RegNum = RegNum / Size;
1760     int RCID = getRegClass(RegKind, RegWidth);
1761     if (RCID == -1)
1762       return false;
1763     const MCRegisterClass RC = TRI->getRegClass(RCID);
1764     if (RegNum >= RC.getNumRegs())
1765       return false;
1766     Reg = RC.getRegister(RegNum);
1767     break;
1768   }
1769 
1770   default:
1771     llvm_unreachable("unexpected register kind");
1772   }
1773 
1774   if (!subtargetHasRegister(*TRI, Reg))
1775     return false;
1776   return true;
1777 }
1778 
1779 std::unique_ptr<AMDGPUOperand> AMDGPUAsmParser::parseRegister() {
1780   const auto &Tok = Parser.getTok();
1781   SMLoc StartLoc = Tok.getLoc();
1782   SMLoc EndLoc = Tok.getEndLoc();
1783   RegisterKind RegKind;
1784   unsigned Reg, RegNum, RegWidth, DwordRegIndex;
1785 
1786   if (!ParseAMDGPURegister(RegKind, Reg, RegNum, RegWidth, &DwordRegIndex)) {
1787     return nullptr;
1788   }
1789   KernelScope.usesRegister(RegKind, DwordRegIndex, RegWidth);
1790   return AMDGPUOperand::CreateReg(this, Reg, StartLoc, EndLoc, false);
1791 }
1792 
1793 bool
1794 AMDGPUAsmParser::parseAbsoluteExpr(int64_t &Val, bool AbsMod) {
1795   if (AbsMod && getLexer().peekTok().is(AsmToken::Pipe) &&
1796       (getLexer().getKind() == AsmToken::Integer ||
1797        getLexer().getKind() == AsmToken::Real)) {
1798     // This is a workaround for handling operands like these:
1799     //     |1.0|
1800     //     |-1|
1801     // This syntax is not compatible with syntax of standard
1802     // MC expressions (due to the trailing '|').
1803 
1804     SMLoc EndLoc;
1805     const MCExpr *Expr;
1806 
1807     if (getParser().parsePrimaryExpr(Expr, EndLoc)) {
1808       return true;
1809     }
1810 
1811     return !Expr->evaluateAsAbsolute(Val);
1812   }
1813 
1814   return getParser().parseAbsoluteExpression(Val);
1815 }
1816 
1817 OperandMatchResultTy
1818 AMDGPUAsmParser::parseImm(OperandVector &Operands, bool AbsMod) {
1819   // TODO: add syntactic sugar for 1/(2*PI)
1820   bool Minus = false;
1821   if (getLexer().getKind() == AsmToken::Minus) {
1822     const AsmToken NextToken = getLexer().peekTok();
1823     if (!NextToken.is(AsmToken::Integer) &&
1824         !NextToken.is(AsmToken::Real)) {
1825         return MatchOperand_NoMatch;
1826     }
1827     Minus = true;
1828     Parser.Lex();
1829   }
1830 
1831   SMLoc S = Parser.getTok().getLoc();
1832   switch(getLexer().getKind()) {
1833   case AsmToken::Integer: {
1834     int64_t IntVal;
1835     if (parseAbsoluteExpr(IntVal, AbsMod))
1836       return MatchOperand_ParseFail;
1837     if (Minus)
1838       IntVal *= -1;
1839     Operands.push_back(AMDGPUOperand::CreateImm(this, IntVal, S));
1840     return MatchOperand_Success;
1841   }
1842   case AsmToken::Real: {
1843     int64_t IntVal;
1844     if (parseAbsoluteExpr(IntVal, AbsMod))
1845       return MatchOperand_ParseFail;
1846 
1847     APFloat F(BitsToDouble(IntVal));
1848     if (Minus)
1849       F.changeSign();
1850     Operands.push_back(
1851         AMDGPUOperand::CreateImm(this, F.bitcastToAPInt().getZExtValue(), S,
1852                                  AMDGPUOperand::ImmTyNone, true));
1853     return MatchOperand_Success;
1854   }
1855   default:
1856     return MatchOperand_NoMatch;
1857   }
1858 }
1859 
1860 OperandMatchResultTy
1861 AMDGPUAsmParser::parseReg(OperandVector &Operands) {
1862   if (auto R = parseRegister()) {
1863     assert(R->isReg());
1864     R->Reg.IsForcedVOP3 = isForcedVOP3();
1865     Operands.push_back(std::move(R));
1866     return MatchOperand_Success;
1867   }
1868   return MatchOperand_NoMatch;
1869 }
1870 
1871 OperandMatchResultTy
1872 AMDGPUAsmParser::parseRegOrImm(OperandVector &Operands, bool AbsMod) {
1873   auto res = parseImm(Operands, AbsMod);
1874   if (res != MatchOperand_NoMatch) {
1875     return res;
1876   }
1877 
1878   return parseReg(Operands);
1879 }
1880 
1881 OperandMatchResultTy
1882 AMDGPUAsmParser::parseRegOrImmWithFPInputMods(OperandVector &Operands,
1883                                               bool AllowImm) {
1884   bool Negate = false, Negate2 = false, Abs = false, Abs2 = false;
1885 
1886   if (getLexer().getKind()== AsmToken::Minus) {
1887     const AsmToken NextToken = getLexer().peekTok();
1888 
1889     // Disable ambiguous constructs like '--1' etc. Should use neg(-1) instead.
1890     if (NextToken.is(AsmToken::Minus)) {
1891       Error(Parser.getTok().getLoc(), "invalid syntax, expected 'neg' modifier");
1892       return MatchOperand_ParseFail;
1893     }
1894 
1895     // '-' followed by an integer literal N should be interpreted as integer
1896     // negation rather than a floating-point NEG modifier applied to N.
1897     // Beside being contr-intuitive, such use of floating-point NEG modifier
1898     // results in different meaning of integer literals used with VOP1/2/C
1899     // and VOP3, for example:
1900     //    v_exp_f32_e32 v5, -1 // VOP1: src0 = 0xFFFFFFFF
1901     //    v_exp_f32_e64 v5, -1 // VOP3: src0 = 0x80000001
1902     // Negative fp literals should be handled likewise for unifomtity
1903     if (!NextToken.is(AsmToken::Integer) && !NextToken.is(AsmToken::Real)) {
1904       Parser.Lex();
1905       Negate = true;
1906     }
1907   }
1908 
1909   if (getLexer().getKind() == AsmToken::Identifier &&
1910       Parser.getTok().getString() == "neg") {
1911     if (Negate) {
1912       Error(Parser.getTok().getLoc(), "expected register or immediate");
1913       return MatchOperand_ParseFail;
1914     }
1915     Parser.Lex();
1916     Negate2 = true;
1917     if (getLexer().isNot(AsmToken::LParen)) {
1918       Error(Parser.getTok().getLoc(), "expected left paren after neg");
1919       return MatchOperand_ParseFail;
1920     }
1921     Parser.Lex();
1922   }
1923 
1924   if (getLexer().getKind() == AsmToken::Identifier &&
1925       Parser.getTok().getString() == "abs") {
1926     Parser.Lex();
1927     Abs2 = true;
1928     if (getLexer().isNot(AsmToken::LParen)) {
1929       Error(Parser.getTok().getLoc(), "expected left paren after abs");
1930       return MatchOperand_ParseFail;
1931     }
1932     Parser.Lex();
1933   }
1934 
1935   if (getLexer().getKind() == AsmToken::Pipe) {
1936     if (Abs2) {
1937       Error(Parser.getTok().getLoc(), "expected register or immediate");
1938       return MatchOperand_ParseFail;
1939     }
1940     Parser.Lex();
1941     Abs = true;
1942   }
1943 
1944   OperandMatchResultTy Res;
1945   if (AllowImm) {
1946     Res = parseRegOrImm(Operands, Abs);
1947   } else {
1948     Res = parseReg(Operands);
1949   }
1950   if (Res != MatchOperand_Success) {
1951     return Res;
1952   }
1953 
1954   AMDGPUOperand::Modifiers Mods;
1955   if (Abs) {
1956     if (getLexer().getKind() != AsmToken::Pipe) {
1957       Error(Parser.getTok().getLoc(), "expected vertical bar");
1958       return MatchOperand_ParseFail;
1959     }
1960     Parser.Lex();
1961     Mods.Abs = true;
1962   }
1963   if (Abs2) {
1964     if (getLexer().isNot(AsmToken::RParen)) {
1965       Error(Parser.getTok().getLoc(), "expected closing parentheses");
1966       return MatchOperand_ParseFail;
1967     }
1968     Parser.Lex();
1969     Mods.Abs = true;
1970   }
1971 
1972   if (Negate) {
1973     Mods.Neg = true;
1974   } else if (Negate2) {
1975     if (getLexer().isNot(AsmToken::RParen)) {
1976       Error(Parser.getTok().getLoc(), "expected closing parentheses");
1977       return MatchOperand_ParseFail;
1978     }
1979     Parser.Lex();
1980     Mods.Neg = true;
1981   }
1982 
1983   if (Mods.hasFPModifiers()) {
1984     AMDGPUOperand &Op = static_cast<AMDGPUOperand &>(*Operands.back());
1985     Op.setModifiers(Mods);
1986   }
1987   return MatchOperand_Success;
1988 }
1989 
1990 OperandMatchResultTy
1991 AMDGPUAsmParser::parseRegOrImmWithIntInputMods(OperandVector &Operands,
1992                                                bool AllowImm) {
1993   bool Sext = false;
1994 
1995   if (getLexer().getKind() == AsmToken::Identifier &&
1996       Parser.getTok().getString() == "sext") {
1997     Parser.Lex();
1998     Sext = true;
1999     if (getLexer().isNot(AsmToken::LParen)) {
2000       Error(Parser.getTok().getLoc(), "expected left paren after sext");
2001       return MatchOperand_ParseFail;
2002     }
2003     Parser.Lex();
2004   }
2005 
2006   OperandMatchResultTy Res;
2007   if (AllowImm) {
2008     Res = parseRegOrImm(Operands);
2009   } else {
2010     Res = parseReg(Operands);
2011   }
2012   if (Res != MatchOperand_Success) {
2013     return Res;
2014   }
2015 
2016   AMDGPUOperand::Modifiers Mods;
2017   if (Sext) {
2018     if (getLexer().isNot(AsmToken::RParen)) {
2019       Error(Parser.getTok().getLoc(), "expected closing parentheses");
2020       return MatchOperand_ParseFail;
2021     }
2022     Parser.Lex();
2023     Mods.Sext = true;
2024   }
2025 
2026   if (Mods.hasIntModifiers()) {
2027     AMDGPUOperand &Op = static_cast<AMDGPUOperand &>(*Operands.back());
2028     Op.setModifiers(Mods);
2029   }
2030 
2031   return MatchOperand_Success;
2032 }
2033 
2034 OperandMatchResultTy
2035 AMDGPUAsmParser::parseRegWithFPInputMods(OperandVector &Operands) {
2036   return parseRegOrImmWithFPInputMods(Operands, false);
2037 }
2038 
2039 OperandMatchResultTy
2040 AMDGPUAsmParser::parseRegWithIntInputMods(OperandVector &Operands) {
2041   return parseRegOrImmWithIntInputMods(Operands, false);
2042 }
2043 
2044 OperandMatchResultTy AMDGPUAsmParser::parseVReg32OrOff(OperandVector &Operands) {
2045   std::unique_ptr<AMDGPUOperand> Reg = parseRegister();
2046   if (Reg) {
2047     Operands.push_back(std::move(Reg));
2048     return MatchOperand_Success;
2049   }
2050 
2051   const AsmToken &Tok = Parser.getTok();
2052   if (Tok.getString() == "off") {
2053     Operands.push_back(AMDGPUOperand::CreateImm(this, 0, Tok.getLoc(),
2054                                                 AMDGPUOperand::ImmTyOff, false));
2055     Parser.Lex();
2056     return MatchOperand_Success;
2057   }
2058 
2059   return MatchOperand_NoMatch;
2060 }
2061 
2062 unsigned AMDGPUAsmParser::checkTargetMatchPredicate(MCInst &Inst) {
2063   uint64_t TSFlags = MII.get(Inst.getOpcode()).TSFlags;
2064 
2065   if ((getForcedEncodingSize() == 32 && (TSFlags & SIInstrFlags::VOP3)) ||
2066       (getForcedEncodingSize() == 64 && !(TSFlags & SIInstrFlags::VOP3)) ||
2067       (isForcedDPP() && !(TSFlags & SIInstrFlags::DPP)) ||
2068       (isForcedSDWA() && !(TSFlags & SIInstrFlags::SDWA)) )
2069     return Match_InvalidOperand;
2070 
2071   if ((TSFlags & SIInstrFlags::VOP3) &&
2072       (TSFlags & SIInstrFlags::VOPAsmPrefer32Bit) &&
2073       getForcedEncodingSize() != 64)
2074     return Match_PreferE32;
2075 
2076   if (Inst.getOpcode() == AMDGPU::V_MAC_F32_sdwa_vi ||
2077       Inst.getOpcode() == AMDGPU::V_MAC_F16_sdwa_vi) {
2078     // v_mac_f32/16 allow only dst_sel == DWORD;
2079     auto OpNum =
2080         AMDGPU::getNamedOperandIdx(Inst.getOpcode(), AMDGPU::OpName::dst_sel);
2081     const auto &Op = Inst.getOperand(OpNum);
2082     if (!Op.isImm() || Op.getImm() != AMDGPU::SDWA::SdwaSel::DWORD) {
2083       return Match_InvalidOperand;
2084     }
2085   }
2086 
2087   if ((TSFlags & SIInstrFlags::FLAT) && !hasFlatOffsets()) {
2088     // FIXME: Produces error without correct column reported.
2089     auto OpNum =
2090         AMDGPU::getNamedOperandIdx(Inst.getOpcode(), AMDGPU::OpName::offset);
2091     const auto &Op = Inst.getOperand(OpNum);
2092     if (Op.getImm() != 0)
2093       return Match_InvalidOperand;
2094   }
2095 
2096   return Match_Success;
2097 }
2098 
2099 // What asm variants we should check
2100 ArrayRef<unsigned> AMDGPUAsmParser::getMatchedVariants() const {
2101   if (getForcedEncodingSize() == 32) {
2102     static const unsigned Variants[] = {AMDGPUAsmVariants::DEFAULT};
2103     return makeArrayRef(Variants);
2104   }
2105 
2106   if (isForcedVOP3()) {
2107     static const unsigned Variants[] = {AMDGPUAsmVariants::VOP3};
2108     return makeArrayRef(Variants);
2109   }
2110 
2111   if (isForcedSDWA()) {
2112     static const unsigned Variants[] = {AMDGPUAsmVariants::SDWA,
2113                                         AMDGPUAsmVariants::SDWA9};
2114     return makeArrayRef(Variants);
2115   }
2116 
2117   if (isForcedDPP()) {
2118     static const unsigned Variants[] = {AMDGPUAsmVariants::DPP};
2119     return makeArrayRef(Variants);
2120   }
2121 
2122   static const unsigned Variants[] = {
2123     AMDGPUAsmVariants::DEFAULT, AMDGPUAsmVariants::VOP3,
2124     AMDGPUAsmVariants::SDWA, AMDGPUAsmVariants::SDWA9, AMDGPUAsmVariants::DPP
2125   };
2126 
2127   return makeArrayRef(Variants);
2128 }
2129 
2130 unsigned AMDGPUAsmParser::findImplicitSGPRReadInVOP(const MCInst &Inst) const {
2131   const MCInstrDesc &Desc = MII.get(Inst.getOpcode());
2132   const unsigned Num = Desc.getNumImplicitUses();
2133   for (unsigned i = 0; i < Num; ++i) {
2134     unsigned Reg = Desc.ImplicitUses[i];
2135     switch (Reg) {
2136     case AMDGPU::FLAT_SCR:
2137     case AMDGPU::VCC:
2138     case AMDGPU::M0:
2139       return Reg;
2140     default:
2141       break;
2142     }
2143   }
2144   return AMDGPU::NoRegister;
2145 }
2146 
2147 // NB: This code is correct only when used to check constant
2148 // bus limitations because GFX7 support no f16 inline constants.
2149 // Note that there are no cases when a GFX7 opcode violates
2150 // constant bus limitations due to the use of an f16 constant.
2151 bool AMDGPUAsmParser::isInlineConstant(const MCInst &Inst,
2152                                        unsigned OpIdx) const {
2153   const MCInstrDesc &Desc = MII.get(Inst.getOpcode());
2154 
2155   if (!AMDGPU::isSISrcOperand(Desc, OpIdx)) {
2156     return false;
2157   }
2158 
2159   const MCOperand &MO = Inst.getOperand(OpIdx);
2160 
2161   int64_t Val = MO.getImm();
2162   auto OpSize = AMDGPU::getOperandSize(Desc, OpIdx);
2163 
2164   switch (OpSize) { // expected operand size
2165   case 8:
2166     return AMDGPU::isInlinableLiteral64(Val, hasInv2PiInlineImm());
2167   case 4:
2168     return AMDGPU::isInlinableLiteral32(Val, hasInv2PiInlineImm());
2169   case 2: {
2170     const unsigned OperandType = Desc.OpInfo[OpIdx].OperandType;
2171     if (OperandType == AMDGPU::OPERAND_REG_INLINE_C_V2INT16 ||
2172         OperandType == AMDGPU::OPERAND_REG_INLINE_C_V2FP16) {
2173       return AMDGPU::isInlinableLiteralV216(Val, hasInv2PiInlineImm());
2174     } else {
2175       return AMDGPU::isInlinableLiteral16(Val, hasInv2PiInlineImm());
2176     }
2177   }
2178   default:
2179     llvm_unreachable("invalid operand size");
2180   }
2181 }
2182 
2183 bool AMDGPUAsmParser::usesConstantBus(const MCInst &Inst, unsigned OpIdx) {
2184   const MCOperand &MO = Inst.getOperand(OpIdx);
2185   if (MO.isImm()) {
2186     return !isInlineConstant(Inst, OpIdx);
2187   }
2188   return !MO.isReg() ||
2189          isSGPR(mc2PseudoReg(MO.getReg()), getContext().getRegisterInfo());
2190 }
2191 
2192 bool AMDGPUAsmParser::validateConstantBusLimitations(const MCInst &Inst) {
2193   const unsigned Opcode = Inst.getOpcode();
2194   const MCInstrDesc &Desc = MII.get(Opcode);
2195   unsigned ConstantBusUseCount = 0;
2196 
2197   if (Desc.TSFlags &
2198       (SIInstrFlags::VOPC |
2199        SIInstrFlags::VOP1 | SIInstrFlags::VOP2 |
2200        SIInstrFlags::VOP3 | SIInstrFlags::VOP3P |
2201        SIInstrFlags::SDWA)) {
2202     // Check special imm operands (used by madmk, etc)
2203     if (AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::imm) != -1) {
2204       ++ConstantBusUseCount;
2205     }
2206 
2207     unsigned SGPRUsed = findImplicitSGPRReadInVOP(Inst);
2208     if (SGPRUsed != AMDGPU::NoRegister) {
2209       ++ConstantBusUseCount;
2210     }
2211 
2212     const int Src0Idx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::src0);
2213     const int Src1Idx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::src1);
2214     const int Src2Idx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::src2);
2215 
2216     const int OpIndices[] = { Src0Idx, Src1Idx, Src2Idx };
2217 
2218     for (int OpIdx : OpIndices) {
2219       if (OpIdx == -1) break;
2220 
2221       const MCOperand &MO = Inst.getOperand(OpIdx);
2222       if (usesConstantBus(Inst, OpIdx)) {
2223         if (MO.isReg()) {
2224           const unsigned Reg = mc2PseudoReg(MO.getReg());
2225           // Pairs of registers with a partial intersections like these
2226           //   s0, s[0:1]
2227           //   flat_scratch_lo, flat_scratch
2228           //   flat_scratch_lo, flat_scratch_hi
2229           // are theoretically valid but they are disabled anyway.
2230           // Note that this code mimics SIInstrInfo::verifyInstruction
2231           if (Reg != SGPRUsed) {
2232             ++ConstantBusUseCount;
2233           }
2234           SGPRUsed = Reg;
2235         } else { // Expression or a literal
2236           ++ConstantBusUseCount;
2237         }
2238       }
2239     }
2240   }
2241 
2242   return ConstantBusUseCount <= 1;
2243 }
2244 
2245 bool AMDGPUAsmParser::validateEarlyClobberLimitations(const MCInst &Inst) {
2246   const unsigned Opcode = Inst.getOpcode();
2247   const MCInstrDesc &Desc = MII.get(Opcode);
2248 
2249   const int DstIdx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::vdst);
2250   if (DstIdx == -1 ||
2251       Desc.getOperandConstraint(DstIdx, MCOI::EARLY_CLOBBER) == -1) {
2252     return true;
2253   }
2254 
2255   const MCRegisterInfo *TRI = getContext().getRegisterInfo();
2256 
2257   const int Src0Idx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::src0);
2258   const int Src1Idx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::src1);
2259   const int Src2Idx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::src2);
2260 
2261   assert(DstIdx != -1);
2262   const MCOperand &Dst = Inst.getOperand(DstIdx);
2263   assert(Dst.isReg());
2264   const unsigned DstReg = mc2PseudoReg(Dst.getReg());
2265 
2266   const int SrcIndices[] = { Src0Idx, Src1Idx, Src2Idx };
2267 
2268   for (int SrcIdx : SrcIndices) {
2269     if (SrcIdx == -1) break;
2270     const MCOperand &Src = Inst.getOperand(SrcIdx);
2271     if (Src.isReg()) {
2272       const unsigned SrcReg = mc2PseudoReg(Src.getReg());
2273       if (isRegIntersect(DstReg, SrcReg, TRI)) {
2274         return false;
2275       }
2276     }
2277   }
2278 
2279   return true;
2280 }
2281 
2282 bool AMDGPUAsmParser::validateIntClampSupported(const MCInst &Inst) {
2283 
2284   const unsigned Opc = Inst.getOpcode();
2285   const MCInstrDesc &Desc = MII.get(Opc);
2286 
2287   if ((Desc.TSFlags & SIInstrFlags::IntClamp) != 0 && !hasIntClamp()) {
2288     int ClampIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::clamp);
2289     assert(ClampIdx != -1);
2290     return Inst.getOperand(ClampIdx).getImm() == 0;
2291   }
2292 
2293   return true;
2294 }
2295 
2296 bool AMDGPUAsmParser::validateMIMGDataSize(const MCInst &Inst) {
2297 
2298   const unsigned Opc = Inst.getOpcode();
2299   const MCInstrDesc &Desc = MII.get(Opc);
2300 
2301   if ((Desc.TSFlags & SIInstrFlags::MIMG) == 0)
2302     return true;
2303 
2304   // Gather4 instructions do not need validation: dst size is hardcoded.
2305   if (Desc.TSFlags & SIInstrFlags::Gather4)
2306     return true;
2307 
2308   int VDataIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::vdata);
2309   int DMaskIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::dmask);
2310   int TFEIdx   = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::tfe);
2311 
2312   assert(VDataIdx != -1);
2313   assert(DMaskIdx != -1);
2314   assert(TFEIdx != -1);
2315 
2316   unsigned VDataSize = AMDGPU::getRegOperandSize(getMRI(), Desc, VDataIdx);
2317   unsigned TFESize = Inst.getOperand(TFEIdx).getImm()? 1 : 0;
2318   unsigned DMask = Inst.getOperand(DMaskIdx).getImm() & 0xf;
2319   if (DMask == 0)
2320     DMask = 1;
2321 
2322   unsigned DataSize = countPopulation(DMask);
2323   if ((Desc.TSFlags & SIInstrFlags::D16) != 0 && hasPackedD16()) {
2324     DataSize = (DataSize + 1) / 2;
2325   }
2326 
2327   return (VDataSize / 4) == DataSize + TFESize;
2328 }
2329 
2330 bool AMDGPUAsmParser::validateMIMGAtomicDMask(const MCInst &Inst) {
2331 
2332   const unsigned Opc = Inst.getOpcode();
2333   const MCInstrDesc &Desc = MII.get(Opc);
2334 
2335   if ((Desc.TSFlags & SIInstrFlags::MIMG) == 0)
2336     return true;
2337   if (!Desc.mayLoad() || !Desc.mayStore())
2338     return true; // Not atomic
2339 
2340   int DMaskIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::dmask);
2341   unsigned DMask = Inst.getOperand(DMaskIdx).getImm() & 0xf;
2342 
2343   // This is an incomplete check because image_atomic_cmpswap
2344   // may only use 0x3 and 0xf while other atomic operations
2345   // may use 0x1 and 0x3. However these limitations are
2346   // verified when we check that dmask matches dst size.
2347   return DMask == 0x1 || DMask == 0x3 || DMask == 0xf;
2348 }
2349 
2350 bool AMDGPUAsmParser::validateMIMGGatherDMask(const MCInst &Inst) {
2351 
2352   const unsigned Opc = Inst.getOpcode();
2353   const MCInstrDesc &Desc = MII.get(Opc);
2354 
2355   if ((Desc.TSFlags & SIInstrFlags::Gather4) == 0)
2356     return true;
2357 
2358   int DMaskIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::dmask);
2359   unsigned DMask = Inst.getOperand(DMaskIdx).getImm() & 0xf;
2360 
2361   // GATHER4 instructions use dmask in a different fashion compared to
2362   // other MIMG instructions. The only useful DMASK values are
2363   // 1=red, 2=green, 4=blue, 8=alpha. (e.g. 1 returns
2364   // (red,red,red,red) etc.) The ISA document doesn't mention
2365   // this.
2366   return DMask == 0x1 || DMask == 0x2 || DMask == 0x4 || DMask == 0x8;
2367 }
2368 
2369 bool AMDGPUAsmParser::validateMIMGR128(const MCInst &Inst) {
2370 
2371   const unsigned Opc = Inst.getOpcode();
2372   const MCInstrDesc &Desc = MII.get(Opc);
2373 
2374   if ((Desc.TSFlags & SIInstrFlags::MIMG) == 0)
2375     return true;
2376 
2377   int Idx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::r128);
2378   assert(Idx != -1);
2379 
2380   bool R128 = (Inst.getOperand(Idx).getImm() != 0);
2381 
2382   return !R128 || hasMIMG_R128();
2383 }
2384 
2385 bool AMDGPUAsmParser::validateMIMGD16(const MCInst &Inst) {
2386 
2387   const unsigned Opc = Inst.getOpcode();
2388   const MCInstrDesc &Desc = MII.get(Opc);
2389 
2390   if ((Desc.TSFlags & SIInstrFlags::MIMG) == 0)
2391     return true;
2392   if ((Desc.TSFlags & SIInstrFlags::D16) == 0)
2393     return true;
2394 
2395   return !isCI() && !isSI();
2396 }
2397 
2398 bool AMDGPUAsmParser::validateInstruction(const MCInst &Inst,
2399                                           const SMLoc &IDLoc) {
2400   if (!validateConstantBusLimitations(Inst)) {
2401     Error(IDLoc,
2402       "invalid operand (violates constant bus restrictions)");
2403     return false;
2404   }
2405   if (!validateEarlyClobberLimitations(Inst)) {
2406     Error(IDLoc,
2407       "destination must be different than all sources");
2408     return false;
2409   }
2410   if (!validateIntClampSupported(Inst)) {
2411     Error(IDLoc,
2412       "integer clamping is not supported on this GPU");
2413     return false;
2414   }
2415   if (!validateMIMGR128(Inst)) {
2416     Error(IDLoc,
2417       "r128 modifier is not supported on this GPU");
2418     return false;
2419   }
2420   // For MUBUF/MTBUF d16 is a part of opcode, so there is nothing to validate.
2421   if (!validateMIMGD16(Inst)) {
2422     Error(IDLoc,
2423       "d16 modifier is not supported on this GPU");
2424     return false;
2425   }
2426   if (!validateMIMGDataSize(Inst)) {
2427     Error(IDLoc,
2428       "image data size does not match dmask and tfe");
2429     return false;
2430   }
2431   if (!validateMIMGAtomicDMask(Inst)) {
2432     Error(IDLoc,
2433       "invalid atomic image dmask");
2434     return false;
2435   }
2436   if (!validateMIMGGatherDMask(Inst)) {
2437     Error(IDLoc,
2438       "invalid image_gather dmask: only one bit must be set");
2439     return false;
2440   }
2441 
2442   return true;
2443 }
2444 
2445 static std::string AMDGPUMnemonicSpellCheck(StringRef S, uint64_t FBS,
2446                                             unsigned VariantID = 0);
2447 
2448 bool AMDGPUAsmParser::MatchAndEmitInstruction(SMLoc IDLoc, unsigned &Opcode,
2449                                               OperandVector &Operands,
2450                                               MCStreamer &Out,
2451                                               uint64_t &ErrorInfo,
2452                                               bool MatchingInlineAsm) {
2453   MCInst Inst;
2454   unsigned Result = Match_Success;
2455   for (auto Variant : getMatchedVariants()) {
2456     uint64_t EI;
2457     auto R = MatchInstructionImpl(Operands, Inst, EI, MatchingInlineAsm,
2458                                   Variant);
2459     // We order match statuses from least to most specific. We use most specific
2460     // status as resulting
2461     // Match_MnemonicFail < Match_InvalidOperand < Match_MissingFeature < Match_PreferE32
2462     if ((R == Match_Success) ||
2463         (R == Match_PreferE32) ||
2464         (R == Match_MissingFeature && Result != Match_PreferE32) ||
2465         (R == Match_InvalidOperand && Result != Match_MissingFeature
2466                                    && Result != Match_PreferE32) ||
2467         (R == Match_MnemonicFail   && Result != Match_InvalidOperand
2468                                    && Result != Match_MissingFeature
2469                                    && Result != Match_PreferE32)) {
2470       Result = R;
2471       ErrorInfo = EI;
2472     }
2473     if (R == Match_Success)
2474       break;
2475   }
2476 
2477   switch (Result) {
2478   default: break;
2479   case Match_Success:
2480     if (!validateInstruction(Inst, IDLoc)) {
2481       return true;
2482     }
2483     Inst.setLoc(IDLoc);
2484     Out.EmitInstruction(Inst, getSTI());
2485     return false;
2486 
2487   case Match_MissingFeature:
2488     return Error(IDLoc, "instruction not supported on this GPU");
2489 
2490   case Match_MnemonicFail: {
2491     uint64_t FBS = ComputeAvailableFeatures(getSTI().getFeatureBits());
2492     std::string Suggestion = AMDGPUMnemonicSpellCheck(
2493         ((AMDGPUOperand &)*Operands[0]).getToken(), FBS);
2494     return Error(IDLoc, "invalid instruction" + Suggestion,
2495                  ((AMDGPUOperand &)*Operands[0]).getLocRange());
2496   }
2497 
2498   case Match_InvalidOperand: {
2499     SMLoc ErrorLoc = IDLoc;
2500     if (ErrorInfo != ~0ULL) {
2501       if (ErrorInfo >= Operands.size()) {
2502         return Error(IDLoc, "too few operands for instruction");
2503       }
2504       ErrorLoc = ((AMDGPUOperand &)*Operands[ErrorInfo]).getStartLoc();
2505       if (ErrorLoc == SMLoc())
2506         ErrorLoc = IDLoc;
2507     }
2508     return Error(ErrorLoc, "invalid operand for instruction");
2509   }
2510 
2511   case Match_PreferE32:
2512     return Error(IDLoc, "internal error: instruction without _e64 suffix "
2513                         "should be encoded as e32");
2514   }
2515   llvm_unreachable("Implement any new match types added!");
2516 }
2517 
2518 bool AMDGPUAsmParser::ParseAsAbsoluteExpression(uint32_t &Ret) {
2519   int64_t Tmp = -1;
2520   if (getLexer().isNot(AsmToken::Integer) && getLexer().isNot(AsmToken::Identifier)) {
2521     return true;
2522   }
2523   if (getParser().parseAbsoluteExpression(Tmp)) {
2524     return true;
2525   }
2526   Ret = static_cast<uint32_t>(Tmp);
2527   return false;
2528 }
2529 
2530 bool AMDGPUAsmParser::ParseDirectiveMajorMinor(uint32_t &Major,
2531                                                uint32_t &Minor) {
2532   if (ParseAsAbsoluteExpression(Major))
2533     return TokError("invalid major version");
2534 
2535   if (getLexer().isNot(AsmToken::Comma))
2536     return TokError("minor version number required, comma expected");
2537   Lex();
2538 
2539   if (ParseAsAbsoluteExpression(Minor))
2540     return TokError("invalid minor version");
2541 
2542   return false;
2543 }
2544 
2545 bool AMDGPUAsmParser::ParseDirectiveHSACodeObjectVersion() {
2546   uint32_t Major;
2547   uint32_t Minor;
2548 
2549   if (ParseDirectiveMajorMinor(Major, Minor))
2550     return true;
2551 
2552   getTargetStreamer().EmitDirectiveHSACodeObjectVersion(Major, Minor);
2553   return false;
2554 }
2555 
2556 bool AMDGPUAsmParser::ParseDirectiveHSACodeObjectISA() {
2557   uint32_t Major;
2558   uint32_t Minor;
2559   uint32_t Stepping;
2560   StringRef VendorName;
2561   StringRef ArchName;
2562 
2563   // If this directive has no arguments, then use the ISA version for the
2564   // targeted GPU.
2565   if (getLexer().is(AsmToken::EndOfStatement)) {
2566     AMDGPU::IsaInfo::IsaVersion ISA =
2567         AMDGPU::IsaInfo::getIsaVersion(getFeatureBits());
2568     getTargetStreamer().EmitDirectiveHSACodeObjectISA(ISA.Major, ISA.Minor,
2569                                                       ISA.Stepping,
2570                                                       "AMD", "AMDGPU");
2571     return false;
2572   }
2573 
2574   if (ParseDirectiveMajorMinor(Major, Minor))
2575     return true;
2576 
2577   if (getLexer().isNot(AsmToken::Comma))
2578     return TokError("stepping version number required, comma expected");
2579   Lex();
2580 
2581   if (ParseAsAbsoluteExpression(Stepping))
2582     return TokError("invalid stepping version");
2583 
2584   if (getLexer().isNot(AsmToken::Comma))
2585     return TokError("vendor name required, comma expected");
2586   Lex();
2587 
2588   if (getLexer().isNot(AsmToken::String))
2589     return TokError("invalid vendor name");
2590 
2591   VendorName = getLexer().getTok().getStringContents();
2592   Lex();
2593 
2594   if (getLexer().isNot(AsmToken::Comma))
2595     return TokError("arch name required, comma expected");
2596   Lex();
2597 
2598   if (getLexer().isNot(AsmToken::String))
2599     return TokError("invalid arch name");
2600 
2601   ArchName = getLexer().getTok().getStringContents();
2602   Lex();
2603 
2604   getTargetStreamer().EmitDirectiveHSACodeObjectISA(Major, Minor, Stepping,
2605                                                     VendorName, ArchName);
2606   return false;
2607 }
2608 
2609 bool AMDGPUAsmParser::ParseAMDKernelCodeTValue(StringRef ID,
2610                                                amd_kernel_code_t &Header) {
2611   // max_scratch_backing_memory_byte_size is deprecated. Ignore it while parsing
2612   // assembly for backwards compatibility.
2613   if (ID == "max_scratch_backing_memory_byte_size") {
2614     Parser.eatToEndOfStatement();
2615     return false;
2616   }
2617 
2618   SmallString<40> ErrStr;
2619   raw_svector_ostream Err(ErrStr);
2620   if (!parseAmdKernelCodeField(ID, getParser(), Header, Err)) {
2621     return TokError(Err.str());
2622   }
2623   Lex();
2624   return false;
2625 }
2626 
2627 bool AMDGPUAsmParser::ParseDirectiveAMDKernelCodeT() {
2628   amd_kernel_code_t Header;
2629   AMDGPU::initDefaultAMDKernelCodeT(Header, getFeatureBits());
2630 
2631   while (true) {
2632     // Lex EndOfStatement.  This is in a while loop, because lexing a comment
2633     // will set the current token to EndOfStatement.
2634     while(getLexer().is(AsmToken::EndOfStatement))
2635       Lex();
2636 
2637     if (getLexer().isNot(AsmToken::Identifier))
2638       return TokError("expected value identifier or .end_amd_kernel_code_t");
2639 
2640     StringRef ID = getLexer().getTok().getIdentifier();
2641     Lex();
2642 
2643     if (ID == ".end_amd_kernel_code_t")
2644       break;
2645 
2646     if (ParseAMDKernelCodeTValue(ID, Header))
2647       return true;
2648   }
2649 
2650   getTargetStreamer().EmitAMDKernelCodeT(Header);
2651 
2652   return false;
2653 }
2654 
2655 bool AMDGPUAsmParser::ParseDirectiveAMDGPUHsaKernel() {
2656   if (getLexer().isNot(AsmToken::Identifier))
2657     return TokError("expected symbol name");
2658 
2659   StringRef KernelName = Parser.getTok().getString();
2660 
2661   getTargetStreamer().EmitAMDGPUSymbolType(KernelName,
2662                                            ELF::STT_AMDGPU_HSA_KERNEL);
2663   Lex();
2664   KernelScope.initialize(getContext());
2665   return false;
2666 }
2667 
2668 bool AMDGPUAsmParser::ParseDirectiveISAVersion() {
2669   if (getSTI().getTargetTriple().getArch() != Triple::amdgcn) {
2670     return Error(getParser().getTok().getLoc(),
2671                  ".amd_amdgpu_isa directive is not available on non-amdgcn "
2672                  "architectures");
2673   }
2674 
2675   auto ISAVersionStringFromASM = getLexer().getTok().getStringContents();
2676 
2677   std::string ISAVersionStringFromSTI;
2678   raw_string_ostream ISAVersionStreamFromSTI(ISAVersionStringFromSTI);
2679   IsaInfo::streamIsaVersion(&getSTI(), ISAVersionStreamFromSTI);
2680 
2681   if (ISAVersionStringFromASM != ISAVersionStreamFromSTI.str()) {
2682     return Error(getParser().getTok().getLoc(),
2683                  ".amd_amdgpu_isa directive does not match triple and/or mcpu "
2684                  "arguments specified through the command line");
2685   }
2686 
2687   getTargetStreamer().EmitISAVersion(ISAVersionStreamFromSTI.str());
2688   Lex();
2689 
2690   return false;
2691 }
2692 
2693 bool AMDGPUAsmParser::ParseDirectiveHSAMetadata() {
2694   if (getSTI().getTargetTriple().getOS() != Triple::AMDHSA) {
2695     return Error(getParser().getTok().getLoc(),
2696                  (Twine(HSAMD::AssemblerDirectiveBegin) + Twine(" directive is "
2697                  "not available on non-amdhsa OSes")).str());
2698   }
2699 
2700   std::string HSAMetadataString;
2701   raw_string_ostream YamlStream(HSAMetadataString);
2702 
2703   getLexer().setSkipSpace(false);
2704 
2705   bool FoundEnd = false;
2706   while (!getLexer().is(AsmToken::Eof)) {
2707     while (getLexer().is(AsmToken::Space)) {
2708       YamlStream << getLexer().getTok().getString();
2709       Lex();
2710     }
2711 
2712     if (getLexer().is(AsmToken::Identifier)) {
2713       StringRef ID = getLexer().getTok().getIdentifier();
2714       if (ID == AMDGPU::HSAMD::AssemblerDirectiveEnd) {
2715         Lex();
2716         FoundEnd = true;
2717         break;
2718       }
2719     }
2720 
2721     YamlStream << Parser.parseStringToEndOfStatement()
2722                << getContext().getAsmInfo()->getSeparatorString();
2723 
2724     Parser.eatToEndOfStatement();
2725   }
2726 
2727   getLexer().setSkipSpace(true);
2728 
2729   if (getLexer().is(AsmToken::Eof) && !FoundEnd) {
2730     return TokError(Twine("expected directive ") +
2731                     Twine(HSAMD::AssemblerDirectiveEnd) + Twine(" not found"));
2732   }
2733 
2734   YamlStream.flush();
2735 
2736   if (!getTargetStreamer().EmitHSAMetadata(HSAMetadataString))
2737     return Error(getParser().getTok().getLoc(), "invalid HSA metadata");
2738 
2739   return false;
2740 }
2741 
2742 bool AMDGPUAsmParser::ParseDirectivePALMetadata() {
2743   if (getSTI().getTargetTriple().getOS() != Triple::AMDPAL) {
2744     return Error(getParser().getTok().getLoc(),
2745                  (Twine(PALMD::AssemblerDirective) + Twine(" directive is "
2746                  "not available on non-amdpal OSes")).str());
2747   }
2748 
2749   PALMD::Metadata PALMetadata;
2750   for (;;) {
2751     uint32_t Value;
2752     if (ParseAsAbsoluteExpression(Value)) {
2753       return TokError(Twine("invalid value in ") +
2754                       Twine(PALMD::AssemblerDirective));
2755     }
2756     PALMetadata.push_back(Value);
2757     if (getLexer().isNot(AsmToken::Comma))
2758       break;
2759     Lex();
2760   }
2761   getTargetStreamer().EmitPALMetadata(PALMetadata);
2762   return false;
2763 }
2764 
2765 bool AMDGPUAsmParser::ParseDirective(AsmToken DirectiveID) {
2766   StringRef IDVal = DirectiveID.getString();
2767 
2768   if (IDVal == ".hsa_code_object_version")
2769     return ParseDirectiveHSACodeObjectVersion();
2770 
2771   if (IDVal == ".hsa_code_object_isa")
2772     return ParseDirectiveHSACodeObjectISA();
2773 
2774   if (IDVal == ".amd_kernel_code_t")
2775     return ParseDirectiveAMDKernelCodeT();
2776 
2777   if (IDVal == ".amdgpu_hsa_kernel")
2778     return ParseDirectiveAMDGPUHsaKernel();
2779 
2780   if (IDVal == ".amd_amdgpu_isa")
2781     return ParseDirectiveISAVersion();
2782 
2783   if (IDVal == AMDGPU::HSAMD::AssemblerDirectiveBegin)
2784     return ParseDirectiveHSAMetadata();
2785 
2786   if (IDVal == PALMD::AssemblerDirective)
2787     return ParseDirectivePALMetadata();
2788 
2789   return true;
2790 }
2791 
2792 bool AMDGPUAsmParser::subtargetHasRegister(const MCRegisterInfo &MRI,
2793                                            unsigned RegNo) const {
2794 
2795   for (MCRegAliasIterator R(AMDGPU::TTMP12_TTMP13_TTMP14_TTMP15, &MRI, true);
2796        R.isValid(); ++R) {
2797     if (*R == RegNo)
2798       return isGFX9();
2799   }
2800 
2801   switch (RegNo) {
2802   case AMDGPU::TBA:
2803   case AMDGPU::TBA_LO:
2804   case AMDGPU::TBA_HI:
2805   case AMDGPU::TMA:
2806   case AMDGPU::TMA_LO:
2807   case AMDGPU::TMA_HI:
2808     return !isGFX9();
2809   case AMDGPU::XNACK_MASK:
2810   case AMDGPU::XNACK_MASK_LO:
2811   case AMDGPU::XNACK_MASK_HI:
2812     return !isCI() && !isSI() && hasXNACK();
2813   default:
2814     break;
2815   }
2816 
2817   if (isCI())
2818     return true;
2819 
2820   if (isSI()) {
2821     // No flat_scr
2822     switch (RegNo) {
2823     case AMDGPU::FLAT_SCR:
2824     case AMDGPU::FLAT_SCR_LO:
2825     case AMDGPU::FLAT_SCR_HI:
2826       return false;
2827     default:
2828       return true;
2829     }
2830   }
2831 
2832   // VI only has 102 SGPRs, so make sure we aren't trying to use the 2 more that
2833   // SI/CI have.
2834   for (MCRegAliasIterator R(AMDGPU::SGPR102_SGPR103, &MRI, true);
2835        R.isValid(); ++R) {
2836     if (*R == RegNo)
2837       return false;
2838   }
2839 
2840   return true;
2841 }
2842 
2843 OperandMatchResultTy
2844 AMDGPUAsmParser::parseOperand(OperandVector &Operands, StringRef Mnemonic) {
2845   // Try to parse with a custom parser
2846   OperandMatchResultTy ResTy = MatchOperandParserImpl(Operands, Mnemonic);
2847 
2848   // If we successfully parsed the operand or if there as an error parsing,
2849   // we are done.
2850   //
2851   // If we are parsing after we reach EndOfStatement then this means we
2852   // are appending default values to the Operands list.  This is only done
2853   // by custom parser, so we shouldn't continue on to the generic parsing.
2854   if (ResTy == MatchOperand_Success || ResTy == MatchOperand_ParseFail ||
2855       getLexer().is(AsmToken::EndOfStatement))
2856     return ResTy;
2857 
2858   ResTy = parseRegOrImm(Operands);
2859 
2860   if (ResTy == MatchOperand_Success)
2861     return ResTy;
2862 
2863   const auto &Tok = Parser.getTok();
2864   SMLoc S = Tok.getLoc();
2865 
2866   const MCExpr *Expr = nullptr;
2867   if (!Parser.parseExpression(Expr)) {
2868     Operands.push_back(AMDGPUOperand::CreateExpr(this, Expr, S));
2869     return MatchOperand_Success;
2870   }
2871 
2872   // Possibly this is an instruction flag like 'gds'.
2873   if (Tok.getKind() == AsmToken::Identifier) {
2874     Operands.push_back(AMDGPUOperand::CreateToken(this, Tok.getString(), S));
2875     Parser.Lex();
2876     return MatchOperand_Success;
2877   }
2878 
2879   return MatchOperand_NoMatch;
2880 }
2881 
2882 StringRef AMDGPUAsmParser::parseMnemonicSuffix(StringRef Name) {
2883   // Clear any forced encodings from the previous instruction.
2884   setForcedEncodingSize(0);
2885   setForcedDPP(false);
2886   setForcedSDWA(false);
2887 
2888   if (Name.endswith("_e64")) {
2889     setForcedEncodingSize(64);
2890     return Name.substr(0, Name.size() - 4);
2891   } else if (Name.endswith("_e32")) {
2892     setForcedEncodingSize(32);
2893     return Name.substr(0, Name.size() - 4);
2894   } else if (Name.endswith("_dpp")) {
2895     setForcedDPP(true);
2896     return Name.substr(0, Name.size() - 4);
2897   } else if (Name.endswith("_sdwa")) {
2898     setForcedSDWA(true);
2899     return Name.substr(0, Name.size() - 5);
2900   }
2901   return Name;
2902 }
2903 
2904 bool AMDGPUAsmParser::ParseInstruction(ParseInstructionInfo &Info,
2905                                        StringRef Name,
2906                                        SMLoc NameLoc, OperandVector &Operands) {
2907   // Add the instruction mnemonic
2908   Name = parseMnemonicSuffix(Name);
2909   Operands.push_back(AMDGPUOperand::CreateToken(this, Name, NameLoc));
2910 
2911   while (!getLexer().is(AsmToken::EndOfStatement)) {
2912     OperandMatchResultTy Res = parseOperand(Operands, Name);
2913 
2914     // Eat the comma or space if there is one.
2915     if (getLexer().is(AsmToken::Comma))
2916       Parser.Lex();
2917 
2918     switch (Res) {
2919       case MatchOperand_Success: break;
2920       case MatchOperand_ParseFail:
2921         Error(getLexer().getLoc(), "failed parsing operand.");
2922         while (!getLexer().is(AsmToken::EndOfStatement)) {
2923           Parser.Lex();
2924         }
2925         return true;
2926       case MatchOperand_NoMatch:
2927         Error(getLexer().getLoc(), "not a valid operand.");
2928         while (!getLexer().is(AsmToken::EndOfStatement)) {
2929           Parser.Lex();
2930         }
2931         return true;
2932     }
2933   }
2934 
2935   return false;
2936 }
2937 
2938 //===----------------------------------------------------------------------===//
2939 // Utility functions
2940 //===----------------------------------------------------------------------===//
2941 
2942 OperandMatchResultTy
2943 AMDGPUAsmParser::parseIntWithPrefix(const char *Prefix, int64_t &Int) {
2944   switch(getLexer().getKind()) {
2945     default: return MatchOperand_NoMatch;
2946     case AsmToken::Identifier: {
2947       StringRef Name = Parser.getTok().getString();
2948       if (!Name.equals(Prefix)) {
2949         return MatchOperand_NoMatch;
2950       }
2951 
2952       Parser.Lex();
2953       if (getLexer().isNot(AsmToken::Colon))
2954         return MatchOperand_ParseFail;
2955 
2956       Parser.Lex();
2957 
2958       bool IsMinus = false;
2959       if (getLexer().getKind() == AsmToken::Minus) {
2960         Parser.Lex();
2961         IsMinus = true;
2962       }
2963 
2964       if (getLexer().isNot(AsmToken::Integer))
2965         return MatchOperand_ParseFail;
2966 
2967       if (getParser().parseAbsoluteExpression(Int))
2968         return MatchOperand_ParseFail;
2969 
2970       if (IsMinus)
2971         Int = -Int;
2972       break;
2973     }
2974   }
2975   return MatchOperand_Success;
2976 }
2977 
2978 OperandMatchResultTy
2979 AMDGPUAsmParser::parseIntWithPrefix(const char *Prefix, OperandVector &Operands,
2980                                     AMDGPUOperand::ImmTy ImmTy,
2981                                     bool (*ConvertResult)(int64_t&)) {
2982   SMLoc S = Parser.getTok().getLoc();
2983   int64_t Value = 0;
2984 
2985   OperandMatchResultTy Res = parseIntWithPrefix(Prefix, Value);
2986   if (Res != MatchOperand_Success)
2987     return Res;
2988 
2989   if (ConvertResult && !ConvertResult(Value)) {
2990     return MatchOperand_ParseFail;
2991   }
2992 
2993   Operands.push_back(AMDGPUOperand::CreateImm(this, Value, S, ImmTy));
2994   return MatchOperand_Success;
2995 }
2996 
2997 OperandMatchResultTy AMDGPUAsmParser::parseOperandArrayWithPrefix(
2998   const char *Prefix,
2999   OperandVector &Operands,
3000   AMDGPUOperand::ImmTy ImmTy,
3001   bool (*ConvertResult)(int64_t&)) {
3002   StringRef Name = Parser.getTok().getString();
3003   if (!Name.equals(Prefix))
3004     return MatchOperand_NoMatch;
3005 
3006   Parser.Lex();
3007   if (getLexer().isNot(AsmToken::Colon))
3008     return MatchOperand_ParseFail;
3009 
3010   Parser.Lex();
3011   if (getLexer().isNot(AsmToken::LBrac))
3012     return MatchOperand_ParseFail;
3013   Parser.Lex();
3014 
3015   unsigned Val = 0;
3016   SMLoc S = Parser.getTok().getLoc();
3017 
3018   // FIXME: How to verify the number of elements matches the number of src
3019   // operands?
3020   for (int I = 0; I < 4; ++I) {
3021     if (I != 0) {
3022       if (getLexer().is(AsmToken::RBrac))
3023         break;
3024 
3025       if (getLexer().isNot(AsmToken::Comma))
3026         return MatchOperand_ParseFail;
3027       Parser.Lex();
3028     }
3029 
3030     if (getLexer().isNot(AsmToken::Integer))
3031       return MatchOperand_ParseFail;
3032 
3033     int64_t Op;
3034     if (getParser().parseAbsoluteExpression(Op))
3035       return MatchOperand_ParseFail;
3036 
3037     if (Op != 0 && Op != 1)
3038       return MatchOperand_ParseFail;
3039     Val |= (Op << I);
3040   }
3041 
3042   Parser.Lex();
3043   Operands.push_back(AMDGPUOperand::CreateImm(this, Val, S, ImmTy));
3044   return MatchOperand_Success;
3045 }
3046 
3047 OperandMatchResultTy
3048 AMDGPUAsmParser::parseNamedBit(const char *Name, OperandVector &Operands,
3049                                AMDGPUOperand::ImmTy ImmTy) {
3050   int64_t Bit = 0;
3051   SMLoc S = Parser.getTok().getLoc();
3052 
3053   // We are at the end of the statement, and this is a default argument, so
3054   // use a default value.
3055   if (getLexer().isNot(AsmToken::EndOfStatement)) {
3056     switch(getLexer().getKind()) {
3057       case AsmToken::Identifier: {
3058         StringRef Tok = Parser.getTok().getString();
3059         if (Tok == Name) {
3060           Bit = 1;
3061           Parser.Lex();
3062         } else if (Tok.startswith("no") && Tok.endswith(Name)) {
3063           Bit = 0;
3064           Parser.Lex();
3065         } else {
3066           return MatchOperand_NoMatch;
3067         }
3068         break;
3069       }
3070       default:
3071         return MatchOperand_NoMatch;
3072     }
3073   }
3074 
3075   Operands.push_back(AMDGPUOperand::CreateImm(this, Bit, S, ImmTy));
3076   return MatchOperand_Success;
3077 }
3078 
3079 static void addOptionalImmOperand(
3080   MCInst& Inst, const OperandVector& Operands,
3081   AMDGPUAsmParser::OptionalImmIndexMap& OptionalIdx,
3082   AMDGPUOperand::ImmTy ImmT,
3083   int64_t Default = 0) {
3084   auto i = OptionalIdx.find(ImmT);
3085   if (i != OptionalIdx.end()) {
3086     unsigned Idx = i->second;
3087     ((AMDGPUOperand &)*Operands[Idx]).addImmOperands(Inst, 1);
3088   } else {
3089     Inst.addOperand(MCOperand::createImm(Default));
3090   }
3091 }
3092 
3093 OperandMatchResultTy
3094 AMDGPUAsmParser::parseStringWithPrefix(StringRef Prefix, StringRef &Value) {
3095   if (getLexer().isNot(AsmToken::Identifier)) {
3096     return MatchOperand_NoMatch;
3097   }
3098   StringRef Tok = Parser.getTok().getString();
3099   if (Tok != Prefix) {
3100     return MatchOperand_NoMatch;
3101   }
3102 
3103   Parser.Lex();
3104   if (getLexer().isNot(AsmToken::Colon)) {
3105     return MatchOperand_ParseFail;
3106   }
3107 
3108   Parser.Lex();
3109   if (getLexer().isNot(AsmToken::Identifier)) {
3110     return MatchOperand_ParseFail;
3111   }
3112 
3113   Value = Parser.getTok().getString();
3114   return MatchOperand_Success;
3115 }
3116 
3117 //===----------------------------------------------------------------------===//
3118 // ds
3119 //===----------------------------------------------------------------------===//
3120 
3121 void AMDGPUAsmParser::cvtDSOffset01(MCInst &Inst,
3122                                     const OperandVector &Operands) {
3123   OptionalImmIndexMap OptionalIdx;
3124 
3125   for (unsigned i = 1, e = Operands.size(); i != e; ++i) {
3126     AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[i]);
3127 
3128     // Add the register arguments
3129     if (Op.isReg()) {
3130       Op.addRegOperands(Inst, 1);
3131       continue;
3132     }
3133 
3134     // Handle optional arguments
3135     OptionalIdx[Op.getImmTy()] = i;
3136   }
3137 
3138   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyOffset0);
3139   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyOffset1);
3140   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyGDS);
3141 
3142   Inst.addOperand(MCOperand::createReg(AMDGPU::M0)); // m0
3143 }
3144 
3145 void AMDGPUAsmParser::cvtDSImpl(MCInst &Inst, const OperandVector &Operands,
3146                                 bool IsGdsHardcoded) {
3147   OptionalImmIndexMap OptionalIdx;
3148 
3149   for (unsigned i = 1, e = Operands.size(); i != e; ++i) {
3150     AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[i]);
3151 
3152     // Add the register arguments
3153     if (Op.isReg()) {
3154       Op.addRegOperands(Inst, 1);
3155       continue;
3156     }
3157 
3158     if (Op.isToken() && Op.getToken() == "gds") {
3159       IsGdsHardcoded = true;
3160       continue;
3161     }
3162 
3163     // Handle optional arguments
3164     OptionalIdx[Op.getImmTy()] = i;
3165   }
3166 
3167   AMDGPUOperand::ImmTy OffsetType =
3168     (Inst.getOpcode() == AMDGPU::DS_SWIZZLE_B32_si ||
3169      Inst.getOpcode() == AMDGPU::DS_SWIZZLE_B32_vi) ? AMDGPUOperand::ImmTySwizzle :
3170                                                       AMDGPUOperand::ImmTyOffset;
3171 
3172   addOptionalImmOperand(Inst, Operands, OptionalIdx, OffsetType);
3173 
3174   if (!IsGdsHardcoded) {
3175     addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyGDS);
3176   }
3177   Inst.addOperand(MCOperand::createReg(AMDGPU::M0)); // m0
3178 }
3179 
3180 void AMDGPUAsmParser::cvtExp(MCInst &Inst, const OperandVector &Operands) {
3181   OptionalImmIndexMap OptionalIdx;
3182 
3183   unsigned OperandIdx[4];
3184   unsigned EnMask = 0;
3185   int SrcIdx = 0;
3186 
3187   for (unsigned i = 1, e = Operands.size(); i != e; ++i) {
3188     AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[i]);
3189 
3190     // Add the register arguments
3191     if (Op.isReg()) {
3192       assert(SrcIdx < 4);
3193       OperandIdx[SrcIdx] = Inst.size();
3194       Op.addRegOperands(Inst, 1);
3195       ++SrcIdx;
3196       continue;
3197     }
3198 
3199     if (Op.isOff()) {
3200       assert(SrcIdx < 4);
3201       OperandIdx[SrcIdx] = Inst.size();
3202       Inst.addOperand(MCOperand::createReg(AMDGPU::NoRegister));
3203       ++SrcIdx;
3204       continue;
3205     }
3206 
3207     if (Op.isImm() && Op.getImmTy() == AMDGPUOperand::ImmTyExpTgt) {
3208       Op.addImmOperands(Inst, 1);
3209       continue;
3210     }
3211 
3212     if (Op.isToken() && Op.getToken() == "done")
3213       continue;
3214 
3215     // Handle optional arguments
3216     OptionalIdx[Op.getImmTy()] = i;
3217   }
3218 
3219   assert(SrcIdx == 4);
3220 
3221   bool Compr = false;
3222   if (OptionalIdx.find(AMDGPUOperand::ImmTyExpCompr) != OptionalIdx.end()) {
3223     Compr = true;
3224     Inst.getOperand(OperandIdx[1]) = Inst.getOperand(OperandIdx[2]);
3225     Inst.getOperand(OperandIdx[2]).setReg(AMDGPU::NoRegister);
3226     Inst.getOperand(OperandIdx[3]).setReg(AMDGPU::NoRegister);
3227   }
3228 
3229   for (auto i = 0; i < SrcIdx; ++i) {
3230     if (Inst.getOperand(OperandIdx[i]).getReg() != AMDGPU::NoRegister) {
3231       EnMask |= Compr? (0x3 << i * 2) : (0x1 << i);
3232     }
3233   }
3234 
3235   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyExpVM);
3236   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyExpCompr);
3237 
3238   Inst.addOperand(MCOperand::createImm(EnMask));
3239 }
3240 
3241 //===----------------------------------------------------------------------===//
3242 // s_waitcnt
3243 //===----------------------------------------------------------------------===//
3244 
3245 static bool
3246 encodeCnt(
3247   const AMDGPU::IsaInfo::IsaVersion ISA,
3248   int64_t &IntVal,
3249   int64_t CntVal,
3250   bool Saturate,
3251   unsigned (*encode)(const IsaInfo::IsaVersion &Version, unsigned, unsigned),
3252   unsigned (*decode)(const IsaInfo::IsaVersion &Version, unsigned))
3253 {
3254   bool Failed = false;
3255 
3256   IntVal = encode(ISA, IntVal, CntVal);
3257   if (CntVal != decode(ISA, IntVal)) {
3258     if (Saturate) {
3259       IntVal = encode(ISA, IntVal, -1);
3260     } else {
3261       Failed = true;
3262     }
3263   }
3264   return Failed;
3265 }
3266 
3267 bool AMDGPUAsmParser::parseCnt(int64_t &IntVal) {
3268   StringRef CntName = Parser.getTok().getString();
3269   int64_t CntVal;
3270 
3271   Parser.Lex();
3272   if (getLexer().isNot(AsmToken::LParen))
3273     return true;
3274 
3275   Parser.Lex();
3276   if (getLexer().isNot(AsmToken::Integer))
3277     return true;
3278 
3279   SMLoc ValLoc = Parser.getTok().getLoc();
3280   if (getParser().parseAbsoluteExpression(CntVal))
3281     return true;
3282 
3283   AMDGPU::IsaInfo::IsaVersion ISA =
3284       AMDGPU::IsaInfo::getIsaVersion(getFeatureBits());
3285 
3286   bool Failed = true;
3287   bool Sat = CntName.endswith("_sat");
3288 
3289   if (CntName == "vmcnt" || CntName == "vmcnt_sat") {
3290     Failed = encodeCnt(ISA, IntVal, CntVal, Sat, encodeVmcnt, decodeVmcnt);
3291   } else if (CntName == "expcnt" || CntName == "expcnt_sat") {
3292     Failed = encodeCnt(ISA, IntVal, CntVal, Sat, encodeExpcnt, decodeExpcnt);
3293   } else if (CntName == "lgkmcnt" || CntName == "lgkmcnt_sat") {
3294     Failed = encodeCnt(ISA, IntVal, CntVal, Sat, encodeLgkmcnt, decodeLgkmcnt);
3295   }
3296 
3297   if (Failed) {
3298     Error(ValLoc, "too large value for " + CntName);
3299     return true;
3300   }
3301 
3302   if (getLexer().isNot(AsmToken::RParen)) {
3303     return true;
3304   }
3305 
3306   Parser.Lex();
3307   if (getLexer().is(AsmToken::Amp) || getLexer().is(AsmToken::Comma)) {
3308     const AsmToken NextToken = getLexer().peekTok();
3309     if (NextToken.is(AsmToken::Identifier)) {
3310       Parser.Lex();
3311     }
3312   }
3313 
3314   return false;
3315 }
3316 
3317 OperandMatchResultTy
3318 AMDGPUAsmParser::parseSWaitCntOps(OperandVector &Operands) {
3319   AMDGPU::IsaInfo::IsaVersion ISA =
3320       AMDGPU::IsaInfo::getIsaVersion(getFeatureBits());
3321   int64_t Waitcnt = getWaitcntBitMask(ISA);
3322   SMLoc S = Parser.getTok().getLoc();
3323 
3324   switch(getLexer().getKind()) {
3325     default: return MatchOperand_ParseFail;
3326     case AsmToken::Integer:
3327       // The operand can be an integer value.
3328       if (getParser().parseAbsoluteExpression(Waitcnt))
3329         return MatchOperand_ParseFail;
3330       break;
3331 
3332     case AsmToken::Identifier:
3333       do {
3334         if (parseCnt(Waitcnt))
3335           return MatchOperand_ParseFail;
3336       } while(getLexer().isNot(AsmToken::EndOfStatement));
3337       break;
3338   }
3339   Operands.push_back(AMDGPUOperand::CreateImm(this, Waitcnt, S));
3340   return MatchOperand_Success;
3341 }
3342 
3343 bool AMDGPUAsmParser::parseHwregConstruct(OperandInfoTy &HwReg, int64_t &Offset,
3344                                           int64_t &Width) {
3345   using namespace llvm::AMDGPU::Hwreg;
3346 
3347   if (Parser.getTok().getString() != "hwreg")
3348     return true;
3349   Parser.Lex();
3350 
3351   if (getLexer().isNot(AsmToken::LParen))
3352     return true;
3353   Parser.Lex();
3354 
3355   if (getLexer().is(AsmToken::Identifier)) {
3356     HwReg.IsSymbolic = true;
3357     HwReg.Id = ID_UNKNOWN_;
3358     const StringRef tok = Parser.getTok().getString();
3359     int Last = ID_SYMBOLIC_LAST_;
3360     if (isSI() || isCI() || isVI())
3361       Last = ID_SYMBOLIC_FIRST_GFX9_;
3362     for (int i = ID_SYMBOLIC_FIRST_; i < Last; ++i) {
3363       if (tok == IdSymbolic[i]) {
3364         HwReg.Id = i;
3365         break;
3366       }
3367     }
3368     Parser.Lex();
3369   } else {
3370     HwReg.IsSymbolic = false;
3371     if (getLexer().isNot(AsmToken::Integer))
3372       return true;
3373     if (getParser().parseAbsoluteExpression(HwReg.Id))
3374       return true;
3375   }
3376 
3377   if (getLexer().is(AsmToken::RParen)) {
3378     Parser.Lex();
3379     return false;
3380   }
3381 
3382   // optional params
3383   if (getLexer().isNot(AsmToken::Comma))
3384     return true;
3385   Parser.Lex();
3386 
3387   if (getLexer().isNot(AsmToken::Integer))
3388     return true;
3389   if (getParser().parseAbsoluteExpression(Offset))
3390     return true;
3391 
3392   if (getLexer().isNot(AsmToken::Comma))
3393     return true;
3394   Parser.Lex();
3395 
3396   if (getLexer().isNot(AsmToken::Integer))
3397     return true;
3398   if (getParser().parseAbsoluteExpression(Width))
3399     return true;
3400 
3401   if (getLexer().isNot(AsmToken::RParen))
3402     return true;
3403   Parser.Lex();
3404 
3405   return false;
3406 }
3407 
3408 OperandMatchResultTy AMDGPUAsmParser::parseHwreg(OperandVector &Operands) {
3409   using namespace llvm::AMDGPU::Hwreg;
3410 
3411   int64_t Imm16Val = 0;
3412   SMLoc S = Parser.getTok().getLoc();
3413 
3414   switch(getLexer().getKind()) {
3415     default: return MatchOperand_NoMatch;
3416     case AsmToken::Integer:
3417       // The operand can be an integer value.
3418       if (getParser().parseAbsoluteExpression(Imm16Val))
3419         return MatchOperand_NoMatch;
3420       if (Imm16Val < 0 || !isUInt<16>(Imm16Val)) {
3421         Error(S, "invalid immediate: only 16-bit values are legal");
3422         // Do not return error code, but create an imm operand anyway and proceed
3423         // to the next operand, if any. That avoids unneccessary error messages.
3424       }
3425       break;
3426 
3427     case AsmToken::Identifier: {
3428         OperandInfoTy HwReg(ID_UNKNOWN_);
3429         int64_t Offset = OFFSET_DEFAULT_;
3430         int64_t Width = WIDTH_M1_DEFAULT_ + 1;
3431         if (parseHwregConstruct(HwReg, Offset, Width))
3432           return MatchOperand_ParseFail;
3433         if (HwReg.Id < 0 || !isUInt<ID_WIDTH_>(HwReg.Id)) {
3434           if (HwReg.IsSymbolic)
3435             Error(S, "invalid symbolic name of hardware register");
3436           else
3437             Error(S, "invalid code of hardware register: only 6-bit values are legal");
3438         }
3439         if (Offset < 0 || !isUInt<OFFSET_WIDTH_>(Offset))
3440           Error(S, "invalid bit offset: only 5-bit values are legal");
3441         if ((Width-1) < 0 || !isUInt<WIDTH_M1_WIDTH_>(Width-1))
3442           Error(S, "invalid bitfield width: only values from 1 to 32 are legal");
3443         Imm16Val = (HwReg.Id << ID_SHIFT_) | (Offset << OFFSET_SHIFT_) | ((Width-1) << WIDTH_M1_SHIFT_);
3444       }
3445       break;
3446   }
3447   Operands.push_back(AMDGPUOperand::CreateImm(this, Imm16Val, S, AMDGPUOperand::ImmTyHwreg));
3448   return MatchOperand_Success;
3449 }
3450 
3451 bool AMDGPUOperand::isSWaitCnt() const {
3452   return isImm();
3453 }
3454 
3455 bool AMDGPUOperand::isHwreg() const {
3456   return isImmTy(ImmTyHwreg);
3457 }
3458 
3459 bool AMDGPUAsmParser::parseSendMsgConstruct(OperandInfoTy &Msg, OperandInfoTy &Operation, int64_t &StreamId) {
3460   using namespace llvm::AMDGPU::SendMsg;
3461 
3462   if (Parser.getTok().getString() != "sendmsg")
3463     return true;
3464   Parser.Lex();
3465 
3466   if (getLexer().isNot(AsmToken::LParen))
3467     return true;
3468   Parser.Lex();
3469 
3470   if (getLexer().is(AsmToken::Identifier)) {
3471     Msg.IsSymbolic = true;
3472     Msg.Id = ID_UNKNOWN_;
3473     const std::string tok = Parser.getTok().getString();
3474     for (int i = ID_GAPS_FIRST_; i < ID_GAPS_LAST_; ++i) {
3475       switch(i) {
3476         default: continue; // Omit gaps.
3477         case ID_INTERRUPT: case ID_GS: case ID_GS_DONE:  case ID_SYSMSG: break;
3478       }
3479       if (tok == IdSymbolic[i]) {
3480         Msg.Id = i;
3481         break;
3482       }
3483     }
3484     Parser.Lex();
3485   } else {
3486     Msg.IsSymbolic = false;
3487     if (getLexer().isNot(AsmToken::Integer))
3488       return true;
3489     if (getParser().parseAbsoluteExpression(Msg.Id))
3490       return true;
3491     if (getLexer().is(AsmToken::Integer))
3492       if (getParser().parseAbsoluteExpression(Msg.Id))
3493         Msg.Id = ID_UNKNOWN_;
3494   }
3495   if (Msg.Id == ID_UNKNOWN_) // Don't know how to parse the rest.
3496     return false;
3497 
3498   if (!(Msg.Id == ID_GS || Msg.Id == ID_GS_DONE || Msg.Id == ID_SYSMSG)) {
3499     if (getLexer().isNot(AsmToken::RParen))
3500       return true;
3501     Parser.Lex();
3502     return false;
3503   }
3504 
3505   if (getLexer().isNot(AsmToken::Comma))
3506     return true;
3507   Parser.Lex();
3508 
3509   assert(Msg.Id == ID_GS || Msg.Id == ID_GS_DONE || Msg.Id == ID_SYSMSG);
3510   Operation.Id = ID_UNKNOWN_;
3511   if (getLexer().is(AsmToken::Identifier)) {
3512     Operation.IsSymbolic = true;
3513     const char* const *S = (Msg.Id == ID_SYSMSG) ? OpSysSymbolic : OpGsSymbolic;
3514     const int F = (Msg.Id == ID_SYSMSG) ? OP_SYS_FIRST_ : OP_GS_FIRST_;
3515     const int L = (Msg.Id == ID_SYSMSG) ? OP_SYS_LAST_ : OP_GS_LAST_;
3516     const StringRef Tok = Parser.getTok().getString();
3517     for (int i = F; i < L; ++i) {
3518       if (Tok == S[i]) {
3519         Operation.Id = i;
3520         break;
3521       }
3522     }
3523     Parser.Lex();
3524   } else {
3525     Operation.IsSymbolic = false;
3526     if (getLexer().isNot(AsmToken::Integer))
3527       return true;
3528     if (getParser().parseAbsoluteExpression(Operation.Id))
3529       return true;
3530   }
3531 
3532   if ((Msg.Id == ID_GS || Msg.Id == ID_GS_DONE) && Operation.Id != OP_GS_NOP) {
3533     // Stream id is optional.
3534     if (getLexer().is(AsmToken::RParen)) {
3535       Parser.Lex();
3536       return false;
3537     }
3538 
3539     if (getLexer().isNot(AsmToken::Comma))
3540       return true;
3541     Parser.Lex();
3542 
3543     if (getLexer().isNot(AsmToken::Integer))
3544       return true;
3545     if (getParser().parseAbsoluteExpression(StreamId))
3546       return true;
3547   }
3548 
3549   if (getLexer().isNot(AsmToken::RParen))
3550     return true;
3551   Parser.Lex();
3552   return false;
3553 }
3554 
3555 OperandMatchResultTy AMDGPUAsmParser::parseInterpSlot(OperandVector &Operands) {
3556   if (getLexer().getKind() != AsmToken::Identifier)
3557     return MatchOperand_NoMatch;
3558 
3559   StringRef Str = Parser.getTok().getString();
3560   int Slot = StringSwitch<int>(Str)
3561     .Case("p10", 0)
3562     .Case("p20", 1)
3563     .Case("p0", 2)
3564     .Default(-1);
3565 
3566   SMLoc S = Parser.getTok().getLoc();
3567   if (Slot == -1)
3568     return MatchOperand_ParseFail;
3569 
3570   Parser.Lex();
3571   Operands.push_back(AMDGPUOperand::CreateImm(this, Slot, S,
3572                                               AMDGPUOperand::ImmTyInterpSlot));
3573   return MatchOperand_Success;
3574 }
3575 
3576 OperandMatchResultTy AMDGPUAsmParser::parseInterpAttr(OperandVector &Operands) {
3577   if (getLexer().getKind() != AsmToken::Identifier)
3578     return MatchOperand_NoMatch;
3579 
3580   StringRef Str = Parser.getTok().getString();
3581   if (!Str.startswith("attr"))
3582     return MatchOperand_NoMatch;
3583 
3584   StringRef Chan = Str.take_back(2);
3585   int AttrChan = StringSwitch<int>(Chan)
3586     .Case(".x", 0)
3587     .Case(".y", 1)
3588     .Case(".z", 2)
3589     .Case(".w", 3)
3590     .Default(-1);
3591   if (AttrChan == -1)
3592     return MatchOperand_ParseFail;
3593 
3594   Str = Str.drop_back(2).drop_front(4);
3595 
3596   uint8_t Attr;
3597   if (Str.getAsInteger(10, Attr))
3598     return MatchOperand_ParseFail;
3599 
3600   SMLoc S = Parser.getTok().getLoc();
3601   Parser.Lex();
3602   if (Attr > 63) {
3603     Error(S, "out of bounds attr");
3604     return MatchOperand_Success;
3605   }
3606 
3607   SMLoc SChan = SMLoc::getFromPointer(Chan.data());
3608 
3609   Operands.push_back(AMDGPUOperand::CreateImm(this, Attr, S,
3610                                               AMDGPUOperand::ImmTyInterpAttr));
3611   Operands.push_back(AMDGPUOperand::CreateImm(this, AttrChan, SChan,
3612                                               AMDGPUOperand::ImmTyAttrChan));
3613   return MatchOperand_Success;
3614 }
3615 
3616 void AMDGPUAsmParser::errorExpTgt() {
3617   Error(Parser.getTok().getLoc(), "invalid exp target");
3618 }
3619 
3620 OperandMatchResultTy AMDGPUAsmParser::parseExpTgtImpl(StringRef Str,
3621                                                       uint8_t &Val) {
3622   if (Str == "null") {
3623     Val = 9;
3624     return MatchOperand_Success;
3625   }
3626 
3627   if (Str.startswith("mrt")) {
3628     Str = Str.drop_front(3);
3629     if (Str == "z") { // == mrtz
3630       Val = 8;
3631       return MatchOperand_Success;
3632     }
3633 
3634     if (Str.getAsInteger(10, Val))
3635       return MatchOperand_ParseFail;
3636 
3637     if (Val > 7)
3638       errorExpTgt();
3639 
3640     return MatchOperand_Success;
3641   }
3642 
3643   if (Str.startswith("pos")) {
3644     Str = Str.drop_front(3);
3645     if (Str.getAsInteger(10, Val))
3646       return MatchOperand_ParseFail;
3647 
3648     if (Val > 3)
3649       errorExpTgt();
3650 
3651     Val += 12;
3652     return MatchOperand_Success;
3653   }
3654 
3655   if (Str.startswith("param")) {
3656     Str = Str.drop_front(5);
3657     if (Str.getAsInteger(10, Val))
3658       return MatchOperand_ParseFail;
3659 
3660     if (Val >= 32)
3661       errorExpTgt();
3662 
3663     Val += 32;
3664     return MatchOperand_Success;
3665   }
3666 
3667   if (Str.startswith("invalid_target_")) {
3668     Str = Str.drop_front(15);
3669     if (Str.getAsInteger(10, Val))
3670       return MatchOperand_ParseFail;
3671 
3672     errorExpTgt();
3673     return MatchOperand_Success;
3674   }
3675 
3676   return MatchOperand_NoMatch;
3677 }
3678 
3679 OperandMatchResultTy AMDGPUAsmParser::parseExpTgt(OperandVector &Operands) {
3680   uint8_t Val;
3681   StringRef Str = Parser.getTok().getString();
3682 
3683   auto Res = parseExpTgtImpl(Str, Val);
3684   if (Res != MatchOperand_Success)
3685     return Res;
3686 
3687   SMLoc S = Parser.getTok().getLoc();
3688   Parser.Lex();
3689 
3690   Operands.push_back(AMDGPUOperand::CreateImm(this, Val, S,
3691                                               AMDGPUOperand::ImmTyExpTgt));
3692   return MatchOperand_Success;
3693 }
3694 
3695 OperandMatchResultTy
3696 AMDGPUAsmParser::parseSendMsgOp(OperandVector &Operands) {
3697   using namespace llvm::AMDGPU::SendMsg;
3698 
3699   int64_t Imm16Val = 0;
3700   SMLoc S = Parser.getTok().getLoc();
3701 
3702   switch(getLexer().getKind()) {
3703   default:
3704     return MatchOperand_NoMatch;
3705   case AsmToken::Integer:
3706     // The operand can be an integer value.
3707     if (getParser().parseAbsoluteExpression(Imm16Val))
3708       return MatchOperand_NoMatch;
3709     if (Imm16Val < 0 || !isUInt<16>(Imm16Val)) {
3710       Error(S, "invalid immediate: only 16-bit values are legal");
3711       // Do not return error code, but create an imm operand anyway and proceed
3712       // to the next operand, if any. That avoids unneccessary error messages.
3713     }
3714     break;
3715   case AsmToken::Identifier: {
3716       OperandInfoTy Msg(ID_UNKNOWN_);
3717       OperandInfoTy Operation(OP_UNKNOWN_);
3718       int64_t StreamId = STREAM_ID_DEFAULT_;
3719       if (parseSendMsgConstruct(Msg, Operation, StreamId))
3720         return MatchOperand_ParseFail;
3721       do {
3722         // Validate and encode message ID.
3723         if (! ((ID_INTERRUPT <= Msg.Id && Msg.Id <= ID_GS_DONE)
3724                 || Msg.Id == ID_SYSMSG)) {
3725           if (Msg.IsSymbolic)
3726             Error(S, "invalid/unsupported symbolic name of message");
3727           else
3728             Error(S, "invalid/unsupported code of message");
3729           break;
3730         }
3731         Imm16Val = (Msg.Id << ID_SHIFT_);
3732         // Validate and encode operation ID.
3733         if (Msg.Id == ID_GS || Msg.Id == ID_GS_DONE) {
3734           if (! (OP_GS_FIRST_ <= Operation.Id && Operation.Id < OP_GS_LAST_)) {
3735             if (Operation.IsSymbolic)
3736               Error(S, "invalid symbolic name of GS_OP");
3737             else
3738               Error(S, "invalid code of GS_OP: only 2-bit values are legal");
3739             break;
3740           }
3741           if (Operation.Id == OP_GS_NOP
3742               && Msg.Id != ID_GS_DONE) {
3743             Error(S, "invalid GS_OP: NOP is for GS_DONE only");
3744             break;
3745           }
3746           Imm16Val |= (Operation.Id << OP_SHIFT_);
3747         }
3748         if (Msg.Id == ID_SYSMSG) {
3749           if (! (OP_SYS_FIRST_ <= Operation.Id && Operation.Id < OP_SYS_LAST_)) {
3750             if (Operation.IsSymbolic)
3751               Error(S, "invalid/unsupported symbolic name of SYSMSG_OP");
3752             else
3753               Error(S, "invalid/unsupported code of SYSMSG_OP");
3754             break;
3755           }
3756           Imm16Val |= (Operation.Id << OP_SHIFT_);
3757         }
3758         // Validate and encode stream ID.
3759         if ((Msg.Id == ID_GS || Msg.Id == ID_GS_DONE) && Operation.Id != OP_GS_NOP) {
3760           if (! (STREAM_ID_FIRST_ <= StreamId && StreamId < STREAM_ID_LAST_)) {
3761             Error(S, "invalid stream id: only 2-bit values are legal");
3762             break;
3763           }
3764           Imm16Val |= (StreamId << STREAM_ID_SHIFT_);
3765         }
3766       } while (false);
3767     }
3768     break;
3769   }
3770   Operands.push_back(AMDGPUOperand::CreateImm(this, Imm16Val, S, AMDGPUOperand::ImmTySendMsg));
3771   return MatchOperand_Success;
3772 }
3773 
3774 bool AMDGPUOperand::isSendMsg() const {
3775   return isImmTy(ImmTySendMsg);
3776 }
3777 
3778 //===----------------------------------------------------------------------===//
3779 // parser helpers
3780 //===----------------------------------------------------------------------===//
3781 
3782 bool
3783 AMDGPUAsmParser::trySkipId(const StringRef Id) {
3784   if (getLexer().getKind() == AsmToken::Identifier &&
3785       Parser.getTok().getString() == Id) {
3786     Parser.Lex();
3787     return true;
3788   }
3789   return false;
3790 }
3791 
3792 bool
3793 AMDGPUAsmParser::trySkipToken(const AsmToken::TokenKind Kind) {
3794   if (getLexer().getKind() == Kind) {
3795     Parser.Lex();
3796     return true;
3797   }
3798   return false;
3799 }
3800 
3801 bool
3802 AMDGPUAsmParser::skipToken(const AsmToken::TokenKind Kind,
3803                            const StringRef ErrMsg) {
3804   if (!trySkipToken(Kind)) {
3805     Error(Parser.getTok().getLoc(), ErrMsg);
3806     return false;
3807   }
3808   return true;
3809 }
3810 
3811 bool
3812 AMDGPUAsmParser::parseExpr(int64_t &Imm) {
3813   return !getParser().parseAbsoluteExpression(Imm);
3814 }
3815 
3816 bool
3817 AMDGPUAsmParser::parseString(StringRef &Val, const StringRef ErrMsg) {
3818   SMLoc S = Parser.getTok().getLoc();
3819   if (getLexer().getKind() == AsmToken::String) {
3820     Val = Parser.getTok().getStringContents();
3821     Parser.Lex();
3822     return true;
3823   } else {
3824     Error(S, ErrMsg);
3825     return false;
3826   }
3827 }
3828 
3829 //===----------------------------------------------------------------------===//
3830 // swizzle
3831 //===----------------------------------------------------------------------===//
3832 
3833 LLVM_READNONE
3834 static unsigned
3835 encodeBitmaskPerm(const unsigned AndMask,
3836                   const unsigned OrMask,
3837                   const unsigned XorMask) {
3838   using namespace llvm::AMDGPU::Swizzle;
3839 
3840   return BITMASK_PERM_ENC |
3841          (AndMask << BITMASK_AND_SHIFT) |
3842          (OrMask  << BITMASK_OR_SHIFT)  |
3843          (XorMask << BITMASK_XOR_SHIFT);
3844 }
3845 
3846 bool
3847 AMDGPUAsmParser::parseSwizzleOperands(const unsigned OpNum, int64_t* Op,
3848                                       const unsigned MinVal,
3849                                       const unsigned MaxVal,
3850                                       const StringRef ErrMsg) {
3851   for (unsigned i = 0; i < OpNum; ++i) {
3852     if (!skipToken(AsmToken::Comma, "expected a comma")){
3853       return false;
3854     }
3855     SMLoc ExprLoc = Parser.getTok().getLoc();
3856     if (!parseExpr(Op[i])) {
3857       return false;
3858     }
3859     if (Op[i] < MinVal || Op[i] > MaxVal) {
3860       Error(ExprLoc, ErrMsg);
3861       return false;
3862     }
3863   }
3864 
3865   return true;
3866 }
3867 
3868 bool
3869 AMDGPUAsmParser::parseSwizzleQuadPerm(int64_t &Imm) {
3870   using namespace llvm::AMDGPU::Swizzle;
3871 
3872   int64_t Lane[LANE_NUM];
3873   if (parseSwizzleOperands(LANE_NUM, Lane, 0, LANE_MAX,
3874                            "expected a 2-bit lane id")) {
3875     Imm = QUAD_PERM_ENC;
3876     for (auto i = 0; i < LANE_NUM; ++i) {
3877       Imm |= Lane[i] << (LANE_SHIFT * i);
3878     }
3879     return true;
3880   }
3881   return false;
3882 }
3883 
3884 bool
3885 AMDGPUAsmParser::parseSwizzleBroadcast(int64_t &Imm) {
3886   using namespace llvm::AMDGPU::Swizzle;
3887 
3888   SMLoc S = Parser.getTok().getLoc();
3889   int64_t GroupSize;
3890   int64_t LaneIdx;
3891 
3892   if (!parseSwizzleOperands(1, &GroupSize,
3893                             2, 32,
3894                             "group size must be in the interval [2,32]")) {
3895     return false;
3896   }
3897   if (!isPowerOf2_64(GroupSize)) {
3898     Error(S, "group size must be a power of two");
3899     return false;
3900   }
3901   if (parseSwizzleOperands(1, &LaneIdx,
3902                            0, GroupSize - 1,
3903                            "lane id must be in the interval [0,group size - 1]")) {
3904     Imm = encodeBitmaskPerm(BITMASK_MAX - GroupSize + 1, LaneIdx, 0);
3905     return true;
3906   }
3907   return false;
3908 }
3909 
3910 bool
3911 AMDGPUAsmParser::parseSwizzleReverse(int64_t &Imm) {
3912   using namespace llvm::AMDGPU::Swizzle;
3913 
3914   SMLoc S = Parser.getTok().getLoc();
3915   int64_t GroupSize;
3916 
3917   if (!parseSwizzleOperands(1, &GroupSize,
3918       2, 32, "group size must be in the interval [2,32]")) {
3919     return false;
3920   }
3921   if (!isPowerOf2_64(GroupSize)) {
3922     Error(S, "group size must be a power of two");
3923     return false;
3924   }
3925 
3926   Imm = encodeBitmaskPerm(BITMASK_MAX, 0, GroupSize - 1);
3927   return true;
3928 }
3929 
3930 bool
3931 AMDGPUAsmParser::parseSwizzleSwap(int64_t &Imm) {
3932   using namespace llvm::AMDGPU::Swizzle;
3933 
3934   SMLoc S = Parser.getTok().getLoc();
3935   int64_t GroupSize;
3936 
3937   if (!parseSwizzleOperands(1, &GroupSize,
3938       1, 16, "group size must be in the interval [1,16]")) {
3939     return false;
3940   }
3941   if (!isPowerOf2_64(GroupSize)) {
3942     Error(S, "group size must be a power of two");
3943     return false;
3944   }
3945 
3946   Imm = encodeBitmaskPerm(BITMASK_MAX, 0, GroupSize);
3947   return true;
3948 }
3949 
3950 bool
3951 AMDGPUAsmParser::parseSwizzleBitmaskPerm(int64_t &Imm) {
3952   using namespace llvm::AMDGPU::Swizzle;
3953 
3954   if (!skipToken(AsmToken::Comma, "expected a comma")) {
3955     return false;
3956   }
3957 
3958   StringRef Ctl;
3959   SMLoc StrLoc = Parser.getTok().getLoc();
3960   if (!parseString(Ctl)) {
3961     return false;
3962   }
3963   if (Ctl.size() != BITMASK_WIDTH) {
3964     Error(StrLoc, "expected a 5-character mask");
3965     return false;
3966   }
3967 
3968   unsigned AndMask = 0;
3969   unsigned OrMask = 0;
3970   unsigned XorMask = 0;
3971 
3972   for (size_t i = 0; i < Ctl.size(); ++i) {
3973     unsigned Mask = 1 << (BITMASK_WIDTH - 1 - i);
3974     switch(Ctl[i]) {
3975     default:
3976       Error(StrLoc, "invalid mask");
3977       return false;
3978     case '0':
3979       break;
3980     case '1':
3981       OrMask |= Mask;
3982       break;
3983     case 'p':
3984       AndMask |= Mask;
3985       break;
3986     case 'i':
3987       AndMask |= Mask;
3988       XorMask |= Mask;
3989       break;
3990     }
3991   }
3992 
3993   Imm = encodeBitmaskPerm(AndMask, OrMask, XorMask);
3994   return true;
3995 }
3996 
3997 bool
3998 AMDGPUAsmParser::parseSwizzleOffset(int64_t &Imm) {
3999 
4000   SMLoc OffsetLoc = Parser.getTok().getLoc();
4001 
4002   if (!parseExpr(Imm)) {
4003     return false;
4004   }
4005   if (!isUInt<16>(Imm)) {
4006     Error(OffsetLoc, "expected a 16-bit offset");
4007     return false;
4008   }
4009   return true;
4010 }
4011 
4012 bool
4013 AMDGPUAsmParser::parseSwizzleMacro(int64_t &Imm) {
4014   using namespace llvm::AMDGPU::Swizzle;
4015 
4016   if (skipToken(AsmToken::LParen, "expected a left parentheses")) {
4017 
4018     SMLoc ModeLoc = Parser.getTok().getLoc();
4019     bool Ok = false;
4020 
4021     if (trySkipId(IdSymbolic[ID_QUAD_PERM])) {
4022       Ok = parseSwizzleQuadPerm(Imm);
4023     } else if (trySkipId(IdSymbolic[ID_BITMASK_PERM])) {
4024       Ok = parseSwizzleBitmaskPerm(Imm);
4025     } else if (trySkipId(IdSymbolic[ID_BROADCAST])) {
4026       Ok = parseSwizzleBroadcast(Imm);
4027     } else if (trySkipId(IdSymbolic[ID_SWAP])) {
4028       Ok = parseSwizzleSwap(Imm);
4029     } else if (trySkipId(IdSymbolic[ID_REVERSE])) {
4030       Ok = parseSwizzleReverse(Imm);
4031     } else {
4032       Error(ModeLoc, "expected a swizzle mode");
4033     }
4034 
4035     return Ok && skipToken(AsmToken::RParen, "expected a closing parentheses");
4036   }
4037 
4038   return false;
4039 }
4040 
4041 OperandMatchResultTy
4042 AMDGPUAsmParser::parseSwizzleOp(OperandVector &Operands) {
4043   SMLoc S = Parser.getTok().getLoc();
4044   int64_t Imm = 0;
4045 
4046   if (trySkipId("offset")) {
4047 
4048     bool Ok = false;
4049     if (skipToken(AsmToken::Colon, "expected a colon")) {
4050       if (trySkipId("swizzle")) {
4051         Ok = parseSwizzleMacro(Imm);
4052       } else {
4053         Ok = parseSwizzleOffset(Imm);
4054       }
4055     }
4056 
4057     Operands.push_back(AMDGPUOperand::CreateImm(this, Imm, S, AMDGPUOperand::ImmTySwizzle));
4058 
4059     return Ok? MatchOperand_Success : MatchOperand_ParseFail;
4060   } else {
4061     // Swizzle "offset" operand is optional.
4062     // If it is omitted, try parsing other optional operands.
4063     return parseOptionalOpr(Operands);
4064   }
4065 }
4066 
4067 bool
4068 AMDGPUOperand::isSwizzle() const {
4069   return isImmTy(ImmTySwizzle);
4070 }
4071 
4072 //===----------------------------------------------------------------------===//
4073 // sopp branch targets
4074 //===----------------------------------------------------------------------===//
4075 
4076 OperandMatchResultTy
4077 AMDGPUAsmParser::parseSOppBrTarget(OperandVector &Operands) {
4078   SMLoc S = Parser.getTok().getLoc();
4079 
4080   switch (getLexer().getKind()) {
4081     default: return MatchOperand_ParseFail;
4082     case AsmToken::Integer: {
4083       int64_t Imm;
4084       if (getParser().parseAbsoluteExpression(Imm))
4085         return MatchOperand_ParseFail;
4086       Operands.push_back(AMDGPUOperand::CreateImm(this, Imm, S));
4087       return MatchOperand_Success;
4088     }
4089 
4090     case AsmToken::Identifier:
4091       Operands.push_back(AMDGPUOperand::CreateExpr(this,
4092           MCSymbolRefExpr::create(getContext().getOrCreateSymbol(
4093                                   Parser.getTok().getString()), getContext()), S));
4094       Parser.Lex();
4095       return MatchOperand_Success;
4096   }
4097 }
4098 
4099 //===----------------------------------------------------------------------===//
4100 // mubuf
4101 //===----------------------------------------------------------------------===//
4102 
4103 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultGLC() const {
4104   return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTyGLC);
4105 }
4106 
4107 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultSLC() const {
4108   return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTySLC);
4109 }
4110 
4111 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultTFE() const {
4112   return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTyTFE);
4113 }
4114 
4115 void AMDGPUAsmParser::cvtMubufImpl(MCInst &Inst,
4116                                const OperandVector &Operands,
4117                                bool IsAtomic,
4118                                bool IsAtomicReturn,
4119                                bool IsLds) {
4120   bool IsLdsOpcode = IsLds;
4121   bool HasLdsModifier = false;
4122   OptionalImmIndexMap OptionalIdx;
4123   assert(IsAtomicReturn ? IsAtomic : true);
4124 
4125   for (unsigned i = 1, e = Operands.size(); i != e; ++i) {
4126     AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[i]);
4127 
4128     // Add the register arguments
4129     if (Op.isReg()) {
4130       Op.addRegOperands(Inst, 1);
4131       continue;
4132     }
4133 
4134     // Handle the case where soffset is an immediate
4135     if (Op.isImm() && Op.getImmTy() == AMDGPUOperand::ImmTyNone) {
4136       Op.addImmOperands(Inst, 1);
4137       continue;
4138     }
4139 
4140     HasLdsModifier = Op.isLDS();
4141 
4142     // Handle tokens like 'offen' which are sometimes hard-coded into the
4143     // asm string.  There are no MCInst operands for these.
4144     if (Op.isToken()) {
4145       continue;
4146     }
4147     assert(Op.isImm());
4148 
4149     // Handle optional arguments
4150     OptionalIdx[Op.getImmTy()] = i;
4151   }
4152 
4153   // This is a workaround for an llvm quirk which may result in an
4154   // incorrect instruction selection. Lds and non-lds versions of
4155   // MUBUF instructions are identical except that lds versions
4156   // have mandatory 'lds' modifier. However this modifier follows
4157   // optional modifiers and llvm asm matcher regards this 'lds'
4158   // modifier as an optional one. As a result, an lds version
4159   // of opcode may be selected even if it has no 'lds' modifier.
4160   if (IsLdsOpcode && !HasLdsModifier) {
4161     int NoLdsOpcode = AMDGPU::getMUBUFNoLdsInst(Inst.getOpcode());
4162     if (NoLdsOpcode != -1) { // Got lds version - correct it.
4163       Inst.setOpcode(NoLdsOpcode);
4164       IsLdsOpcode = false;
4165     }
4166   }
4167 
4168   // Copy $vdata_in operand and insert as $vdata for MUBUF_Atomic RTN insns.
4169   if (IsAtomicReturn) {
4170     MCInst::iterator I = Inst.begin(); // $vdata_in is always at the beginning.
4171     Inst.insert(I, *I);
4172   }
4173 
4174   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyOffset);
4175   if (!IsAtomic) { // glc is hard-coded.
4176     addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyGLC);
4177   }
4178   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySLC);
4179 
4180   if (!IsLdsOpcode) { // tfe is not legal with lds opcodes
4181     addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyTFE);
4182   }
4183 }
4184 
4185 void AMDGPUAsmParser::cvtMtbuf(MCInst &Inst, const OperandVector &Operands) {
4186   OptionalImmIndexMap OptionalIdx;
4187 
4188   for (unsigned i = 1, e = Operands.size(); i != e; ++i) {
4189     AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[i]);
4190 
4191     // Add the register arguments
4192     if (Op.isReg()) {
4193       Op.addRegOperands(Inst, 1);
4194       continue;
4195     }
4196 
4197     // Handle the case where soffset is an immediate
4198     if (Op.isImm() && Op.getImmTy() == AMDGPUOperand::ImmTyNone) {
4199       Op.addImmOperands(Inst, 1);
4200       continue;
4201     }
4202 
4203     // Handle tokens like 'offen' which are sometimes hard-coded into the
4204     // asm string.  There are no MCInst operands for these.
4205     if (Op.isToken()) {
4206       continue;
4207     }
4208     assert(Op.isImm());
4209 
4210     // Handle optional arguments
4211     OptionalIdx[Op.getImmTy()] = i;
4212   }
4213 
4214   addOptionalImmOperand(Inst, Operands, OptionalIdx,
4215                         AMDGPUOperand::ImmTyOffset);
4216   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyDFMT);
4217   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyNFMT);
4218   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyGLC);
4219   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySLC);
4220   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyTFE);
4221 }
4222 
4223 //===----------------------------------------------------------------------===//
4224 // mimg
4225 //===----------------------------------------------------------------------===//
4226 
4227 void AMDGPUAsmParser::cvtMIMG(MCInst &Inst, const OperandVector &Operands,
4228                               bool IsAtomic) {
4229   unsigned I = 1;
4230   const MCInstrDesc &Desc = MII.get(Inst.getOpcode());
4231   for (unsigned J = 0; J < Desc.getNumDefs(); ++J) {
4232     ((AMDGPUOperand &)*Operands[I++]).addRegOperands(Inst, 1);
4233   }
4234 
4235   if (IsAtomic) {
4236     // Add src, same as dst
4237     assert(Desc.getNumDefs() == 1);
4238     ((AMDGPUOperand &)*Operands[I - 1]).addRegOperands(Inst, 1);
4239   }
4240 
4241   OptionalImmIndexMap OptionalIdx;
4242 
4243   for (unsigned E = Operands.size(); I != E; ++I) {
4244     AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[I]);
4245 
4246     // Add the register arguments
4247     if (Op.isReg()) {
4248       Op.addRegOperands(Inst, 1);
4249     } else if (Op.isImmModifier()) {
4250       OptionalIdx[Op.getImmTy()] = I;
4251     } else {
4252       llvm_unreachable("unexpected operand type");
4253     }
4254   }
4255 
4256   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyDMask);
4257   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyUNorm);
4258   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyGLC);
4259   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySLC);
4260   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyR128);
4261   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyTFE);
4262   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyLWE);
4263   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyDA);
4264 }
4265 
4266 void AMDGPUAsmParser::cvtMIMGAtomic(MCInst &Inst, const OperandVector &Operands) {
4267   cvtMIMG(Inst, Operands, true);
4268 }
4269 
4270 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultDMask() const {
4271   return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTyDMask);
4272 }
4273 
4274 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultUNorm() const {
4275   return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTyUNorm);
4276 }
4277 
4278 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultDA() const {
4279   return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTyDA);
4280 }
4281 
4282 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultR128() const {
4283   return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTyR128);
4284 }
4285 
4286 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultLWE() const {
4287   return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTyLWE);
4288 }
4289 
4290 //===----------------------------------------------------------------------===//
4291 // smrd
4292 //===----------------------------------------------------------------------===//
4293 
4294 bool AMDGPUOperand::isSMRDOffset8() const {
4295   return isImm() && isUInt<8>(getImm());
4296 }
4297 
4298 bool AMDGPUOperand::isSMRDOffset20() const {
4299   return isImm() && isUInt<20>(getImm());
4300 }
4301 
4302 bool AMDGPUOperand::isSMRDLiteralOffset() const {
4303   // 32-bit literals are only supported on CI and we only want to use them
4304   // when the offset is > 8-bits.
4305   return isImm() && !isUInt<8>(getImm()) && isUInt<32>(getImm());
4306 }
4307 
4308 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultSMRDOffset8() const {
4309   return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTyOffset);
4310 }
4311 
4312 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultSMRDOffset20() const {
4313   return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTyOffset);
4314 }
4315 
4316 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultSMRDLiteralOffset() const {
4317   return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTyOffset);
4318 }
4319 
4320 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultOffsetU12() const {
4321   return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTyOffset);
4322 }
4323 
4324 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultOffsetS13() const {
4325   return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTyOffset);
4326 }
4327 
4328 //===----------------------------------------------------------------------===//
4329 // vop3
4330 //===----------------------------------------------------------------------===//
4331 
4332 static bool ConvertOmodMul(int64_t &Mul) {
4333   if (Mul != 1 && Mul != 2 && Mul != 4)
4334     return false;
4335 
4336   Mul >>= 1;
4337   return true;
4338 }
4339 
4340 static bool ConvertOmodDiv(int64_t &Div) {
4341   if (Div == 1) {
4342     Div = 0;
4343     return true;
4344   }
4345 
4346   if (Div == 2) {
4347     Div = 3;
4348     return true;
4349   }
4350 
4351   return false;
4352 }
4353 
4354 static bool ConvertBoundCtrl(int64_t &BoundCtrl) {
4355   if (BoundCtrl == 0) {
4356     BoundCtrl = 1;
4357     return true;
4358   }
4359 
4360   if (BoundCtrl == -1) {
4361     BoundCtrl = 0;
4362     return true;
4363   }
4364 
4365   return false;
4366 }
4367 
4368 // Note: the order in this table matches the order of operands in AsmString.
4369 static const OptionalOperand AMDGPUOptionalOperandTable[] = {
4370   {"offen",   AMDGPUOperand::ImmTyOffen, true, nullptr},
4371   {"idxen",   AMDGPUOperand::ImmTyIdxen, true, nullptr},
4372   {"addr64",  AMDGPUOperand::ImmTyAddr64, true, nullptr},
4373   {"offset0", AMDGPUOperand::ImmTyOffset0, false, nullptr},
4374   {"offset1", AMDGPUOperand::ImmTyOffset1, false, nullptr},
4375   {"gds",     AMDGPUOperand::ImmTyGDS, true, nullptr},
4376   {"lds",     AMDGPUOperand::ImmTyLDS, true, nullptr},
4377   {"offset",  AMDGPUOperand::ImmTyOffset, false, nullptr},
4378   {"inst_offset", AMDGPUOperand::ImmTyInstOffset, false, nullptr},
4379   {"dfmt",    AMDGPUOperand::ImmTyDFMT, false, nullptr},
4380   {"nfmt",    AMDGPUOperand::ImmTyNFMT, false, nullptr},
4381   {"glc",     AMDGPUOperand::ImmTyGLC, true, nullptr},
4382   {"slc",     AMDGPUOperand::ImmTySLC, true, nullptr},
4383   {"tfe",     AMDGPUOperand::ImmTyTFE, true, nullptr},
4384   {"d16",     AMDGPUOperand::ImmTyD16, true, nullptr},
4385   {"high",    AMDGPUOperand::ImmTyHigh, true, nullptr},
4386   {"clamp",   AMDGPUOperand::ImmTyClampSI, true, nullptr},
4387   {"omod",    AMDGPUOperand::ImmTyOModSI, false, ConvertOmodMul},
4388   {"unorm",   AMDGPUOperand::ImmTyUNorm, true, nullptr},
4389   {"da",      AMDGPUOperand::ImmTyDA,    true, nullptr},
4390   {"r128",    AMDGPUOperand::ImmTyR128,  true, nullptr},
4391   {"lwe",     AMDGPUOperand::ImmTyLWE,   true, nullptr},
4392   {"dmask",   AMDGPUOperand::ImmTyDMask, false, nullptr},
4393   {"row_mask",   AMDGPUOperand::ImmTyDppRowMask, false, nullptr},
4394   {"bank_mask",  AMDGPUOperand::ImmTyDppBankMask, false, nullptr},
4395   {"bound_ctrl", AMDGPUOperand::ImmTyDppBoundCtrl, false, ConvertBoundCtrl},
4396   {"dst_sel",    AMDGPUOperand::ImmTySdwaDstSel, false, nullptr},
4397   {"src0_sel",   AMDGPUOperand::ImmTySdwaSrc0Sel, false, nullptr},
4398   {"src1_sel",   AMDGPUOperand::ImmTySdwaSrc1Sel, false, nullptr},
4399   {"dst_unused", AMDGPUOperand::ImmTySdwaDstUnused, false, nullptr},
4400   {"compr", AMDGPUOperand::ImmTyExpCompr, true, nullptr },
4401   {"vm", AMDGPUOperand::ImmTyExpVM, true, nullptr},
4402   {"op_sel", AMDGPUOperand::ImmTyOpSel, false, nullptr},
4403   {"op_sel_hi", AMDGPUOperand::ImmTyOpSelHi, false, nullptr},
4404   {"neg_lo", AMDGPUOperand::ImmTyNegLo, false, nullptr},
4405   {"neg_hi", AMDGPUOperand::ImmTyNegHi, false, nullptr}
4406 };
4407 
4408 OperandMatchResultTy AMDGPUAsmParser::parseOptionalOperand(OperandVector &Operands) {
4409   unsigned size = Operands.size();
4410   assert(size > 0);
4411 
4412   OperandMatchResultTy res = parseOptionalOpr(Operands);
4413 
4414   // This is a hack to enable hardcoded mandatory operands which follow
4415   // optional operands.
4416   //
4417   // Current design assumes that all operands after the first optional operand
4418   // are also optional. However implementation of some instructions violates
4419   // this rule (see e.g. flat/global atomic which have hardcoded 'glc' operands).
4420   //
4421   // To alleviate this problem, we have to (implicitly) parse extra operands
4422   // to make sure autogenerated parser of custom operands never hit hardcoded
4423   // mandatory operands.
4424 
4425   if (size == 1 || ((AMDGPUOperand &)*Operands[size - 1]).isRegKind()) {
4426 
4427     // We have parsed the first optional operand.
4428     // Parse as many operands as necessary to skip all mandatory operands.
4429 
4430     for (unsigned i = 0; i < MAX_OPR_LOOKAHEAD; ++i) {
4431       if (res != MatchOperand_Success ||
4432           getLexer().is(AsmToken::EndOfStatement)) break;
4433       if (getLexer().is(AsmToken::Comma)) Parser.Lex();
4434       res = parseOptionalOpr(Operands);
4435     }
4436   }
4437 
4438   return res;
4439 }
4440 
4441 OperandMatchResultTy AMDGPUAsmParser::parseOptionalOpr(OperandVector &Operands) {
4442   OperandMatchResultTy res;
4443   for (const OptionalOperand &Op : AMDGPUOptionalOperandTable) {
4444     // try to parse any optional operand here
4445     if (Op.IsBit) {
4446       res = parseNamedBit(Op.Name, Operands, Op.Type);
4447     } else if (Op.Type == AMDGPUOperand::ImmTyOModSI) {
4448       res = parseOModOperand(Operands);
4449     } else if (Op.Type == AMDGPUOperand::ImmTySdwaDstSel ||
4450                Op.Type == AMDGPUOperand::ImmTySdwaSrc0Sel ||
4451                Op.Type == AMDGPUOperand::ImmTySdwaSrc1Sel) {
4452       res = parseSDWASel(Operands, Op.Name, Op.Type);
4453     } else if (Op.Type == AMDGPUOperand::ImmTySdwaDstUnused) {
4454       res = parseSDWADstUnused(Operands);
4455     } else if (Op.Type == AMDGPUOperand::ImmTyOpSel ||
4456                Op.Type == AMDGPUOperand::ImmTyOpSelHi ||
4457                Op.Type == AMDGPUOperand::ImmTyNegLo ||
4458                Op.Type == AMDGPUOperand::ImmTyNegHi) {
4459       res = parseOperandArrayWithPrefix(Op.Name, Operands, Op.Type,
4460                                         Op.ConvertResult);
4461     } else {
4462       res = parseIntWithPrefix(Op.Name, Operands, Op.Type, Op.ConvertResult);
4463     }
4464     if (res != MatchOperand_NoMatch) {
4465       return res;
4466     }
4467   }
4468   return MatchOperand_NoMatch;
4469 }
4470 
4471 OperandMatchResultTy AMDGPUAsmParser::parseOModOperand(OperandVector &Operands) {
4472   StringRef Name = Parser.getTok().getString();
4473   if (Name == "mul") {
4474     return parseIntWithPrefix("mul", Operands,
4475                               AMDGPUOperand::ImmTyOModSI, ConvertOmodMul);
4476   }
4477 
4478   if (Name == "div") {
4479     return parseIntWithPrefix("div", Operands,
4480                               AMDGPUOperand::ImmTyOModSI, ConvertOmodDiv);
4481   }
4482 
4483   return MatchOperand_NoMatch;
4484 }
4485 
4486 void AMDGPUAsmParser::cvtVOP3OpSel(MCInst &Inst, const OperandVector &Operands) {
4487   cvtVOP3P(Inst, Operands);
4488 
4489   int Opc = Inst.getOpcode();
4490 
4491   int SrcNum;
4492   const int Ops[] = { AMDGPU::OpName::src0,
4493                       AMDGPU::OpName::src1,
4494                       AMDGPU::OpName::src2 };
4495   for (SrcNum = 0;
4496        SrcNum < 3 && AMDGPU::getNamedOperandIdx(Opc, Ops[SrcNum]) != -1;
4497        ++SrcNum);
4498   assert(SrcNum > 0);
4499 
4500   int OpSelIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::op_sel);
4501   unsigned OpSel = Inst.getOperand(OpSelIdx).getImm();
4502 
4503   if ((OpSel & (1 << SrcNum)) != 0) {
4504     int ModIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src0_modifiers);
4505     uint32_t ModVal = Inst.getOperand(ModIdx).getImm();
4506     Inst.getOperand(ModIdx).setImm(ModVal | SISrcMods::DST_OP_SEL);
4507   }
4508 }
4509 
4510 static bool isRegOrImmWithInputMods(const MCInstrDesc &Desc, unsigned OpNum) {
4511       // 1. This operand is input modifiers
4512   return Desc.OpInfo[OpNum].OperandType == AMDGPU::OPERAND_INPUT_MODS
4513       // 2. This is not last operand
4514       && Desc.NumOperands > (OpNum + 1)
4515       // 3. Next operand is register class
4516       && Desc.OpInfo[OpNum + 1].RegClass != -1
4517       // 4. Next register is not tied to any other operand
4518       && Desc.getOperandConstraint(OpNum + 1, MCOI::OperandConstraint::TIED_TO) == -1;
4519 }
4520 
4521 void AMDGPUAsmParser::cvtVOP3Interp(MCInst &Inst, const OperandVector &Operands)
4522 {
4523   OptionalImmIndexMap OptionalIdx;
4524   unsigned Opc = Inst.getOpcode();
4525 
4526   unsigned I = 1;
4527   const MCInstrDesc &Desc = MII.get(Inst.getOpcode());
4528   for (unsigned J = 0; J < Desc.getNumDefs(); ++J) {
4529     ((AMDGPUOperand &)*Operands[I++]).addRegOperands(Inst, 1);
4530   }
4531 
4532   for (unsigned E = Operands.size(); I != E; ++I) {
4533     AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[I]);
4534     if (isRegOrImmWithInputMods(Desc, Inst.getNumOperands())) {
4535       Op.addRegOrImmWithFPInputModsOperands(Inst, 2);
4536     } else if (Op.isInterpSlot() ||
4537                Op.isInterpAttr() ||
4538                Op.isAttrChan()) {
4539       Inst.addOperand(MCOperand::createImm(Op.Imm.Val));
4540     } else if (Op.isImmModifier()) {
4541       OptionalIdx[Op.getImmTy()] = I;
4542     } else {
4543       llvm_unreachable("unhandled operand type");
4544     }
4545   }
4546 
4547   if (AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::high) != -1) {
4548     addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyHigh);
4549   }
4550 
4551   if (AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::clamp) != -1) {
4552     addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyClampSI);
4553   }
4554 
4555   if (AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::omod) != -1) {
4556     addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyOModSI);
4557   }
4558 }
4559 
4560 void AMDGPUAsmParser::cvtVOP3(MCInst &Inst, const OperandVector &Operands,
4561                               OptionalImmIndexMap &OptionalIdx) {
4562   unsigned Opc = Inst.getOpcode();
4563 
4564   unsigned I = 1;
4565   const MCInstrDesc &Desc = MII.get(Inst.getOpcode());
4566   for (unsigned J = 0; J < Desc.getNumDefs(); ++J) {
4567     ((AMDGPUOperand &)*Operands[I++]).addRegOperands(Inst, 1);
4568   }
4569 
4570   if (AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src0_modifiers) != -1) {
4571     // This instruction has src modifiers
4572     for (unsigned E = Operands.size(); I != E; ++I) {
4573       AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[I]);
4574       if (isRegOrImmWithInputMods(Desc, Inst.getNumOperands())) {
4575         Op.addRegOrImmWithFPInputModsOperands(Inst, 2);
4576       } else if (Op.isImmModifier()) {
4577         OptionalIdx[Op.getImmTy()] = I;
4578       } else if (Op.isRegOrImm()) {
4579         Op.addRegOrImmOperands(Inst, 1);
4580       } else {
4581         llvm_unreachable("unhandled operand type");
4582       }
4583     }
4584   } else {
4585     // No src modifiers
4586     for (unsigned E = Operands.size(); I != E; ++I) {
4587       AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[I]);
4588       if (Op.isMod()) {
4589         OptionalIdx[Op.getImmTy()] = I;
4590       } else {
4591         Op.addRegOrImmOperands(Inst, 1);
4592       }
4593     }
4594   }
4595 
4596   if (AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::clamp) != -1) {
4597     addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyClampSI);
4598   }
4599 
4600   if (AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::omod) != -1) {
4601     addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyOModSI);
4602   }
4603 
4604   // special case v_mac_{f16, f32}:
4605   // it has src2 register operand that is tied to dst operand
4606   // we don't allow modifiers for this operand in assembler so src2_modifiers
4607   // should be 0
4608   if (Opc == AMDGPU::V_MAC_F32_e64_si || Opc == AMDGPU::V_MAC_F32_e64_vi ||
4609       Opc == AMDGPU::V_MAC_F16_e64_vi) {
4610     auto it = Inst.begin();
4611     std::advance(it, AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src2_modifiers));
4612     it = Inst.insert(it, MCOperand::createImm(0)); // no modifiers for src2
4613     ++it;
4614     Inst.insert(it, Inst.getOperand(0)); // src2 = dst
4615   }
4616 }
4617 
4618 void AMDGPUAsmParser::cvtVOP3(MCInst &Inst, const OperandVector &Operands) {
4619   OptionalImmIndexMap OptionalIdx;
4620   cvtVOP3(Inst, Operands, OptionalIdx);
4621 }
4622 
4623 void AMDGPUAsmParser::cvtVOP3P(MCInst &Inst,
4624                                const OperandVector &Operands) {
4625   OptionalImmIndexMap OptIdx;
4626   const int Opc = Inst.getOpcode();
4627   const MCInstrDesc &Desc = MII.get(Opc);
4628 
4629   const bool IsPacked = (Desc.TSFlags & SIInstrFlags::IsPacked) != 0;
4630 
4631   cvtVOP3(Inst, Operands, OptIdx);
4632 
4633   if (AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::vdst_in) != -1) {
4634     assert(!IsPacked);
4635     Inst.addOperand(Inst.getOperand(0));
4636   }
4637 
4638   // FIXME: This is messy. Parse the modifiers as if it was a normal VOP3
4639   // instruction, and then figure out where to actually put the modifiers
4640 
4641   addOptionalImmOperand(Inst, Operands, OptIdx, AMDGPUOperand::ImmTyOpSel);
4642 
4643   int OpSelHiIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::op_sel_hi);
4644   if (OpSelHiIdx != -1) {
4645     int DefaultVal = IsPacked ? -1 : 0;
4646     addOptionalImmOperand(Inst, Operands, OptIdx, AMDGPUOperand::ImmTyOpSelHi,
4647                           DefaultVal);
4648   }
4649 
4650   int NegLoIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::neg_lo);
4651   if (NegLoIdx != -1) {
4652     assert(IsPacked);
4653     addOptionalImmOperand(Inst, Operands, OptIdx, AMDGPUOperand::ImmTyNegLo);
4654     addOptionalImmOperand(Inst, Operands, OptIdx, AMDGPUOperand::ImmTyNegHi);
4655   }
4656 
4657   const int Ops[] = { AMDGPU::OpName::src0,
4658                       AMDGPU::OpName::src1,
4659                       AMDGPU::OpName::src2 };
4660   const int ModOps[] = { AMDGPU::OpName::src0_modifiers,
4661                          AMDGPU::OpName::src1_modifiers,
4662                          AMDGPU::OpName::src2_modifiers };
4663 
4664   int OpSelIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::op_sel);
4665 
4666   unsigned OpSel = Inst.getOperand(OpSelIdx).getImm();
4667   unsigned OpSelHi = 0;
4668   unsigned NegLo = 0;
4669   unsigned NegHi = 0;
4670 
4671   if (OpSelHiIdx != -1) {
4672     OpSelHi = Inst.getOperand(OpSelHiIdx).getImm();
4673   }
4674 
4675   if (NegLoIdx != -1) {
4676     int NegHiIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::neg_hi);
4677     NegLo = Inst.getOperand(NegLoIdx).getImm();
4678     NegHi = Inst.getOperand(NegHiIdx).getImm();
4679   }
4680 
4681   for (int J = 0; J < 3; ++J) {
4682     int OpIdx = AMDGPU::getNamedOperandIdx(Opc, Ops[J]);
4683     if (OpIdx == -1)
4684       break;
4685 
4686     uint32_t ModVal = 0;
4687 
4688     if ((OpSel & (1 << J)) != 0)
4689       ModVal |= SISrcMods::OP_SEL_0;
4690 
4691     if ((OpSelHi & (1 << J)) != 0)
4692       ModVal |= SISrcMods::OP_SEL_1;
4693 
4694     if ((NegLo & (1 << J)) != 0)
4695       ModVal |= SISrcMods::NEG;
4696 
4697     if ((NegHi & (1 << J)) != 0)
4698       ModVal |= SISrcMods::NEG_HI;
4699 
4700     int ModIdx = AMDGPU::getNamedOperandIdx(Opc, ModOps[J]);
4701 
4702     Inst.getOperand(ModIdx).setImm(Inst.getOperand(ModIdx).getImm() | ModVal);
4703   }
4704 }
4705 
4706 //===----------------------------------------------------------------------===//
4707 // dpp
4708 //===----------------------------------------------------------------------===//
4709 
4710 bool AMDGPUOperand::isDPPCtrl() const {
4711   bool result = isImm() && getImmTy() == ImmTyDppCtrl && isUInt<9>(getImm());
4712   if (result) {
4713     int64_t Imm = getImm();
4714     return ((Imm >= 0x000) && (Imm <= 0x0ff)) ||
4715            ((Imm >= 0x101) && (Imm <= 0x10f)) ||
4716            ((Imm >= 0x111) && (Imm <= 0x11f)) ||
4717            ((Imm >= 0x121) && (Imm <= 0x12f)) ||
4718            (Imm == 0x130) ||
4719            (Imm == 0x134) ||
4720            (Imm == 0x138) ||
4721            (Imm == 0x13c) ||
4722            (Imm == 0x140) ||
4723            (Imm == 0x141) ||
4724            (Imm == 0x142) ||
4725            (Imm == 0x143);
4726   }
4727   return false;
4728 }
4729 
4730 bool AMDGPUOperand::isGPRIdxMode() const {
4731   return isImm() && isUInt<4>(getImm());
4732 }
4733 
4734 bool AMDGPUOperand::isS16Imm() const {
4735   return isImm() && (isInt<16>(getImm()) || isUInt<16>(getImm()));
4736 }
4737 
4738 bool AMDGPUOperand::isU16Imm() const {
4739   return isImm() && isUInt<16>(getImm());
4740 }
4741 
4742 OperandMatchResultTy
4743 AMDGPUAsmParser::parseDPPCtrl(OperandVector &Operands) {
4744   SMLoc S = Parser.getTok().getLoc();
4745   StringRef Prefix;
4746   int64_t Int;
4747 
4748   if (getLexer().getKind() == AsmToken::Identifier) {
4749     Prefix = Parser.getTok().getString();
4750   } else {
4751     return MatchOperand_NoMatch;
4752   }
4753 
4754   if (Prefix == "row_mirror") {
4755     Int = 0x140;
4756     Parser.Lex();
4757   } else if (Prefix == "row_half_mirror") {
4758     Int = 0x141;
4759     Parser.Lex();
4760   } else {
4761     // Check to prevent parseDPPCtrlOps from eating invalid tokens
4762     if (Prefix != "quad_perm"
4763         && Prefix != "row_shl"
4764         && Prefix != "row_shr"
4765         && Prefix != "row_ror"
4766         && Prefix != "wave_shl"
4767         && Prefix != "wave_rol"
4768         && Prefix != "wave_shr"
4769         && Prefix != "wave_ror"
4770         && Prefix != "row_bcast") {
4771       return MatchOperand_NoMatch;
4772     }
4773 
4774     Parser.Lex();
4775     if (getLexer().isNot(AsmToken::Colon))
4776       return MatchOperand_ParseFail;
4777 
4778     if (Prefix == "quad_perm") {
4779       // quad_perm:[%d,%d,%d,%d]
4780       Parser.Lex();
4781       if (getLexer().isNot(AsmToken::LBrac))
4782         return MatchOperand_ParseFail;
4783       Parser.Lex();
4784 
4785       if (getParser().parseAbsoluteExpression(Int) || !(0 <= Int && Int <=3))
4786         return MatchOperand_ParseFail;
4787 
4788       for (int i = 0; i < 3; ++i) {
4789         if (getLexer().isNot(AsmToken::Comma))
4790           return MatchOperand_ParseFail;
4791         Parser.Lex();
4792 
4793         int64_t Temp;
4794         if (getParser().parseAbsoluteExpression(Temp) || !(0 <= Temp && Temp <=3))
4795           return MatchOperand_ParseFail;
4796         const int shift = i*2 + 2;
4797         Int += (Temp << shift);
4798       }
4799 
4800       if (getLexer().isNot(AsmToken::RBrac))
4801         return MatchOperand_ParseFail;
4802       Parser.Lex();
4803     } else {
4804       // sel:%d
4805       Parser.Lex();
4806       if (getParser().parseAbsoluteExpression(Int))
4807         return MatchOperand_ParseFail;
4808 
4809       if (Prefix == "row_shl" && 1 <= Int && Int <= 15) {
4810         Int |= 0x100;
4811       } else if (Prefix == "row_shr" && 1 <= Int && Int <= 15) {
4812         Int |= 0x110;
4813       } else if (Prefix == "row_ror" && 1 <= Int && Int <= 15) {
4814         Int |= 0x120;
4815       } else if (Prefix == "wave_shl" && 1 == Int) {
4816         Int = 0x130;
4817       } else if (Prefix == "wave_rol" && 1 == Int) {
4818         Int = 0x134;
4819       } else if (Prefix == "wave_shr" && 1 == Int) {
4820         Int = 0x138;
4821       } else if (Prefix == "wave_ror" && 1 == Int) {
4822         Int = 0x13C;
4823       } else if (Prefix == "row_bcast") {
4824         if (Int == 15) {
4825           Int = 0x142;
4826         } else if (Int == 31) {
4827           Int = 0x143;
4828         } else {
4829           return MatchOperand_ParseFail;
4830         }
4831       } else {
4832         return MatchOperand_ParseFail;
4833       }
4834     }
4835   }
4836 
4837   Operands.push_back(AMDGPUOperand::CreateImm(this, Int, S, AMDGPUOperand::ImmTyDppCtrl));
4838   return MatchOperand_Success;
4839 }
4840 
4841 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultRowMask() const {
4842   return AMDGPUOperand::CreateImm(this, 0xf, SMLoc(), AMDGPUOperand::ImmTyDppRowMask);
4843 }
4844 
4845 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultBankMask() const {
4846   return AMDGPUOperand::CreateImm(this, 0xf, SMLoc(), AMDGPUOperand::ImmTyDppBankMask);
4847 }
4848 
4849 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultBoundCtrl() const {
4850   return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTyDppBoundCtrl);
4851 }
4852 
4853 void AMDGPUAsmParser::cvtDPP(MCInst &Inst, const OperandVector &Operands) {
4854   OptionalImmIndexMap OptionalIdx;
4855 
4856   unsigned I = 1;
4857   const MCInstrDesc &Desc = MII.get(Inst.getOpcode());
4858   for (unsigned J = 0; J < Desc.getNumDefs(); ++J) {
4859     ((AMDGPUOperand &)*Operands[I++]).addRegOperands(Inst, 1);
4860   }
4861 
4862   // All DPP instructions with at least one source operand have a fake "old"
4863   // source at the beginning that's tied to the dst operand. Handle it here.
4864   if (Desc.getNumOperands() >= 2)
4865     Inst.addOperand(Inst.getOperand(0));
4866 
4867   for (unsigned E = Operands.size(); I != E; ++I) {
4868     AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[I]);
4869     // Add the register arguments
4870     if (Op.isReg() && Op.Reg.RegNo == AMDGPU::VCC) {
4871       // VOP2b (v_add_u32, v_sub_u32 ...) dpp use "vcc" token.
4872       // Skip it.
4873       continue;
4874     } if (isRegOrImmWithInputMods(Desc, Inst.getNumOperands())) {
4875       Op.addRegWithFPInputModsOperands(Inst, 2);
4876     } else if (Op.isDPPCtrl()) {
4877       Op.addImmOperands(Inst, 1);
4878     } else if (Op.isImm()) {
4879       // Handle optional arguments
4880       OptionalIdx[Op.getImmTy()] = I;
4881     } else {
4882       llvm_unreachable("Invalid operand type");
4883     }
4884   }
4885 
4886   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyDppRowMask, 0xf);
4887   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyDppBankMask, 0xf);
4888   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyDppBoundCtrl);
4889 }
4890 
4891 //===----------------------------------------------------------------------===//
4892 // sdwa
4893 //===----------------------------------------------------------------------===//
4894 
4895 OperandMatchResultTy
4896 AMDGPUAsmParser::parseSDWASel(OperandVector &Operands, StringRef Prefix,
4897                               AMDGPUOperand::ImmTy Type) {
4898   using namespace llvm::AMDGPU::SDWA;
4899 
4900   SMLoc S = Parser.getTok().getLoc();
4901   StringRef Value;
4902   OperandMatchResultTy res;
4903 
4904   res = parseStringWithPrefix(Prefix, Value);
4905   if (res != MatchOperand_Success) {
4906     return res;
4907   }
4908 
4909   int64_t Int;
4910   Int = StringSwitch<int64_t>(Value)
4911         .Case("BYTE_0", SdwaSel::BYTE_0)
4912         .Case("BYTE_1", SdwaSel::BYTE_1)
4913         .Case("BYTE_2", SdwaSel::BYTE_2)
4914         .Case("BYTE_3", SdwaSel::BYTE_3)
4915         .Case("WORD_0", SdwaSel::WORD_0)
4916         .Case("WORD_1", SdwaSel::WORD_1)
4917         .Case("DWORD", SdwaSel::DWORD)
4918         .Default(0xffffffff);
4919   Parser.Lex(); // eat last token
4920 
4921   if (Int == 0xffffffff) {
4922     return MatchOperand_ParseFail;
4923   }
4924 
4925   Operands.push_back(AMDGPUOperand::CreateImm(this, Int, S, Type));
4926   return MatchOperand_Success;
4927 }
4928 
4929 OperandMatchResultTy
4930 AMDGPUAsmParser::parseSDWADstUnused(OperandVector &Operands) {
4931   using namespace llvm::AMDGPU::SDWA;
4932 
4933   SMLoc S = Parser.getTok().getLoc();
4934   StringRef Value;
4935   OperandMatchResultTy res;
4936 
4937   res = parseStringWithPrefix("dst_unused", Value);
4938   if (res != MatchOperand_Success) {
4939     return res;
4940   }
4941 
4942   int64_t Int;
4943   Int = StringSwitch<int64_t>(Value)
4944         .Case("UNUSED_PAD", DstUnused::UNUSED_PAD)
4945         .Case("UNUSED_SEXT", DstUnused::UNUSED_SEXT)
4946         .Case("UNUSED_PRESERVE", DstUnused::UNUSED_PRESERVE)
4947         .Default(0xffffffff);
4948   Parser.Lex(); // eat last token
4949 
4950   if (Int == 0xffffffff) {
4951     return MatchOperand_ParseFail;
4952   }
4953 
4954   Operands.push_back(AMDGPUOperand::CreateImm(this, Int, S, AMDGPUOperand::ImmTySdwaDstUnused));
4955   return MatchOperand_Success;
4956 }
4957 
4958 void AMDGPUAsmParser::cvtSdwaVOP1(MCInst &Inst, const OperandVector &Operands) {
4959   cvtSDWA(Inst, Operands, SIInstrFlags::VOP1);
4960 }
4961 
4962 void AMDGPUAsmParser::cvtSdwaVOP2(MCInst &Inst, const OperandVector &Operands) {
4963   cvtSDWA(Inst, Operands, SIInstrFlags::VOP2);
4964 }
4965 
4966 void AMDGPUAsmParser::cvtSdwaVOP2b(MCInst &Inst, const OperandVector &Operands) {
4967   cvtSDWA(Inst, Operands, SIInstrFlags::VOP2, true);
4968 }
4969 
4970 void AMDGPUAsmParser::cvtSdwaVOPC(MCInst &Inst, const OperandVector &Operands) {
4971   cvtSDWA(Inst, Operands, SIInstrFlags::VOPC, isVI());
4972 }
4973 
4974 void AMDGPUAsmParser::cvtSDWA(MCInst &Inst, const OperandVector &Operands,
4975                               uint64_t BasicInstType, bool skipVcc) {
4976   using namespace llvm::AMDGPU::SDWA;
4977 
4978   OptionalImmIndexMap OptionalIdx;
4979   bool skippedVcc = false;
4980 
4981   unsigned I = 1;
4982   const MCInstrDesc &Desc = MII.get(Inst.getOpcode());
4983   for (unsigned J = 0; J < Desc.getNumDefs(); ++J) {
4984     ((AMDGPUOperand &)*Operands[I++]).addRegOperands(Inst, 1);
4985   }
4986 
4987   for (unsigned E = Operands.size(); I != E; ++I) {
4988     AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[I]);
4989     if (skipVcc && !skippedVcc && Op.isReg() && Op.Reg.RegNo == AMDGPU::VCC) {
4990       // VOP2b (v_add_u32, v_sub_u32 ...) sdwa use "vcc" token as dst.
4991       // Skip it if it's 2nd (e.g. v_add_i32_sdwa v1, vcc, v2, v3)
4992       // or 4th (v_addc_u32_sdwa v1, vcc, v2, v3, vcc) operand.
4993       // Skip VCC only if we didn't skip it on previous iteration.
4994       if (BasicInstType == SIInstrFlags::VOP2 &&
4995           (Inst.getNumOperands() == 1 || Inst.getNumOperands() == 5)) {
4996         skippedVcc = true;
4997         continue;
4998       } else if (BasicInstType == SIInstrFlags::VOPC &&
4999                  Inst.getNumOperands() == 0) {
5000         skippedVcc = true;
5001         continue;
5002       }
5003     }
5004     if (isRegOrImmWithInputMods(Desc, Inst.getNumOperands())) {
5005       Op.addRegOrImmWithInputModsOperands(Inst, 2);
5006     } else if (Op.isImm()) {
5007       // Handle optional arguments
5008       OptionalIdx[Op.getImmTy()] = I;
5009     } else {
5010       llvm_unreachable("Invalid operand type");
5011     }
5012     skippedVcc = false;
5013   }
5014 
5015   if (Inst.getOpcode() != AMDGPU::V_NOP_sdwa_gfx9 &&
5016       Inst.getOpcode() != AMDGPU::V_NOP_sdwa_vi) {
5017     // v_nop_sdwa_sdwa_vi/gfx9 has no optional sdwa arguments
5018     switch (BasicInstType) {
5019     case SIInstrFlags::VOP1:
5020       addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyClampSI, 0);
5021       if (AMDGPU::getNamedOperandIdx(Inst.getOpcode(), AMDGPU::OpName::omod) != -1) {
5022         addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyOModSI, 0);
5023       }
5024       addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySdwaDstSel, SdwaSel::DWORD);
5025       addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySdwaDstUnused, DstUnused::UNUSED_PRESERVE);
5026       addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySdwaSrc0Sel, SdwaSel::DWORD);
5027       break;
5028 
5029     case SIInstrFlags::VOP2:
5030       addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyClampSI, 0);
5031       if (AMDGPU::getNamedOperandIdx(Inst.getOpcode(), AMDGPU::OpName::omod) != -1) {
5032         addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyOModSI, 0);
5033       }
5034       addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySdwaDstSel, SdwaSel::DWORD);
5035       addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySdwaDstUnused, DstUnused::UNUSED_PRESERVE);
5036       addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySdwaSrc0Sel, SdwaSel::DWORD);
5037       addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySdwaSrc1Sel, SdwaSel::DWORD);
5038       break;
5039 
5040     case SIInstrFlags::VOPC:
5041       addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyClampSI, 0);
5042       addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySdwaSrc0Sel, SdwaSel::DWORD);
5043       addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySdwaSrc1Sel, SdwaSel::DWORD);
5044       break;
5045 
5046     default:
5047       llvm_unreachable("Invalid instruction type. Only VOP1, VOP2 and VOPC allowed");
5048     }
5049   }
5050 
5051   // special case v_mac_{f16, f32}:
5052   // it has src2 register operand that is tied to dst operand
5053   if (Inst.getOpcode() == AMDGPU::V_MAC_F32_sdwa_vi ||
5054       Inst.getOpcode() == AMDGPU::V_MAC_F16_sdwa_vi)  {
5055     auto it = Inst.begin();
5056     std::advance(
5057       it, AMDGPU::getNamedOperandIdx(Inst.getOpcode(), AMDGPU::OpName::src2));
5058     Inst.insert(it, Inst.getOperand(0)); // src2 = dst
5059   }
5060 }
5061 
5062 /// Force static initialization.
5063 extern "C" void LLVMInitializeAMDGPUAsmParser() {
5064   RegisterMCAsmParser<AMDGPUAsmParser> A(getTheAMDGPUTarget());
5065   RegisterMCAsmParser<AMDGPUAsmParser> B(getTheGCNTarget());
5066 }
5067 
5068 #define GET_REGISTER_MATCHER
5069 #define GET_MATCHER_IMPLEMENTATION
5070 #define GET_MNEMONIC_SPELL_CHECKER
5071 #include "AMDGPUGenAsmMatcher.inc"
5072 
5073 // This fuction should be defined after auto-generated include so that we have
5074 // MatchClassKind enum defined
5075 unsigned AMDGPUAsmParser::validateTargetOperandClass(MCParsedAsmOperand &Op,
5076                                                      unsigned Kind) {
5077   // Tokens like "glc" would be parsed as immediate operands in ParseOperand().
5078   // But MatchInstructionImpl() expects to meet token and fails to validate
5079   // operand. This method checks if we are given immediate operand but expect to
5080   // get corresponding token.
5081   AMDGPUOperand &Operand = (AMDGPUOperand&)Op;
5082   switch (Kind) {
5083   case MCK_addr64:
5084     return Operand.isAddr64() ? Match_Success : Match_InvalidOperand;
5085   case MCK_gds:
5086     return Operand.isGDS() ? Match_Success : Match_InvalidOperand;
5087   case MCK_lds:
5088     return Operand.isLDS() ? Match_Success : Match_InvalidOperand;
5089   case MCK_glc:
5090     return Operand.isGLC() ? Match_Success : Match_InvalidOperand;
5091   case MCK_d16:
5092     return Operand.isD16() ? Match_Success : Match_InvalidOperand;
5093   case MCK_idxen:
5094     return Operand.isIdxen() ? Match_Success : Match_InvalidOperand;
5095   case MCK_offen:
5096     return Operand.isOffen() ? Match_Success : Match_InvalidOperand;
5097   case MCK_SSrcB32:
5098     // When operands have expression values, they will return true for isToken,
5099     // because it is not possible to distinguish between a token and an
5100     // expression at parse time. MatchInstructionImpl() will always try to
5101     // match an operand as a token, when isToken returns true, and when the
5102     // name of the expression is not a valid token, the match will fail,
5103     // so we need to handle it here.
5104     return Operand.isSSrcB32() ? Match_Success : Match_InvalidOperand;
5105   case MCK_SSrcF32:
5106     return Operand.isSSrcF32() ? Match_Success : Match_InvalidOperand;
5107   case MCK_SoppBrTarget:
5108     return Operand.isSoppBrTarget() ? Match_Success : Match_InvalidOperand;
5109   case MCK_VReg32OrOff:
5110     return Operand.isVReg32OrOff() ? Match_Success : Match_InvalidOperand;
5111   case MCK_InterpSlot:
5112     return Operand.isInterpSlot() ? Match_Success : Match_InvalidOperand;
5113   case MCK_Attr:
5114     return Operand.isInterpAttr() ? Match_Success : Match_InvalidOperand;
5115   case MCK_AttrChan:
5116     return Operand.isAttrChan() ? Match_Success : Match_InvalidOperand;
5117   default:
5118     return Match_InvalidOperand;
5119   }
5120 }
5121