1 //===- AMDGPUAsmParser.cpp - Parse SI asm to MCInst instructions ----------===//
2 //
3 //                     The LLVM Compiler Infrastructure
4 //
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
7 //
8 //===----------------------------------------------------------------------===//
9 
10 #include "AMDGPU.h"
11 #include "AMDKernelCodeT.h"
12 #include "MCTargetDesc/AMDGPUMCTargetDesc.h"
13 #include "MCTargetDesc/AMDGPUTargetStreamer.h"
14 #include "SIDefines.h"
15 #include "SIInstrInfo.h"
16 #include "Utils/AMDGPUAsmUtils.h"
17 #include "Utils/AMDGPUBaseInfo.h"
18 #include "Utils/AMDKernelCodeTUtils.h"
19 #include "llvm/ADT/APFloat.h"
20 #include "llvm/ADT/APInt.h"
21 #include "llvm/ADT/ArrayRef.h"
22 #include "llvm/ADT/STLExtras.h"
23 #include "llvm/ADT/SmallBitVector.h"
24 #include "llvm/ADT/SmallString.h"
25 #include "llvm/ADT/StringRef.h"
26 #include "llvm/ADT/StringSwitch.h"
27 #include "llvm/ADT/Twine.h"
28 #include "llvm/BinaryFormat/ELF.h"
29 #include "llvm/CodeGen/MachineValueType.h"
30 #include "llvm/MC/MCAsmInfo.h"
31 #include "llvm/MC/MCContext.h"
32 #include "llvm/MC/MCExpr.h"
33 #include "llvm/MC/MCInst.h"
34 #include "llvm/MC/MCInstrDesc.h"
35 #include "llvm/MC/MCInstrInfo.h"
36 #include "llvm/MC/MCParser/MCAsmLexer.h"
37 #include "llvm/MC/MCParser/MCAsmParser.h"
38 #include "llvm/MC/MCParser/MCAsmParserExtension.h"
39 #include "llvm/MC/MCParser/MCParsedAsmOperand.h"
40 #include "llvm/MC/MCParser/MCTargetAsmParser.h"
41 #include "llvm/MC/MCRegisterInfo.h"
42 #include "llvm/MC/MCStreamer.h"
43 #include "llvm/MC/MCSubtargetInfo.h"
44 #include "llvm/MC/MCSymbol.h"
45 #include "llvm/Support/AMDGPUMetadata.h"
46 #include "llvm/Support/Casting.h"
47 #include "llvm/Support/Compiler.h"
48 #include "llvm/Support/ErrorHandling.h"
49 #include "llvm/Support/MathExtras.h"
50 #include "llvm/Support/SMLoc.h"
51 #include "llvm/Support/TargetRegistry.h"
52 #include "llvm/Support/raw_ostream.h"
53 #include <algorithm>
54 #include <cassert>
55 #include <cstdint>
56 #include <cstring>
57 #include <iterator>
58 #include <map>
59 #include <memory>
60 #include <string>
61 
62 using namespace llvm;
63 using namespace llvm::AMDGPU;
64 
65 namespace {
66 
67 class AMDGPUAsmParser;
68 
69 enum RegisterKind { IS_UNKNOWN, IS_VGPR, IS_SGPR, IS_TTMP, IS_SPECIAL };
70 
71 //===----------------------------------------------------------------------===//
72 // Operand
73 //===----------------------------------------------------------------------===//
74 
75 class AMDGPUOperand : public MCParsedAsmOperand {
76   enum KindTy {
77     Token,
78     Immediate,
79     Register,
80     Expression
81   } Kind;
82 
83   SMLoc StartLoc, EndLoc;
84   const AMDGPUAsmParser *AsmParser;
85 
86 public:
87   AMDGPUOperand(KindTy Kind_, const AMDGPUAsmParser *AsmParser_)
88     : MCParsedAsmOperand(), Kind(Kind_), AsmParser(AsmParser_) {}
89 
90   using Ptr = std::unique_ptr<AMDGPUOperand>;
91 
92   struct Modifiers {
93     bool Abs = false;
94     bool Neg = false;
95     bool Sext = false;
96 
97     bool hasFPModifiers() const { return Abs || Neg; }
98     bool hasIntModifiers() const { return Sext; }
99     bool hasModifiers() const { return hasFPModifiers() || hasIntModifiers(); }
100 
101     int64_t getFPModifiersOperand() const {
102       int64_t Operand = 0;
103       Operand |= Abs ? SISrcMods::ABS : 0;
104       Operand |= Neg ? SISrcMods::NEG : 0;
105       return Operand;
106     }
107 
108     int64_t getIntModifiersOperand() const {
109       int64_t Operand = 0;
110       Operand |= Sext ? SISrcMods::SEXT : 0;
111       return Operand;
112     }
113 
114     int64_t getModifiersOperand() const {
115       assert(!(hasFPModifiers() && hasIntModifiers())
116            && "fp and int modifiers should not be used simultaneously");
117       if (hasFPModifiers()) {
118         return getFPModifiersOperand();
119       } else if (hasIntModifiers()) {
120         return getIntModifiersOperand();
121       } else {
122         return 0;
123       }
124     }
125 
126     friend raw_ostream &operator <<(raw_ostream &OS, AMDGPUOperand::Modifiers Mods);
127   };
128 
129   enum ImmTy {
130     ImmTyNone,
131     ImmTyGDS,
132     ImmTyLDS,
133     ImmTyOffen,
134     ImmTyIdxen,
135     ImmTyAddr64,
136     ImmTyOffset,
137     ImmTyInstOffset,
138     ImmTyOffset0,
139     ImmTyOffset1,
140     ImmTyGLC,
141     ImmTySLC,
142     ImmTyTFE,
143     ImmTyD16,
144     ImmTyClampSI,
145     ImmTyOModSI,
146     ImmTyDppCtrl,
147     ImmTyDppRowMask,
148     ImmTyDppBankMask,
149     ImmTyDppBoundCtrl,
150     ImmTySdwaDstSel,
151     ImmTySdwaSrc0Sel,
152     ImmTySdwaSrc1Sel,
153     ImmTySdwaDstUnused,
154     ImmTyDMask,
155     ImmTyUNorm,
156     ImmTyDA,
157     ImmTyR128,
158     ImmTyLWE,
159     ImmTyExpTgt,
160     ImmTyExpCompr,
161     ImmTyExpVM,
162     ImmTyDFMT,
163     ImmTyNFMT,
164     ImmTyHwreg,
165     ImmTyOff,
166     ImmTySendMsg,
167     ImmTyInterpSlot,
168     ImmTyInterpAttr,
169     ImmTyAttrChan,
170     ImmTyOpSel,
171     ImmTyOpSelHi,
172     ImmTyNegLo,
173     ImmTyNegHi,
174     ImmTySwizzle,
175     ImmTyHigh
176   };
177 
178   struct TokOp {
179     const char *Data;
180     unsigned Length;
181   };
182 
183   struct ImmOp {
184     int64_t Val;
185     ImmTy Type;
186     bool IsFPImm;
187     Modifiers Mods;
188   };
189 
190   struct RegOp {
191     unsigned RegNo;
192     bool IsForcedVOP3;
193     Modifiers Mods;
194   };
195 
196   union {
197     TokOp Tok;
198     ImmOp Imm;
199     RegOp Reg;
200     const MCExpr *Expr;
201   };
202 
203   bool isToken() const override {
204     if (Kind == Token)
205       return true;
206 
207     if (Kind != Expression || !Expr)
208       return false;
209 
210     // When parsing operands, we can't always tell if something was meant to be
211     // a token, like 'gds', or an expression that references a global variable.
212     // In this case, we assume the string is an expression, and if we need to
213     // interpret is a token, then we treat the symbol name as the token.
214     return isa<MCSymbolRefExpr>(Expr);
215   }
216 
217   bool isImm() const override {
218     return Kind == Immediate;
219   }
220 
221   bool isInlinableImm(MVT type) const;
222   bool isLiteralImm(MVT type) const;
223 
224   bool isRegKind() const {
225     return Kind == Register;
226   }
227 
228   bool isReg() const override {
229     return isRegKind() && !hasModifiers();
230   }
231 
232   bool isRegOrImmWithInputMods(MVT type) const {
233     return isRegKind() || isInlinableImm(type);
234   }
235 
236   bool isRegOrImmWithInt16InputMods() const {
237     return isRegOrImmWithInputMods(MVT::i16);
238   }
239 
240   bool isRegOrImmWithInt32InputMods() const {
241     return isRegOrImmWithInputMods(MVT::i32);
242   }
243 
244   bool isRegOrImmWithInt64InputMods() const {
245     return isRegOrImmWithInputMods(MVT::i64);
246   }
247 
248   bool isRegOrImmWithFP16InputMods() const {
249     return isRegOrImmWithInputMods(MVT::f16);
250   }
251 
252   bool isRegOrImmWithFP32InputMods() const {
253     return isRegOrImmWithInputMods(MVT::f32);
254   }
255 
256   bool isRegOrImmWithFP64InputMods() const {
257     return isRegOrImmWithInputMods(MVT::f64);
258   }
259 
260   bool isVReg() const {
261     return isRegClass(AMDGPU::VGPR_32RegClassID) ||
262            isRegClass(AMDGPU::VReg_64RegClassID) ||
263            isRegClass(AMDGPU::VReg_96RegClassID) ||
264            isRegClass(AMDGPU::VReg_128RegClassID) ||
265            isRegClass(AMDGPU::VReg_256RegClassID) ||
266            isRegClass(AMDGPU::VReg_512RegClassID);
267   }
268 
269   bool isVReg32OrOff() const {
270     return isOff() || isRegClass(AMDGPU::VGPR_32RegClassID);
271   }
272 
273   bool isSDWAOperand(MVT type) const;
274   bool isSDWAFP16Operand() const;
275   bool isSDWAFP32Operand() const;
276   bool isSDWAInt16Operand() const;
277   bool isSDWAInt32Operand() const;
278 
279   bool isImmTy(ImmTy ImmT) const {
280     return isImm() && Imm.Type == ImmT;
281   }
282 
283   bool isImmModifier() const {
284     return isImm() && Imm.Type != ImmTyNone;
285   }
286 
287   bool isClampSI() const { return isImmTy(ImmTyClampSI); }
288   bool isOModSI() const { return isImmTy(ImmTyOModSI); }
289   bool isDMask() const { return isImmTy(ImmTyDMask); }
290   bool isUNorm() const { return isImmTy(ImmTyUNorm); }
291   bool isDA() const { return isImmTy(ImmTyDA); }
292   bool isR128() const { return isImmTy(ImmTyR128); }
293   bool isLWE() const { return isImmTy(ImmTyLWE); }
294   bool isOff() const { return isImmTy(ImmTyOff); }
295   bool isExpTgt() const { return isImmTy(ImmTyExpTgt); }
296   bool isExpVM() const { return isImmTy(ImmTyExpVM); }
297   bool isExpCompr() const { return isImmTy(ImmTyExpCompr); }
298   bool isOffen() const { return isImmTy(ImmTyOffen); }
299   bool isIdxen() const { return isImmTy(ImmTyIdxen); }
300   bool isAddr64() const { return isImmTy(ImmTyAddr64); }
301   bool isOffset() const { return isImmTy(ImmTyOffset) && isUInt<16>(getImm()); }
302   bool isOffset0() const { return isImmTy(ImmTyOffset0) && isUInt<16>(getImm()); }
303   bool isOffset1() const { return isImmTy(ImmTyOffset1) && isUInt<8>(getImm()); }
304 
305   bool isOffsetU12() const { return (isImmTy(ImmTyOffset) || isImmTy(ImmTyInstOffset)) && isUInt<12>(getImm()); }
306   bool isOffsetS13() const { return (isImmTy(ImmTyOffset) || isImmTy(ImmTyInstOffset)) && isInt<13>(getImm()); }
307   bool isGDS() const { return isImmTy(ImmTyGDS); }
308   bool isLDS() const { return isImmTy(ImmTyLDS); }
309   bool isGLC() const { return isImmTy(ImmTyGLC); }
310   bool isSLC() const { return isImmTy(ImmTySLC); }
311   bool isTFE() const { return isImmTy(ImmTyTFE); }
312   bool isD16() const { return isImmTy(ImmTyD16); }
313   bool isDFMT() const { return isImmTy(ImmTyDFMT) && isUInt<8>(getImm()); }
314   bool isNFMT() const { return isImmTy(ImmTyNFMT) && isUInt<8>(getImm()); }
315   bool isBankMask() const { return isImmTy(ImmTyDppBankMask); }
316   bool isRowMask() const { return isImmTy(ImmTyDppRowMask); }
317   bool isBoundCtrl() const { return isImmTy(ImmTyDppBoundCtrl); }
318   bool isSDWADstSel() const { return isImmTy(ImmTySdwaDstSel); }
319   bool isSDWASrc0Sel() const { return isImmTy(ImmTySdwaSrc0Sel); }
320   bool isSDWASrc1Sel() const { return isImmTy(ImmTySdwaSrc1Sel); }
321   bool isSDWADstUnused() const { return isImmTy(ImmTySdwaDstUnused); }
322   bool isInterpSlot() const { return isImmTy(ImmTyInterpSlot); }
323   bool isInterpAttr() const { return isImmTy(ImmTyInterpAttr); }
324   bool isAttrChan() const { return isImmTy(ImmTyAttrChan); }
325   bool isOpSel() const { return isImmTy(ImmTyOpSel); }
326   bool isOpSelHi() const { return isImmTy(ImmTyOpSelHi); }
327   bool isNegLo() const { return isImmTy(ImmTyNegLo); }
328   bool isNegHi() const { return isImmTy(ImmTyNegHi); }
329   bool isHigh() const { return isImmTy(ImmTyHigh); }
330 
331   bool isMod() const {
332     return isClampSI() || isOModSI();
333   }
334 
335   bool isRegOrImm() const {
336     return isReg() || isImm();
337   }
338 
339   bool isRegClass(unsigned RCID) const;
340 
341   bool isRegOrInlineNoMods(unsigned RCID, MVT type) const {
342     return (isRegClass(RCID) || isInlinableImm(type)) && !hasModifiers();
343   }
344 
345   bool isSCSrcB16() const {
346     return isRegOrInlineNoMods(AMDGPU::SReg_32RegClassID, MVT::i16);
347   }
348 
349   bool isSCSrcV2B16() const {
350     return isSCSrcB16();
351   }
352 
353   bool isSCSrcB32() const {
354     return isRegOrInlineNoMods(AMDGPU::SReg_32RegClassID, MVT::i32);
355   }
356 
357   bool isSCSrcB64() const {
358     return isRegOrInlineNoMods(AMDGPU::SReg_64RegClassID, MVT::i64);
359   }
360 
361   bool isSCSrcF16() const {
362     return isRegOrInlineNoMods(AMDGPU::SReg_32RegClassID, MVT::f16);
363   }
364 
365   bool isSCSrcV2F16() const {
366     return isSCSrcF16();
367   }
368 
369   bool isSCSrcF32() const {
370     return isRegOrInlineNoMods(AMDGPU::SReg_32RegClassID, MVT::f32);
371   }
372 
373   bool isSCSrcF64() const {
374     return isRegOrInlineNoMods(AMDGPU::SReg_64RegClassID, MVT::f64);
375   }
376 
377   bool isSSrcB32() const {
378     return isSCSrcB32() || isLiteralImm(MVT::i32) || isExpr();
379   }
380 
381   bool isSSrcB16() const {
382     return isSCSrcB16() || isLiteralImm(MVT::i16);
383   }
384 
385   bool isSSrcV2B16() const {
386     llvm_unreachable("cannot happen");
387     return isSSrcB16();
388   }
389 
390   bool isSSrcB64() const {
391     // TODO: Find out how SALU supports extension of 32-bit literals to 64 bits.
392     // See isVSrc64().
393     return isSCSrcB64() || isLiteralImm(MVT::i64);
394   }
395 
396   bool isSSrcF32() const {
397     return isSCSrcB32() || isLiteralImm(MVT::f32) || isExpr();
398   }
399 
400   bool isSSrcF64() const {
401     return isSCSrcB64() || isLiteralImm(MVT::f64);
402   }
403 
404   bool isSSrcF16() const {
405     return isSCSrcB16() || isLiteralImm(MVT::f16);
406   }
407 
408   bool isSSrcV2F16() const {
409     llvm_unreachable("cannot happen");
410     return isSSrcF16();
411   }
412 
413   bool isVCSrcB32() const {
414     return isRegOrInlineNoMods(AMDGPU::VS_32RegClassID, MVT::i32);
415   }
416 
417   bool isVCSrcB64() const {
418     return isRegOrInlineNoMods(AMDGPU::VS_64RegClassID, MVT::i64);
419   }
420 
421   bool isVCSrcB16() const {
422     return isRegOrInlineNoMods(AMDGPU::VS_32RegClassID, MVT::i16);
423   }
424 
425   bool isVCSrcV2B16() const {
426     return isVCSrcB16();
427   }
428 
429   bool isVCSrcF32() const {
430     return isRegOrInlineNoMods(AMDGPU::VS_32RegClassID, MVT::f32);
431   }
432 
433   bool isVCSrcF64() const {
434     return isRegOrInlineNoMods(AMDGPU::VS_64RegClassID, MVT::f64);
435   }
436 
437   bool isVCSrcF16() const {
438     return isRegOrInlineNoMods(AMDGPU::VS_32RegClassID, MVT::f16);
439   }
440 
441   bool isVCSrcV2F16() const {
442     return isVCSrcF16();
443   }
444 
445   bool isVSrcB32() const {
446     return isVCSrcF32() || isLiteralImm(MVT::i32);
447   }
448 
449   bool isVSrcB64() const {
450     return isVCSrcF64() || isLiteralImm(MVT::i64);
451   }
452 
453   bool isVSrcB16() const {
454     return isVCSrcF16() || isLiteralImm(MVT::i16);
455   }
456 
457   bool isVSrcV2B16() const {
458     llvm_unreachable("cannot happen");
459     return isVSrcB16();
460   }
461 
462   bool isVSrcF32() const {
463     return isVCSrcF32() || isLiteralImm(MVT::f32);
464   }
465 
466   bool isVSrcF64() const {
467     return isVCSrcF64() || isLiteralImm(MVT::f64);
468   }
469 
470   bool isVSrcF16() const {
471     return isVCSrcF16() || isLiteralImm(MVT::f16);
472   }
473 
474   bool isVSrcV2F16() const {
475     llvm_unreachable("cannot happen");
476     return isVSrcF16();
477   }
478 
479   bool isKImmFP32() const {
480     return isLiteralImm(MVT::f32);
481   }
482 
483   bool isKImmFP16() const {
484     return isLiteralImm(MVT::f16);
485   }
486 
487   bool isMem() const override {
488     return false;
489   }
490 
491   bool isExpr() const {
492     return Kind == Expression;
493   }
494 
495   bool isSoppBrTarget() const {
496     return isExpr() || isImm();
497   }
498 
499   bool isSWaitCnt() const;
500   bool isHwreg() const;
501   bool isSendMsg() const;
502   bool isSwizzle() const;
503   bool isSMRDOffset8() const;
504   bool isSMRDOffset20() const;
505   bool isSMRDLiteralOffset() const;
506   bool isDPPCtrl() const;
507   bool isGPRIdxMode() const;
508   bool isS16Imm() const;
509   bool isU16Imm() const;
510 
511   StringRef getExpressionAsToken() const {
512     assert(isExpr());
513     const MCSymbolRefExpr *S = cast<MCSymbolRefExpr>(Expr);
514     return S->getSymbol().getName();
515   }
516 
517   StringRef getToken() const {
518     assert(isToken());
519 
520     if (Kind == Expression)
521       return getExpressionAsToken();
522 
523     return StringRef(Tok.Data, Tok.Length);
524   }
525 
526   int64_t getImm() const {
527     assert(isImm());
528     return Imm.Val;
529   }
530 
531   ImmTy getImmTy() const {
532     assert(isImm());
533     return Imm.Type;
534   }
535 
536   unsigned getReg() const override {
537     return Reg.RegNo;
538   }
539 
540   SMLoc getStartLoc() const override {
541     return StartLoc;
542   }
543 
544   SMLoc getEndLoc() const override {
545     return EndLoc;
546   }
547 
548   SMRange getLocRange() const {
549     return SMRange(StartLoc, EndLoc);
550   }
551 
552   Modifiers getModifiers() const {
553     assert(isRegKind() || isImmTy(ImmTyNone));
554     return isRegKind() ? Reg.Mods : Imm.Mods;
555   }
556 
557   void setModifiers(Modifiers Mods) {
558     assert(isRegKind() || isImmTy(ImmTyNone));
559     if (isRegKind())
560       Reg.Mods = Mods;
561     else
562       Imm.Mods = Mods;
563   }
564 
565   bool hasModifiers() const {
566     return getModifiers().hasModifiers();
567   }
568 
569   bool hasFPModifiers() const {
570     return getModifiers().hasFPModifiers();
571   }
572 
573   bool hasIntModifiers() const {
574     return getModifiers().hasIntModifiers();
575   }
576 
577   uint64_t applyInputFPModifiers(uint64_t Val, unsigned Size) const;
578 
579   void addImmOperands(MCInst &Inst, unsigned N, bool ApplyModifiers = true) const;
580 
581   void addLiteralImmOperand(MCInst &Inst, int64_t Val, bool ApplyModifiers) const;
582 
583   template <unsigned Bitwidth>
584   void addKImmFPOperands(MCInst &Inst, unsigned N) const;
585 
586   void addKImmFP16Operands(MCInst &Inst, unsigned N) const {
587     addKImmFPOperands<16>(Inst, N);
588   }
589 
590   void addKImmFP32Operands(MCInst &Inst, unsigned N) const {
591     addKImmFPOperands<32>(Inst, N);
592   }
593 
594   void addRegOperands(MCInst &Inst, unsigned N) const;
595 
596   void addRegOrImmOperands(MCInst &Inst, unsigned N) const {
597     if (isRegKind())
598       addRegOperands(Inst, N);
599     else if (isExpr())
600       Inst.addOperand(MCOperand::createExpr(Expr));
601     else
602       addImmOperands(Inst, N);
603   }
604 
605   void addRegOrImmWithInputModsOperands(MCInst &Inst, unsigned N) const {
606     Modifiers Mods = getModifiers();
607     Inst.addOperand(MCOperand::createImm(Mods.getModifiersOperand()));
608     if (isRegKind()) {
609       addRegOperands(Inst, N);
610     } else {
611       addImmOperands(Inst, N, false);
612     }
613   }
614 
615   void addRegOrImmWithFPInputModsOperands(MCInst &Inst, unsigned N) const {
616     assert(!hasIntModifiers());
617     addRegOrImmWithInputModsOperands(Inst, N);
618   }
619 
620   void addRegOrImmWithIntInputModsOperands(MCInst &Inst, unsigned N) const {
621     assert(!hasFPModifiers());
622     addRegOrImmWithInputModsOperands(Inst, N);
623   }
624 
625   void addRegWithInputModsOperands(MCInst &Inst, unsigned N) const {
626     Modifiers Mods = getModifiers();
627     Inst.addOperand(MCOperand::createImm(Mods.getModifiersOperand()));
628     assert(isRegKind());
629     addRegOperands(Inst, N);
630   }
631 
632   void addRegWithFPInputModsOperands(MCInst &Inst, unsigned N) const {
633     assert(!hasIntModifiers());
634     addRegWithInputModsOperands(Inst, N);
635   }
636 
637   void addRegWithIntInputModsOperands(MCInst &Inst, unsigned N) const {
638     assert(!hasFPModifiers());
639     addRegWithInputModsOperands(Inst, N);
640   }
641 
642   void addSoppBrTargetOperands(MCInst &Inst, unsigned N) const {
643     if (isImm())
644       addImmOperands(Inst, N);
645     else {
646       assert(isExpr());
647       Inst.addOperand(MCOperand::createExpr(Expr));
648     }
649   }
650 
651   static void printImmTy(raw_ostream& OS, ImmTy Type) {
652     switch (Type) {
653     case ImmTyNone: OS << "None"; break;
654     case ImmTyGDS: OS << "GDS"; break;
655     case ImmTyLDS: OS << "LDS"; break;
656     case ImmTyOffen: OS << "Offen"; break;
657     case ImmTyIdxen: OS << "Idxen"; break;
658     case ImmTyAddr64: OS << "Addr64"; break;
659     case ImmTyOffset: OS << "Offset"; break;
660     case ImmTyInstOffset: OS << "InstOffset"; break;
661     case ImmTyOffset0: OS << "Offset0"; break;
662     case ImmTyOffset1: OS << "Offset1"; break;
663     case ImmTyGLC: OS << "GLC"; break;
664     case ImmTySLC: OS << "SLC"; break;
665     case ImmTyTFE: OS << "TFE"; break;
666     case ImmTyD16: OS << "D16"; break;
667     case ImmTyDFMT: OS << "DFMT"; break;
668     case ImmTyNFMT: OS << "NFMT"; break;
669     case ImmTyClampSI: OS << "ClampSI"; break;
670     case ImmTyOModSI: OS << "OModSI"; break;
671     case ImmTyDppCtrl: OS << "DppCtrl"; break;
672     case ImmTyDppRowMask: OS << "DppRowMask"; break;
673     case ImmTyDppBankMask: OS << "DppBankMask"; break;
674     case ImmTyDppBoundCtrl: OS << "DppBoundCtrl"; break;
675     case ImmTySdwaDstSel: OS << "SdwaDstSel"; break;
676     case ImmTySdwaSrc0Sel: OS << "SdwaSrc0Sel"; break;
677     case ImmTySdwaSrc1Sel: OS << "SdwaSrc1Sel"; break;
678     case ImmTySdwaDstUnused: OS << "SdwaDstUnused"; break;
679     case ImmTyDMask: OS << "DMask"; break;
680     case ImmTyUNorm: OS << "UNorm"; break;
681     case ImmTyDA: OS << "DA"; break;
682     case ImmTyR128: OS << "R128"; break;
683     case ImmTyLWE: OS << "LWE"; break;
684     case ImmTyOff: OS << "Off"; break;
685     case ImmTyExpTgt: OS << "ExpTgt"; break;
686     case ImmTyExpCompr: OS << "ExpCompr"; break;
687     case ImmTyExpVM: OS << "ExpVM"; break;
688     case ImmTyHwreg: OS << "Hwreg"; break;
689     case ImmTySendMsg: OS << "SendMsg"; break;
690     case ImmTyInterpSlot: OS << "InterpSlot"; break;
691     case ImmTyInterpAttr: OS << "InterpAttr"; break;
692     case ImmTyAttrChan: OS << "AttrChan"; break;
693     case ImmTyOpSel: OS << "OpSel"; break;
694     case ImmTyOpSelHi: OS << "OpSelHi"; break;
695     case ImmTyNegLo: OS << "NegLo"; break;
696     case ImmTyNegHi: OS << "NegHi"; break;
697     case ImmTySwizzle: OS << "Swizzle"; break;
698     case ImmTyHigh: OS << "High"; break;
699     }
700   }
701 
702   void print(raw_ostream &OS) const override {
703     switch (Kind) {
704     case Register:
705       OS << "<register " << getReg() << " mods: " << Reg.Mods << '>';
706       break;
707     case Immediate:
708       OS << '<' << getImm();
709       if (getImmTy() != ImmTyNone) {
710         OS << " type: "; printImmTy(OS, getImmTy());
711       }
712       OS << " mods: " << Imm.Mods << '>';
713       break;
714     case Token:
715       OS << '\'' << getToken() << '\'';
716       break;
717     case Expression:
718       OS << "<expr " << *Expr << '>';
719       break;
720     }
721   }
722 
723   static AMDGPUOperand::Ptr CreateImm(const AMDGPUAsmParser *AsmParser,
724                                       int64_t Val, SMLoc Loc,
725                                       ImmTy Type = ImmTyNone,
726                                       bool IsFPImm = false) {
727     auto Op = llvm::make_unique<AMDGPUOperand>(Immediate, AsmParser);
728     Op->Imm.Val = Val;
729     Op->Imm.IsFPImm = IsFPImm;
730     Op->Imm.Type = Type;
731     Op->Imm.Mods = Modifiers();
732     Op->StartLoc = Loc;
733     Op->EndLoc = Loc;
734     return Op;
735   }
736 
737   static AMDGPUOperand::Ptr CreateToken(const AMDGPUAsmParser *AsmParser,
738                                         StringRef Str, SMLoc Loc,
739                                         bool HasExplicitEncodingSize = true) {
740     auto Res = llvm::make_unique<AMDGPUOperand>(Token, AsmParser);
741     Res->Tok.Data = Str.data();
742     Res->Tok.Length = Str.size();
743     Res->StartLoc = Loc;
744     Res->EndLoc = Loc;
745     return Res;
746   }
747 
748   static AMDGPUOperand::Ptr CreateReg(const AMDGPUAsmParser *AsmParser,
749                                       unsigned RegNo, SMLoc S,
750                                       SMLoc E,
751                                       bool ForceVOP3) {
752     auto Op = llvm::make_unique<AMDGPUOperand>(Register, AsmParser);
753     Op->Reg.RegNo = RegNo;
754     Op->Reg.Mods = Modifiers();
755     Op->Reg.IsForcedVOP3 = ForceVOP3;
756     Op->StartLoc = S;
757     Op->EndLoc = E;
758     return Op;
759   }
760 
761   static AMDGPUOperand::Ptr CreateExpr(const AMDGPUAsmParser *AsmParser,
762                                        const class MCExpr *Expr, SMLoc S) {
763     auto Op = llvm::make_unique<AMDGPUOperand>(Expression, AsmParser);
764     Op->Expr = Expr;
765     Op->StartLoc = S;
766     Op->EndLoc = S;
767     return Op;
768   }
769 };
770 
771 raw_ostream &operator <<(raw_ostream &OS, AMDGPUOperand::Modifiers Mods) {
772   OS << "abs:" << Mods.Abs << " neg: " << Mods.Neg << " sext:" << Mods.Sext;
773   return OS;
774 }
775 
776 //===----------------------------------------------------------------------===//
777 // AsmParser
778 //===----------------------------------------------------------------------===//
779 
780 // Holds info related to the current kernel, e.g. count of SGPRs used.
781 // Kernel scope begins at .amdgpu_hsa_kernel directive, ends at next
782 // .amdgpu_hsa_kernel or at EOF.
783 class KernelScopeInfo {
784   int SgprIndexUnusedMin = -1;
785   int VgprIndexUnusedMin = -1;
786   MCContext *Ctx = nullptr;
787 
788   void usesSgprAt(int i) {
789     if (i >= SgprIndexUnusedMin) {
790       SgprIndexUnusedMin = ++i;
791       if (Ctx) {
792         MCSymbol * const Sym = Ctx->getOrCreateSymbol(Twine(".kernel.sgpr_count"));
793         Sym->setVariableValue(MCConstantExpr::create(SgprIndexUnusedMin, *Ctx));
794       }
795     }
796   }
797 
798   void usesVgprAt(int i) {
799     if (i >= VgprIndexUnusedMin) {
800       VgprIndexUnusedMin = ++i;
801       if (Ctx) {
802         MCSymbol * const Sym = Ctx->getOrCreateSymbol(Twine(".kernel.vgpr_count"));
803         Sym->setVariableValue(MCConstantExpr::create(VgprIndexUnusedMin, *Ctx));
804       }
805     }
806   }
807 
808 public:
809   KernelScopeInfo() = default;
810 
811   void initialize(MCContext &Context) {
812     Ctx = &Context;
813     usesSgprAt(SgprIndexUnusedMin = -1);
814     usesVgprAt(VgprIndexUnusedMin = -1);
815   }
816 
817   void usesRegister(RegisterKind RegKind, unsigned DwordRegIndex, unsigned RegWidth) {
818     switch (RegKind) {
819       case IS_SGPR: usesSgprAt(DwordRegIndex + RegWidth - 1); break;
820       case IS_VGPR: usesVgprAt(DwordRegIndex + RegWidth - 1); break;
821       default: break;
822     }
823   }
824 };
825 
826 class AMDGPUAsmParser : public MCTargetAsmParser {
827   MCAsmParser &Parser;
828 
829   // Number of extra operands parsed after the first optional operand.
830   // This may be necessary to skip hardcoded mandatory operands.
831   static const unsigned MAX_OPR_LOOKAHEAD = 8;
832 
833   unsigned ForcedEncodingSize = 0;
834   bool ForcedDPP = false;
835   bool ForcedSDWA = false;
836   KernelScopeInfo KernelScope;
837 
838   /// @name Auto-generated Match Functions
839   /// {
840 
841 #define GET_ASSEMBLER_HEADER
842 #include "AMDGPUGenAsmMatcher.inc"
843 
844   /// }
845 
846 private:
847   bool ParseAsAbsoluteExpression(uint32_t &Ret);
848   bool ParseDirectiveMajorMinor(uint32_t &Major, uint32_t &Minor);
849   bool ParseDirectiveHSACodeObjectVersion();
850   bool ParseDirectiveHSACodeObjectISA();
851   bool ParseAMDKernelCodeTValue(StringRef ID, amd_kernel_code_t &Header);
852   bool ParseDirectiveAMDKernelCodeT();
853   bool subtargetHasRegister(const MCRegisterInfo &MRI, unsigned RegNo) const;
854   bool ParseDirectiveAMDGPUHsaKernel();
855 
856   bool ParseDirectiveISAVersion();
857   bool ParseDirectiveHSAMetadata();
858   bool ParseDirectivePALMetadata();
859 
860   bool AddNextRegisterToList(unsigned& Reg, unsigned& RegWidth,
861                              RegisterKind RegKind, unsigned Reg1,
862                              unsigned RegNum);
863   bool ParseAMDGPURegister(RegisterKind& RegKind, unsigned& Reg,
864                            unsigned& RegNum, unsigned& RegWidth,
865                            unsigned *DwordRegIndex);
866   void cvtMubufImpl(MCInst &Inst, const OperandVector &Operands,
867                     bool IsAtomic, bool IsAtomicReturn, bool IsLds = false);
868   void cvtDSImpl(MCInst &Inst, const OperandVector &Operands,
869                  bool IsGdsHardcoded);
870 
871 public:
872   enum AMDGPUMatchResultTy {
873     Match_PreferE32 = FIRST_TARGET_MATCH_RESULT_TY
874   };
875 
876   using OptionalImmIndexMap = std::map<AMDGPUOperand::ImmTy, unsigned>;
877 
878   AMDGPUAsmParser(const MCSubtargetInfo &STI, MCAsmParser &_Parser,
879                const MCInstrInfo &MII,
880                const MCTargetOptions &Options)
881       : MCTargetAsmParser(Options, STI, MII), Parser(_Parser) {
882     MCAsmParserExtension::Initialize(Parser);
883 
884     if (getFeatureBits().none()) {
885       // Set default features.
886       copySTI().ToggleFeature("SOUTHERN_ISLANDS");
887     }
888 
889     setAvailableFeatures(ComputeAvailableFeatures(getFeatureBits()));
890 
891     {
892       // TODO: make those pre-defined variables read-only.
893       // Currently there is none suitable machinery in the core llvm-mc for this.
894       // MCSymbol::isRedefinable is intended for another purpose, and
895       // AsmParser::parseDirectiveSet() cannot be specialized for specific target.
896       AMDGPU::IsaInfo::IsaVersion ISA =
897           AMDGPU::IsaInfo::getIsaVersion(getFeatureBits());
898       MCContext &Ctx = getContext();
899       MCSymbol *Sym =
900           Ctx.getOrCreateSymbol(Twine(".option.machine_version_major"));
901       Sym->setVariableValue(MCConstantExpr::create(ISA.Major, Ctx));
902       Sym = Ctx.getOrCreateSymbol(Twine(".option.machine_version_minor"));
903       Sym->setVariableValue(MCConstantExpr::create(ISA.Minor, Ctx));
904       Sym = Ctx.getOrCreateSymbol(Twine(".option.machine_version_stepping"));
905       Sym->setVariableValue(MCConstantExpr::create(ISA.Stepping, Ctx));
906     }
907     KernelScope.initialize(getContext());
908   }
909 
910   bool hasXNACK() const {
911     return AMDGPU::hasXNACK(getSTI());
912   }
913 
914   bool hasMIMG_R128() const {
915     return AMDGPU::hasMIMG_R128(getSTI());
916   }
917 
918   bool hasPackedD16() const {
919     return AMDGPU::hasPackedD16(getSTI());
920   }
921 
922   bool isSI() const {
923     return AMDGPU::isSI(getSTI());
924   }
925 
926   bool isCI() const {
927     return AMDGPU::isCI(getSTI());
928   }
929 
930   bool isVI() const {
931     return AMDGPU::isVI(getSTI());
932   }
933 
934   bool isGFX9() const {
935     return AMDGPU::isGFX9(getSTI());
936   }
937 
938   bool hasInv2PiInlineImm() const {
939     return getFeatureBits()[AMDGPU::FeatureInv2PiInlineImm];
940   }
941 
942   bool hasFlatOffsets() const {
943     return getFeatureBits()[AMDGPU::FeatureFlatInstOffsets];
944   }
945 
946   bool hasSGPR102_SGPR103() const {
947     return !isVI();
948   }
949 
950   bool hasIntClamp() const {
951     return getFeatureBits()[AMDGPU::FeatureIntClamp];
952   }
953 
954   AMDGPUTargetStreamer &getTargetStreamer() {
955     MCTargetStreamer &TS = *getParser().getStreamer().getTargetStreamer();
956     return static_cast<AMDGPUTargetStreamer &>(TS);
957   }
958 
959   const MCRegisterInfo *getMRI() const {
960     // We need this const_cast because for some reason getContext() is not const
961     // in MCAsmParser.
962     return const_cast<AMDGPUAsmParser*>(this)->getContext().getRegisterInfo();
963   }
964 
965   const MCInstrInfo *getMII() const {
966     return &MII;
967   }
968 
969   const FeatureBitset &getFeatureBits() const {
970     return getSTI().getFeatureBits();
971   }
972 
973   void setForcedEncodingSize(unsigned Size) { ForcedEncodingSize = Size; }
974   void setForcedDPP(bool ForceDPP_) { ForcedDPP = ForceDPP_; }
975   void setForcedSDWA(bool ForceSDWA_) { ForcedSDWA = ForceSDWA_; }
976 
977   unsigned getForcedEncodingSize() const { return ForcedEncodingSize; }
978   bool isForcedVOP3() const { return ForcedEncodingSize == 64; }
979   bool isForcedDPP() const { return ForcedDPP; }
980   bool isForcedSDWA() const { return ForcedSDWA; }
981   ArrayRef<unsigned> getMatchedVariants() const;
982 
983   std::unique_ptr<AMDGPUOperand> parseRegister();
984   bool ParseRegister(unsigned &RegNo, SMLoc &StartLoc, SMLoc &EndLoc) override;
985   unsigned checkTargetMatchPredicate(MCInst &Inst) override;
986   unsigned validateTargetOperandClass(MCParsedAsmOperand &Op,
987                                       unsigned Kind) override;
988   bool MatchAndEmitInstruction(SMLoc IDLoc, unsigned &Opcode,
989                                OperandVector &Operands, MCStreamer &Out,
990                                uint64_t &ErrorInfo,
991                                bool MatchingInlineAsm) override;
992   bool ParseDirective(AsmToken DirectiveID) override;
993   OperandMatchResultTy parseOperand(OperandVector &Operands, StringRef Mnemonic);
994   StringRef parseMnemonicSuffix(StringRef Name);
995   bool ParseInstruction(ParseInstructionInfo &Info, StringRef Name,
996                         SMLoc NameLoc, OperandVector &Operands) override;
997   //bool ProcessInstruction(MCInst &Inst);
998 
999   OperandMatchResultTy parseIntWithPrefix(const char *Prefix, int64_t &Int);
1000 
1001   OperandMatchResultTy
1002   parseIntWithPrefix(const char *Prefix, OperandVector &Operands,
1003                      AMDGPUOperand::ImmTy ImmTy = AMDGPUOperand::ImmTyNone,
1004                      bool (*ConvertResult)(int64_t &) = nullptr);
1005 
1006   OperandMatchResultTy parseOperandArrayWithPrefix(
1007     const char *Prefix,
1008     OperandVector &Operands,
1009     AMDGPUOperand::ImmTy ImmTy = AMDGPUOperand::ImmTyNone,
1010     bool (*ConvertResult)(int64_t&) = nullptr);
1011 
1012   OperandMatchResultTy
1013   parseNamedBit(const char *Name, OperandVector &Operands,
1014                 AMDGPUOperand::ImmTy ImmTy = AMDGPUOperand::ImmTyNone);
1015   OperandMatchResultTy parseStringWithPrefix(StringRef Prefix,
1016                                              StringRef &Value);
1017 
1018   bool parseAbsoluteExpr(int64_t &Val, bool AbsMod = false);
1019   OperandMatchResultTy parseImm(OperandVector &Operands, bool AbsMod = false);
1020   OperandMatchResultTy parseReg(OperandVector &Operands);
1021   OperandMatchResultTy parseRegOrImm(OperandVector &Operands, bool AbsMod = false);
1022   OperandMatchResultTy parseRegOrImmWithFPInputMods(OperandVector &Operands, bool AllowImm = true);
1023   OperandMatchResultTy parseRegOrImmWithIntInputMods(OperandVector &Operands, bool AllowImm = true);
1024   OperandMatchResultTy parseRegWithFPInputMods(OperandVector &Operands);
1025   OperandMatchResultTy parseRegWithIntInputMods(OperandVector &Operands);
1026   OperandMatchResultTy parseVReg32OrOff(OperandVector &Operands);
1027 
1028   void cvtDSOffset01(MCInst &Inst, const OperandVector &Operands);
1029   void cvtDS(MCInst &Inst, const OperandVector &Operands) { cvtDSImpl(Inst, Operands, false); }
1030   void cvtDSGds(MCInst &Inst, const OperandVector &Operands) { cvtDSImpl(Inst, Operands, true); }
1031   void cvtExp(MCInst &Inst, const OperandVector &Operands);
1032 
1033   bool parseCnt(int64_t &IntVal);
1034   OperandMatchResultTy parseSWaitCntOps(OperandVector &Operands);
1035   OperandMatchResultTy parseHwreg(OperandVector &Operands);
1036 
1037 private:
1038   struct OperandInfoTy {
1039     int64_t Id;
1040     bool IsSymbolic = false;
1041 
1042     OperandInfoTy(int64_t Id_) : Id(Id_) {}
1043   };
1044 
1045   bool parseSendMsgConstruct(OperandInfoTy &Msg, OperandInfoTy &Operation, int64_t &StreamId);
1046   bool parseHwregConstruct(OperandInfoTy &HwReg, int64_t &Offset, int64_t &Width);
1047 
1048   void errorExpTgt();
1049   OperandMatchResultTy parseExpTgtImpl(StringRef Str, uint8_t &Val);
1050 
1051   bool validateInstruction(const MCInst &Inst, const SMLoc &IDLoc);
1052   bool validateConstantBusLimitations(const MCInst &Inst);
1053   bool validateEarlyClobberLimitations(const MCInst &Inst);
1054   bool validateIntClampSupported(const MCInst &Inst);
1055   bool validateMIMGAtomicDMask(const MCInst &Inst);
1056   bool validateMIMGGatherDMask(const MCInst &Inst);
1057   bool validateMIMGDataSize(const MCInst &Inst);
1058   bool validateMIMGR128(const MCInst &Inst);
1059   bool validateMIMGD16(const MCInst &Inst);
1060   bool usesConstantBus(const MCInst &Inst, unsigned OpIdx);
1061   bool isInlineConstant(const MCInst &Inst, unsigned OpIdx) const;
1062   unsigned findImplicitSGPRReadInVOP(const MCInst &Inst) const;
1063 
1064   bool trySkipId(const StringRef Id);
1065   bool trySkipToken(const AsmToken::TokenKind Kind);
1066   bool skipToken(const AsmToken::TokenKind Kind, const StringRef ErrMsg);
1067   bool parseString(StringRef &Val, const StringRef ErrMsg = "expected a string");
1068   bool parseExpr(int64_t &Imm);
1069 
1070 public:
1071   OperandMatchResultTy parseOptionalOperand(OperandVector &Operands);
1072   OperandMatchResultTy parseOptionalOpr(OperandVector &Operands);
1073 
1074   OperandMatchResultTy parseExpTgt(OperandVector &Operands);
1075   OperandMatchResultTy parseSendMsgOp(OperandVector &Operands);
1076   OperandMatchResultTy parseInterpSlot(OperandVector &Operands);
1077   OperandMatchResultTy parseInterpAttr(OperandVector &Operands);
1078   OperandMatchResultTy parseSOppBrTarget(OperandVector &Operands);
1079 
1080   bool parseSwizzleOperands(const unsigned OpNum, int64_t* Op,
1081                             const unsigned MinVal,
1082                             const unsigned MaxVal,
1083                             const StringRef ErrMsg);
1084   OperandMatchResultTy parseSwizzleOp(OperandVector &Operands);
1085   bool parseSwizzleOffset(int64_t &Imm);
1086   bool parseSwizzleMacro(int64_t &Imm);
1087   bool parseSwizzleQuadPerm(int64_t &Imm);
1088   bool parseSwizzleBitmaskPerm(int64_t &Imm);
1089   bool parseSwizzleBroadcast(int64_t &Imm);
1090   bool parseSwizzleSwap(int64_t &Imm);
1091   bool parseSwizzleReverse(int64_t &Imm);
1092 
1093   void cvtMubuf(MCInst &Inst, const OperandVector &Operands) { cvtMubufImpl(Inst, Operands, false, false); }
1094   void cvtMubufAtomic(MCInst &Inst, const OperandVector &Operands) { cvtMubufImpl(Inst, Operands, true, false); }
1095   void cvtMubufAtomicReturn(MCInst &Inst, const OperandVector &Operands) { cvtMubufImpl(Inst, Operands, true, true); }
1096   void cvtMubufLds(MCInst &Inst, const OperandVector &Operands) { cvtMubufImpl(Inst, Operands, false, false, true); }
1097   void cvtMtbuf(MCInst &Inst, const OperandVector &Operands);
1098 
1099   AMDGPUOperand::Ptr defaultGLC() const;
1100   AMDGPUOperand::Ptr defaultSLC() const;
1101   AMDGPUOperand::Ptr defaultTFE() const;
1102 
1103   AMDGPUOperand::Ptr defaultD16() const;
1104   AMDGPUOperand::Ptr defaultDMask() const;
1105   AMDGPUOperand::Ptr defaultUNorm() const;
1106   AMDGPUOperand::Ptr defaultDA() const;
1107   AMDGPUOperand::Ptr defaultR128() const;
1108   AMDGPUOperand::Ptr defaultLWE() const;
1109   AMDGPUOperand::Ptr defaultSMRDOffset8() const;
1110   AMDGPUOperand::Ptr defaultSMRDOffset20() const;
1111   AMDGPUOperand::Ptr defaultSMRDLiteralOffset() const;
1112   AMDGPUOperand::Ptr defaultOffsetU12() const;
1113   AMDGPUOperand::Ptr defaultOffsetS13() const;
1114 
1115   OperandMatchResultTy parseOModOperand(OperandVector &Operands);
1116 
1117   void cvtVOP3(MCInst &Inst, const OperandVector &Operands,
1118                OptionalImmIndexMap &OptionalIdx);
1119   void cvtVOP3OpSel(MCInst &Inst, const OperandVector &Operands);
1120   void cvtVOP3(MCInst &Inst, const OperandVector &Operands);
1121   void cvtVOP3P(MCInst &Inst, const OperandVector &Operands);
1122 
1123   void cvtVOP3Interp(MCInst &Inst, const OperandVector &Operands);
1124 
1125   void cvtMIMG(MCInst &Inst, const OperandVector &Operands,
1126                bool IsAtomic = false);
1127   void cvtMIMGAtomic(MCInst &Inst, const OperandVector &Operands);
1128 
1129   OperandMatchResultTy parseDPPCtrl(OperandVector &Operands);
1130   AMDGPUOperand::Ptr defaultRowMask() const;
1131   AMDGPUOperand::Ptr defaultBankMask() const;
1132   AMDGPUOperand::Ptr defaultBoundCtrl() const;
1133   void cvtDPP(MCInst &Inst, const OperandVector &Operands);
1134 
1135   OperandMatchResultTy parseSDWASel(OperandVector &Operands, StringRef Prefix,
1136                                     AMDGPUOperand::ImmTy Type);
1137   OperandMatchResultTy parseSDWADstUnused(OperandVector &Operands);
1138   void cvtSdwaVOP1(MCInst &Inst, const OperandVector &Operands);
1139   void cvtSdwaVOP2(MCInst &Inst, const OperandVector &Operands);
1140   void cvtSdwaVOP2b(MCInst &Inst, const OperandVector &Operands);
1141   void cvtSdwaVOPC(MCInst &Inst, const OperandVector &Operands);
1142   void cvtSDWA(MCInst &Inst, const OperandVector &Operands,
1143                 uint64_t BasicInstType, bool skipVcc = false);
1144 };
1145 
1146 struct OptionalOperand {
1147   const char *Name;
1148   AMDGPUOperand::ImmTy Type;
1149   bool IsBit;
1150   bool (*ConvertResult)(int64_t&);
1151 };
1152 
1153 } // end anonymous namespace
1154 
1155 // May be called with integer type with equivalent bitwidth.
1156 static const fltSemantics *getFltSemantics(unsigned Size) {
1157   switch (Size) {
1158   case 4:
1159     return &APFloat::IEEEsingle();
1160   case 8:
1161     return &APFloat::IEEEdouble();
1162   case 2:
1163     return &APFloat::IEEEhalf();
1164   default:
1165     llvm_unreachable("unsupported fp type");
1166   }
1167 }
1168 
1169 static const fltSemantics *getFltSemantics(MVT VT) {
1170   return getFltSemantics(VT.getSizeInBits() / 8);
1171 }
1172 
1173 static const fltSemantics *getOpFltSemantics(uint8_t OperandType) {
1174   switch (OperandType) {
1175   case AMDGPU::OPERAND_REG_IMM_INT32:
1176   case AMDGPU::OPERAND_REG_IMM_FP32:
1177   case AMDGPU::OPERAND_REG_INLINE_C_INT32:
1178   case AMDGPU::OPERAND_REG_INLINE_C_FP32:
1179     return &APFloat::IEEEsingle();
1180   case AMDGPU::OPERAND_REG_IMM_INT64:
1181   case AMDGPU::OPERAND_REG_IMM_FP64:
1182   case AMDGPU::OPERAND_REG_INLINE_C_INT64:
1183   case AMDGPU::OPERAND_REG_INLINE_C_FP64:
1184     return &APFloat::IEEEdouble();
1185   case AMDGPU::OPERAND_REG_IMM_INT16:
1186   case AMDGPU::OPERAND_REG_IMM_FP16:
1187   case AMDGPU::OPERAND_REG_INLINE_C_INT16:
1188   case AMDGPU::OPERAND_REG_INLINE_C_FP16:
1189   case AMDGPU::OPERAND_REG_INLINE_C_V2INT16:
1190   case AMDGPU::OPERAND_REG_INLINE_C_V2FP16:
1191     return &APFloat::IEEEhalf();
1192   default:
1193     llvm_unreachable("unsupported fp type");
1194   }
1195 }
1196 
1197 //===----------------------------------------------------------------------===//
1198 // Operand
1199 //===----------------------------------------------------------------------===//
1200 
1201 static bool canLosslesslyConvertToFPType(APFloat &FPLiteral, MVT VT) {
1202   bool Lost;
1203 
1204   // Convert literal to single precision
1205   APFloat::opStatus Status = FPLiteral.convert(*getFltSemantics(VT),
1206                                                APFloat::rmNearestTiesToEven,
1207                                                &Lost);
1208   // We allow precision lost but not overflow or underflow
1209   if (Status != APFloat::opOK &&
1210       Lost &&
1211       ((Status & APFloat::opOverflow)  != 0 ||
1212        (Status & APFloat::opUnderflow) != 0)) {
1213     return false;
1214   }
1215 
1216   return true;
1217 }
1218 
1219 bool AMDGPUOperand::isInlinableImm(MVT type) const {
1220   if (!isImmTy(ImmTyNone)) {
1221     // Only plain immediates are inlinable (e.g. "clamp" attribute is not)
1222     return false;
1223   }
1224   // TODO: We should avoid using host float here. It would be better to
1225   // check the float bit values which is what a few other places do.
1226   // We've had bot failures before due to weird NaN support on mips hosts.
1227 
1228   APInt Literal(64, Imm.Val);
1229 
1230   if (Imm.IsFPImm) { // We got fp literal token
1231     if (type == MVT::f64 || type == MVT::i64) { // Expected 64-bit operand
1232       return AMDGPU::isInlinableLiteral64(Imm.Val,
1233                                           AsmParser->hasInv2PiInlineImm());
1234     }
1235 
1236     APFloat FPLiteral(APFloat::IEEEdouble(), APInt(64, Imm.Val));
1237     if (!canLosslesslyConvertToFPType(FPLiteral, type))
1238       return false;
1239 
1240     if (type.getScalarSizeInBits() == 16) {
1241       return AMDGPU::isInlinableLiteral16(
1242         static_cast<int16_t>(FPLiteral.bitcastToAPInt().getZExtValue()),
1243         AsmParser->hasInv2PiInlineImm());
1244     }
1245 
1246     // Check if single precision literal is inlinable
1247     return AMDGPU::isInlinableLiteral32(
1248       static_cast<int32_t>(FPLiteral.bitcastToAPInt().getZExtValue()),
1249       AsmParser->hasInv2PiInlineImm());
1250   }
1251 
1252   // We got int literal token.
1253   if (type == MVT::f64 || type == MVT::i64) { // Expected 64-bit operand
1254     return AMDGPU::isInlinableLiteral64(Imm.Val,
1255                                         AsmParser->hasInv2PiInlineImm());
1256   }
1257 
1258   if (type.getScalarSizeInBits() == 16) {
1259     return AMDGPU::isInlinableLiteral16(
1260       static_cast<int16_t>(Literal.getLoBits(16).getSExtValue()),
1261       AsmParser->hasInv2PiInlineImm());
1262   }
1263 
1264   return AMDGPU::isInlinableLiteral32(
1265     static_cast<int32_t>(Literal.getLoBits(32).getZExtValue()),
1266     AsmParser->hasInv2PiInlineImm());
1267 }
1268 
1269 bool AMDGPUOperand::isLiteralImm(MVT type) const {
1270   // Check that this immediate can be added as literal
1271   if (!isImmTy(ImmTyNone)) {
1272     return false;
1273   }
1274 
1275   if (!Imm.IsFPImm) {
1276     // We got int literal token.
1277 
1278     if (type == MVT::f64 && hasFPModifiers()) {
1279       // Cannot apply fp modifiers to int literals preserving the same semantics
1280       // for VOP1/2/C and VOP3 because of integer truncation. To avoid ambiguity,
1281       // disable these cases.
1282       return false;
1283     }
1284 
1285     unsigned Size = type.getSizeInBits();
1286     if (Size == 64)
1287       Size = 32;
1288 
1289     // FIXME: 64-bit operands can zero extend, sign extend, or pad zeroes for FP
1290     // types.
1291     return isUIntN(Size, Imm.Val) || isIntN(Size, Imm.Val);
1292   }
1293 
1294   // We got fp literal token
1295   if (type == MVT::f64) { // Expected 64-bit fp operand
1296     // We would set low 64-bits of literal to zeroes but we accept this literals
1297     return true;
1298   }
1299 
1300   if (type == MVT::i64) { // Expected 64-bit int operand
1301     // We don't allow fp literals in 64-bit integer instructions. It is
1302     // unclear how we should encode them.
1303     return false;
1304   }
1305 
1306   APFloat FPLiteral(APFloat::IEEEdouble(), APInt(64, Imm.Val));
1307   return canLosslesslyConvertToFPType(FPLiteral, type);
1308 }
1309 
1310 bool AMDGPUOperand::isRegClass(unsigned RCID) const {
1311   return isRegKind() && AsmParser->getMRI()->getRegClass(RCID).contains(getReg());
1312 }
1313 
1314 bool AMDGPUOperand::isSDWAOperand(MVT type) const {
1315   if (AsmParser->isVI())
1316     return isVReg();
1317   else if (AsmParser->isGFX9())
1318     return isRegKind() || isInlinableImm(type);
1319   else
1320     return false;
1321 }
1322 
1323 bool AMDGPUOperand::isSDWAFP16Operand() const {
1324   return isSDWAOperand(MVT::f16);
1325 }
1326 
1327 bool AMDGPUOperand::isSDWAFP32Operand() const {
1328   return isSDWAOperand(MVT::f32);
1329 }
1330 
1331 bool AMDGPUOperand::isSDWAInt16Operand() const {
1332   return isSDWAOperand(MVT::i16);
1333 }
1334 
1335 bool AMDGPUOperand::isSDWAInt32Operand() const {
1336   return isSDWAOperand(MVT::i32);
1337 }
1338 
1339 uint64_t AMDGPUOperand::applyInputFPModifiers(uint64_t Val, unsigned Size) const
1340 {
1341   assert(isImmTy(ImmTyNone) && Imm.Mods.hasFPModifiers());
1342   assert(Size == 2 || Size == 4 || Size == 8);
1343 
1344   const uint64_t FpSignMask = (1ULL << (Size * 8 - 1));
1345 
1346   if (Imm.Mods.Abs) {
1347     Val &= ~FpSignMask;
1348   }
1349   if (Imm.Mods.Neg) {
1350     Val ^= FpSignMask;
1351   }
1352 
1353   return Val;
1354 }
1355 
1356 void AMDGPUOperand::addImmOperands(MCInst &Inst, unsigned N, bool ApplyModifiers) const {
1357   if (AMDGPU::isSISrcOperand(AsmParser->getMII()->get(Inst.getOpcode()),
1358                              Inst.getNumOperands())) {
1359     addLiteralImmOperand(Inst, Imm.Val,
1360                          ApplyModifiers &
1361                          isImmTy(ImmTyNone) && Imm.Mods.hasFPModifiers());
1362   } else {
1363     assert(!isImmTy(ImmTyNone) || !hasModifiers());
1364     Inst.addOperand(MCOperand::createImm(Imm.Val));
1365   }
1366 }
1367 
1368 void AMDGPUOperand::addLiteralImmOperand(MCInst &Inst, int64_t Val, bool ApplyModifiers) const {
1369   const auto& InstDesc = AsmParser->getMII()->get(Inst.getOpcode());
1370   auto OpNum = Inst.getNumOperands();
1371   // Check that this operand accepts literals
1372   assert(AMDGPU::isSISrcOperand(InstDesc, OpNum));
1373 
1374   if (ApplyModifiers) {
1375     assert(AMDGPU::isSISrcFPOperand(InstDesc, OpNum));
1376     const unsigned Size = Imm.IsFPImm ? sizeof(double) : getOperandSize(InstDesc, OpNum);
1377     Val = applyInputFPModifiers(Val, Size);
1378   }
1379 
1380   APInt Literal(64, Val);
1381   uint8_t OpTy = InstDesc.OpInfo[OpNum].OperandType;
1382 
1383   if (Imm.IsFPImm) { // We got fp literal token
1384     switch (OpTy) {
1385     case AMDGPU::OPERAND_REG_IMM_INT64:
1386     case AMDGPU::OPERAND_REG_IMM_FP64:
1387     case AMDGPU::OPERAND_REG_INLINE_C_INT64:
1388     case AMDGPU::OPERAND_REG_INLINE_C_FP64:
1389       if (AMDGPU::isInlinableLiteral64(Literal.getZExtValue(),
1390                                        AsmParser->hasInv2PiInlineImm())) {
1391         Inst.addOperand(MCOperand::createImm(Literal.getZExtValue()));
1392         return;
1393       }
1394 
1395       // Non-inlineable
1396       if (AMDGPU::isSISrcFPOperand(InstDesc, OpNum)) { // Expected 64-bit fp operand
1397         // For fp operands we check if low 32 bits are zeros
1398         if (Literal.getLoBits(32) != 0) {
1399           const_cast<AMDGPUAsmParser *>(AsmParser)->Warning(Inst.getLoc(),
1400           "Can't encode literal as exact 64-bit floating-point operand. "
1401           "Low 32-bits will be set to zero");
1402         }
1403 
1404         Inst.addOperand(MCOperand::createImm(Literal.lshr(32).getZExtValue()));
1405         return;
1406       }
1407 
1408       // We don't allow fp literals in 64-bit integer instructions. It is
1409       // unclear how we should encode them. This case should be checked earlier
1410       // in predicate methods (isLiteralImm())
1411       llvm_unreachable("fp literal in 64-bit integer instruction.");
1412 
1413     case AMDGPU::OPERAND_REG_IMM_INT32:
1414     case AMDGPU::OPERAND_REG_IMM_FP32:
1415     case AMDGPU::OPERAND_REG_INLINE_C_INT32:
1416     case AMDGPU::OPERAND_REG_INLINE_C_FP32:
1417     case AMDGPU::OPERAND_REG_IMM_INT16:
1418     case AMDGPU::OPERAND_REG_IMM_FP16:
1419     case AMDGPU::OPERAND_REG_INLINE_C_INT16:
1420     case AMDGPU::OPERAND_REG_INLINE_C_FP16:
1421     case AMDGPU::OPERAND_REG_INLINE_C_V2INT16:
1422     case AMDGPU::OPERAND_REG_INLINE_C_V2FP16: {
1423       bool lost;
1424       APFloat FPLiteral(APFloat::IEEEdouble(), Literal);
1425       // Convert literal to single precision
1426       FPLiteral.convert(*getOpFltSemantics(OpTy),
1427                         APFloat::rmNearestTiesToEven, &lost);
1428       // We allow precision lost but not overflow or underflow. This should be
1429       // checked earlier in isLiteralImm()
1430 
1431       uint64_t ImmVal = FPLiteral.bitcastToAPInt().getZExtValue();
1432       if (OpTy == AMDGPU::OPERAND_REG_INLINE_C_V2INT16 ||
1433           OpTy == AMDGPU::OPERAND_REG_INLINE_C_V2FP16) {
1434         ImmVal |= (ImmVal << 16);
1435       }
1436 
1437       Inst.addOperand(MCOperand::createImm(ImmVal));
1438       return;
1439     }
1440     default:
1441       llvm_unreachable("invalid operand size");
1442     }
1443 
1444     return;
1445   }
1446 
1447    // We got int literal token.
1448   // Only sign extend inline immediates.
1449   // FIXME: No errors on truncation
1450   switch (OpTy) {
1451   case AMDGPU::OPERAND_REG_IMM_INT32:
1452   case AMDGPU::OPERAND_REG_IMM_FP32:
1453   case AMDGPU::OPERAND_REG_INLINE_C_INT32:
1454   case AMDGPU::OPERAND_REG_INLINE_C_FP32:
1455     if (isInt<32>(Val) &&
1456         AMDGPU::isInlinableLiteral32(static_cast<int32_t>(Val),
1457                                      AsmParser->hasInv2PiInlineImm())) {
1458       Inst.addOperand(MCOperand::createImm(Val));
1459       return;
1460     }
1461 
1462     Inst.addOperand(MCOperand::createImm(Val & 0xffffffff));
1463     return;
1464 
1465   case AMDGPU::OPERAND_REG_IMM_INT64:
1466   case AMDGPU::OPERAND_REG_IMM_FP64:
1467   case AMDGPU::OPERAND_REG_INLINE_C_INT64:
1468   case AMDGPU::OPERAND_REG_INLINE_C_FP64:
1469     if (AMDGPU::isInlinableLiteral64(Val, AsmParser->hasInv2PiInlineImm())) {
1470       Inst.addOperand(MCOperand::createImm(Val));
1471       return;
1472     }
1473 
1474     Inst.addOperand(MCOperand::createImm(Lo_32(Val)));
1475     return;
1476 
1477   case AMDGPU::OPERAND_REG_IMM_INT16:
1478   case AMDGPU::OPERAND_REG_IMM_FP16:
1479   case AMDGPU::OPERAND_REG_INLINE_C_INT16:
1480   case AMDGPU::OPERAND_REG_INLINE_C_FP16:
1481     if (isInt<16>(Val) &&
1482         AMDGPU::isInlinableLiteral16(static_cast<int16_t>(Val),
1483                                      AsmParser->hasInv2PiInlineImm())) {
1484       Inst.addOperand(MCOperand::createImm(Val));
1485       return;
1486     }
1487 
1488     Inst.addOperand(MCOperand::createImm(Val & 0xffff));
1489     return;
1490 
1491   case AMDGPU::OPERAND_REG_INLINE_C_V2INT16:
1492   case AMDGPU::OPERAND_REG_INLINE_C_V2FP16: {
1493     auto LiteralVal = static_cast<uint16_t>(Literal.getLoBits(16).getZExtValue());
1494     assert(AMDGPU::isInlinableLiteral16(LiteralVal,
1495                                         AsmParser->hasInv2PiInlineImm()));
1496 
1497     uint32_t ImmVal = static_cast<uint32_t>(LiteralVal) << 16 |
1498                       static_cast<uint32_t>(LiteralVal);
1499     Inst.addOperand(MCOperand::createImm(ImmVal));
1500     return;
1501   }
1502   default:
1503     llvm_unreachable("invalid operand size");
1504   }
1505 }
1506 
1507 template <unsigned Bitwidth>
1508 void AMDGPUOperand::addKImmFPOperands(MCInst &Inst, unsigned N) const {
1509   APInt Literal(64, Imm.Val);
1510 
1511   if (!Imm.IsFPImm) {
1512     // We got int literal token.
1513     Inst.addOperand(MCOperand::createImm(Literal.getLoBits(Bitwidth).getZExtValue()));
1514     return;
1515   }
1516 
1517   bool Lost;
1518   APFloat FPLiteral(APFloat::IEEEdouble(), Literal);
1519   FPLiteral.convert(*getFltSemantics(Bitwidth / 8),
1520                     APFloat::rmNearestTiesToEven, &Lost);
1521   Inst.addOperand(MCOperand::createImm(FPLiteral.bitcastToAPInt().getZExtValue()));
1522 }
1523 
1524 void AMDGPUOperand::addRegOperands(MCInst &Inst, unsigned N) const {
1525   Inst.addOperand(MCOperand::createReg(AMDGPU::getMCReg(getReg(), AsmParser->getSTI())));
1526 }
1527 
1528 //===----------------------------------------------------------------------===//
1529 // AsmParser
1530 //===----------------------------------------------------------------------===//
1531 
1532 static int getRegClass(RegisterKind Is, unsigned RegWidth) {
1533   if (Is == IS_VGPR) {
1534     switch (RegWidth) {
1535       default: return -1;
1536       case 1: return AMDGPU::VGPR_32RegClassID;
1537       case 2: return AMDGPU::VReg_64RegClassID;
1538       case 3: return AMDGPU::VReg_96RegClassID;
1539       case 4: return AMDGPU::VReg_128RegClassID;
1540       case 8: return AMDGPU::VReg_256RegClassID;
1541       case 16: return AMDGPU::VReg_512RegClassID;
1542     }
1543   } else if (Is == IS_TTMP) {
1544     switch (RegWidth) {
1545       default: return -1;
1546       case 1: return AMDGPU::TTMP_32RegClassID;
1547       case 2: return AMDGPU::TTMP_64RegClassID;
1548       case 4: return AMDGPU::TTMP_128RegClassID;
1549       case 8: return AMDGPU::TTMP_256RegClassID;
1550       case 16: return AMDGPU::TTMP_512RegClassID;
1551     }
1552   } else if (Is == IS_SGPR) {
1553     switch (RegWidth) {
1554       default: return -1;
1555       case 1: return AMDGPU::SGPR_32RegClassID;
1556       case 2: return AMDGPU::SGPR_64RegClassID;
1557       case 4: return AMDGPU::SGPR_128RegClassID;
1558       case 8: return AMDGPU::SGPR_256RegClassID;
1559       case 16: return AMDGPU::SGPR_512RegClassID;
1560     }
1561   }
1562   return -1;
1563 }
1564 
1565 static unsigned getSpecialRegForName(StringRef RegName) {
1566   return StringSwitch<unsigned>(RegName)
1567     .Case("exec", AMDGPU::EXEC)
1568     .Case("vcc", AMDGPU::VCC)
1569     .Case("flat_scratch", AMDGPU::FLAT_SCR)
1570     .Case("xnack_mask", AMDGPU::XNACK_MASK)
1571     .Case("m0", AMDGPU::M0)
1572     .Case("scc", AMDGPU::SCC)
1573     .Case("tba", AMDGPU::TBA)
1574     .Case("tma", AMDGPU::TMA)
1575     .Case("flat_scratch_lo", AMDGPU::FLAT_SCR_LO)
1576     .Case("flat_scratch_hi", AMDGPU::FLAT_SCR_HI)
1577     .Case("xnack_mask_lo", AMDGPU::XNACK_MASK_LO)
1578     .Case("xnack_mask_hi", AMDGPU::XNACK_MASK_HI)
1579     .Case("vcc_lo", AMDGPU::VCC_LO)
1580     .Case("vcc_hi", AMDGPU::VCC_HI)
1581     .Case("exec_lo", AMDGPU::EXEC_LO)
1582     .Case("exec_hi", AMDGPU::EXEC_HI)
1583     .Case("tma_lo", AMDGPU::TMA_LO)
1584     .Case("tma_hi", AMDGPU::TMA_HI)
1585     .Case("tba_lo", AMDGPU::TBA_LO)
1586     .Case("tba_hi", AMDGPU::TBA_HI)
1587     .Default(0);
1588 }
1589 
1590 bool AMDGPUAsmParser::ParseRegister(unsigned &RegNo, SMLoc &StartLoc,
1591                                     SMLoc &EndLoc) {
1592   auto R = parseRegister();
1593   if (!R) return true;
1594   assert(R->isReg());
1595   RegNo = R->getReg();
1596   StartLoc = R->getStartLoc();
1597   EndLoc = R->getEndLoc();
1598   return false;
1599 }
1600 
1601 bool AMDGPUAsmParser::AddNextRegisterToList(unsigned &Reg, unsigned &RegWidth,
1602                                             RegisterKind RegKind, unsigned Reg1,
1603                                             unsigned RegNum) {
1604   switch (RegKind) {
1605   case IS_SPECIAL:
1606     if (Reg == AMDGPU::EXEC_LO && Reg1 == AMDGPU::EXEC_HI) {
1607       Reg = AMDGPU::EXEC;
1608       RegWidth = 2;
1609       return true;
1610     }
1611     if (Reg == AMDGPU::FLAT_SCR_LO && Reg1 == AMDGPU::FLAT_SCR_HI) {
1612       Reg = AMDGPU::FLAT_SCR;
1613       RegWidth = 2;
1614       return true;
1615     }
1616     if (Reg == AMDGPU::XNACK_MASK_LO && Reg1 == AMDGPU::XNACK_MASK_HI) {
1617       Reg = AMDGPU::XNACK_MASK;
1618       RegWidth = 2;
1619       return true;
1620     }
1621     if (Reg == AMDGPU::VCC_LO && Reg1 == AMDGPU::VCC_HI) {
1622       Reg = AMDGPU::VCC;
1623       RegWidth = 2;
1624       return true;
1625     }
1626     if (Reg == AMDGPU::TBA_LO && Reg1 == AMDGPU::TBA_HI) {
1627       Reg = AMDGPU::TBA;
1628       RegWidth = 2;
1629       return true;
1630     }
1631     if (Reg == AMDGPU::TMA_LO && Reg1 == AMDGPU::TMA_HI) {
1632       Reg = AMDGPU::TMA;
1633       RegWidth = 2;
1634       return true;
1635     }
1636     return false;
1637   case IS_VGPR:
1638   case IS_SGPR:
1639   case IS_TTMP:
1640     if (Reg1 != Reg + RegWidth) {
1641       return false;
1642     }
1643     RegWidth++;
1644     return true;
1645   default:
1646     llvm_unreachable("unexpected register kind");
1647   }
1648 }
1649 
1650 bool AMDGPUAsmParser::ParseAMDGPURegister(RegisterKind &RegKind, unsigned &Reg,
1651                                           unsigned &RegNum, unsigned &RegWidth,
1652                                           unsigned *DwordRegIndex) {
1653   if (DwordRegIndex) { *DwordRegIndex = 0; }
1654   const MCRegisterInfo *TRI = getContext().getRegisterInfo();
1655   if (getLexer().is(AsmToken::Identifier)) {
1656     StringRef RegName = Parser.getTok().getString();
1657     if ((Reg = getSpecialRegForName(RegName))) {
1658       Parser.Lex();
1659       RegKind = IS_SPECIAL;
1660     } else {
1661       unsigned RegNumIndex = 0;
1662       if (RegName[0] == 'v') {
1663         RegNumIndex = 1;
1664         RegKind = IS_VGPR;
1665       } else if (RegName[0] == 's') {
1666         RegNumIndex = 1;
1667         RegKind = IS_SGPR;
1668       } else if (RegName.startswith("ttmp")) {
1669         RegNumIndex = strlen("ttmp");
1670         RegKind = IS_TTMP;
1671       } else {
1672         return false;
1673       }
1674       if (RegName.size() > RegNumIndex) {
1675         // Single 32-bit register: vXX.
1676         if (RegName.substr(RegNumIndex).getAsInteger(10, RegNum))
1677           return false;
1678         Parser.Lex();
1679         RegWidth = 1;
1680       } else {
1681         // Range of registers: v[XX:YY]. ":YY" is optional.
1682         Parser.Lex();
1683         int64_t RegLo, RegHi;
1684         if (getLexer().isNot(AsmToken::LBrac))
1685           return false;
1686         Parser.Lex();
1687 
1688         if (getParser().parseAbsoluteExpression(RegLo))
1689           return false;
1690 
1691         const bool isRBrace = getLexer().is(AsmToken::RBrac);
1692         if (!isRBrace && getLexer().isNot(AsmToken::Colon))
1693           return false;
1694         Parser.Lex();
1695 
1696         if (isRBrace) {
1697           RegHi = RegLo;
1698         } else {
1699           if (getParser().parseAbsoluteExpression(RegHi))
1700             return false;
1701 
1702           if (getLexer().isNot(AsmToken::RBrac))
1703             return false;
1704           Parser.Lex();
1705         }
1706         RegNum = (unsigned) RegLo;
1707         RegWidth = (RegHi - RegLo) + 1;
1708       }
1709     }
1710   } else if (getLexer().is(AsmToken::LBrac)) {
1711     // List of consecutive registers: [s0,s1,s2,s3]
1712     Parser.Lex();
1713     if (!ParseAMDGPURegister(RegKind, Reg, RegNum, RegWidth, nullptr))
1714       return false;
1715     if (RegWidth != 1)
1716       return false;
1717     RegisterKind RegKind1;
1718     unsigned Reg1, RegNum1, RegWidth1;
1719     do {
1720       if (getLexer().is(AsmToken::Comma)) {
1721         Parser.Lex();
1722       } else if (getLexer().is(AsmToken::RBrac)) {
1723         Parser.Lex();
1724         break;
1725       } else if (ParseAMDGPURegister(RegKind1, Reg1, RegNum1, RegWidth1, nullptr)) {
1726         if (RegWidth1 != 1) {
1727           return false;
1728         }
1729         if (RegKind1 != RegKind) {
1730           return false;
1731         }
1732         if (!AddNextRegisterToList(Reg, RegWidth, RegKind1, Reg1, RegNum1)) {
1733           return false;
1734         }
1735       } else {
1736         return false;
1737       }
1738     } while (true);
1739   } else {
1740     return false;
1741   }
1742   switch (RegKind) {
1743   case IS_SPECIAL:
1744     RegNum = 0;
1745     RegWidth = 1;
1746     break;
1747   case IS_VGPR:
1748   case IS_SGPR:
1749   case IS_TTMP:
1750   {
1751     unsigned Size = 1;
1752     if (RegKind == IS_SGPR || RegKind == IS_TTMP) {
1753       // SGPR and TTMP registers must be aligned. Max required alignment is 4 dwords.
1754       Size = std::min(RegWidth, 4u);
1755     }
1756     if (RegNum % Size != 0)
1757       return false;
1758     if (DwordRegIndex) { *DwordRegIndex = RegNum; }
1759     RegNum = RegNum / Size;
1760     int RCID = getRegClass(RegKind, RegWidth);
1761     if (RCID == -1)
1762       return false;
1763     const MCRegisterClass RC = TRI->getRegClass(RCID);
1764     if (RegNum >= RC.getNumRegs())
1765       return false;
1766     Reg = RC.getRegister(RegNum);
1767     break;
1768   }
1769 
1770   default:
1771     llvm_unreachable("unexpected register kind");
1772   }
1773 
1774   if (!subtargetHasRegister(*TRI, Reg))
1775     return false;
1776   return true;
1777 }
1778 
1779 std::unique_ptr<AMDGPUOperand> AMDGPUAsmParser::parseRegister() {
1780   const auto &Tok = Parser.getTok();
1781   SMLoc StartLoc = Tok.getLoc();
1782   SMLoc EndLoc = Tok.getEndLoc();
1783   RegisterKind RegKind;
1784   unsigned Reg, RegNum, RegWidth, DwordRegIndex;
1785 
1786   if (!ParseAMDGPURegister(RegKind, Reg, RegNum, RegWidth, &DwordRegIndex)) {
1787     return nullptr;
1788   }
1789   KernelScope.usesRegister(RegKind, DwordRegIndex, RegWidth);
1790   return AMDGPUOperand::CreateReg(this, Reg, StartLoc, EndLoc, false);
1791 }
1792 
1793 bool
1794 AMDGPUAsmParser::parseAbsoluteExpr(int64_t &Val, bool AbsMod) {
1795   if (AbsMod && getLexer().peekTok().is(AsmToken::Pipe) &&
1796       (getLexer().getKind() == AsmToken::Integer ||
1797        getLexer().getKind() == AsmToken::Real)) {
1798     // This is a workaround for handling operands like these:
1799     //     |1.0|
1800     //     |-1|
1801     // This syntax is not compatible with syntax of standard
1802     // MC expressions (due to the trailing '|').
1803 
1804     SMLoc EndLoc;
1805     const MCExpr *Expr;
1806 
1807     if (getParser().parsePrimaryExpr(Expr, EndLoc)) {
1808       return true;
1809     }
1810 
1811     return !Expr->evaluateAsAbsolute(Val);
1812   }
1813 
1814   return getParser().parseAbsoluteExpression(Val);
1815 }
1816 
1817 OperandMatchResultTy
1818 AMDGPUAsmParser::parseImm(OperandVector &Operands, bool AbsMod) {
1819   // TODO: add syntactic sugar for 1/(2*PI)
1820   bool Minus = false;
1821   if (getLexer().getKind() == AsmToken::Minus) {
1822     const AsmToken NextToken = getLexer().peekTok();
1823     if (!NextToken.is(AsmToken::Integer) &&
1824         !NextToken.is(AsmToken::Real)) {
1825         return MatchOperand_NoMatch;
1826     }
1827     Minus = true;
1828     Parser.Lex();
1829   }
1830 
1831   SMLoc S = Parser.getTok().getLoc();
1832   switch(getLexer().getKind()) {
1833   case AsmToken::Integer: {
1834     int64_t IntVal;
1835     if (parseAbsoluteExpr(IntVal, AbsMod))
1836       return MatchOperand_ParseFail;
1837     if (Minus)
1838       IntVal *= -1;
1839     Operands.push_back(AMDGPUOperand::CreateImm(this, IntVal, S));
1840     return MatchOperand_Success;
1841   }
1842   case AsmToken::Real: {
1843     int64_t IntVal;
1844     if (parseAbsoluteExpr(IntVal, AbsMod))
1845       return MatchOperand_ParseFail;
1846 
1847     APFloat F(BitsToDouble(IntVal));
1848     if (Minus)
1849       F.changeSign();
1850     Operands.push_back(
1851         AMDGPUOperand::CreateImm(this, F.bitcastToAPInt().getZExtValue(), S,
1852                                  AMDGPUOperand::ImmTyNone, true));
1853     return MatchOperand_Success;
1854   }
1855   default:
1856     return MatchOperand_NoMatch;
1857   }
1858 }
1859 
1860 OperandMatchResultTy
1861 AMDGPUAsmParser::parseReg(OperandVector &Operands) {
1862   if (auto R = parseRegister()) {
1863     assert(R->isReg());
1864     R->Reg.IsForcedVOP3 = isForcedVOP3();
1865     Operands.push_back(std::move(R));
1866     return MatchOperand_Success;
1867   }
1868   return MatchOperand_NoMatch;
1869 }
1870 
1871 OperandMatchResultTy
1872 AMDGPUAsmParser::parseRegOrImm(OperandVector &Operands, bool AbsMod) {
1873   auto res = parseImm(Operands, AbsMod);
1874   if (res != MatchOperand_NoMatch) {
1875     return res;
1876   }
1877 
1878   return parseReg(Operands);
1879 }
1880 
1881 OperandMatchResultTy
1882 AMDGPUAsmParser::parseRegOrImmWithFPInputMods(OperandVector &Operands,
1883                                               bool AllowImm) {
1884   bool Negate = false, Negate2 = false, Abs = false, Abs2 = false;
1885 
1886   if (getLexer().getKind()== AsmToken::Minus) {
1887     const AsmToken NextToken = getLexer().peekTok();
1888 
1889     // Disable ambiguous constructs like '--1' etc. Should use neg(-1) instead.
1890     if (NextToken.is(AsmToken::Minus)) {
1891       Error(Parser.getTok().getLoc(), "invalid syntax, expected 'neg' modifier");
1892       return MatchOperand_ParseFail;
1893     }
1894 
1895     // '-' followed by an integer literal N should be interpreted as integer
1896     // negation rather than a floating-point NEG modifier applied to N.
1897     // Beside being contr-intuitive, such use of floating-point NEG modifier
1898     // results in different meaning of integer literals used with VOP1/2/C
1899     // and VOP3, for example:
1900     //    v_exp_f32_e32 v5, -1 // VOP1: src0 = 0xFFFFFFFF
1901     //    v_exp_f32_e64 v5, -1 // VOP3: src0 = 0x80000001
1902     // Negative fp literals should be handled likewise for unifomtity
1903     if (!NextToken.is(AsmToken::Integer) && !NextToken.is(AsmToken::Real)) {
1904       Parser.Lex();
1905       Negate = true;
1906     }
1907   }
1908 
1909   if (getLexer().getKind() == AsmToken::Identifier &&
1910       Parser.getTok().getString() == "neg") {
1911     if (Negate) {
1912       Error(Parser.getTok().getLoc(), "expected register or immediate");
1913       return MatchOperand_ParseFail;
1914     }
1915     Parser.Lex();
1916     Negate2 = true;
1917     if (getLexer().isNot(AsmToken::LParen)) {
1918       Error(Parser.getTok().getLoc(), "expected left paren after neg");
1919       return MatchOperand_ParseFail;
1920     }
1921     Parser.Lex();
1922   }
1923 
1924   if (getLexer().getKind() == AsmToken::Identifier &&
1925       Parser.getTok().getString() == "abs") {
1926     Parser.Lex();
1927     Abs2 = true;
1928     if (getLexer().isNot(AsmToken::LParen)) {
1929       Error(Parser.getTok().getLoc(), "expected left paren after abs");
1930       return MatchOperand_ParseFail;
1931     }
1932     Parser.Lex();
1933   }
1934 
1935   if (getLexer().getKind() == AsmToken::Pipe) {
1936     if (Abs2) {
1937       Error(Parser.getTok().getLoc(), "expected register or immediate");
1938       return MatchOperand_ParseFail;
1939     }
1940     Parser.Lex();
1941     Abs = true;
1942   }
1943 
1944   OperandMatchResultTy Res;
1945   if (AllowImm) {
1946     Res = parseRegOrImm(Operands, Abs);
1947   } else {
1948     Res = parseReg(Operands);
1949   }
1950   if (Res != MatchOperand_Success) {
1951     return Res;
1952   }
1953 
1954   AMDGPUOperand::Modifiers Mods;
1955   if (Abs) {
1956     if (getLexer().getKind() != AsmToken::Pipe) {
1957       Error(Parser.getTok().getLoc(), "expected vertical bar");
1958       return MatchOperand_ParseFail;
1959     }
1960     Parser.Lex();
1961     Mods.Abs = true;
1962   }
1963   if (Abs2) {
1964     if (getLexer().isNot(AsmToken::RParen)) {
1965       Error(Parser.getTok().getLoc(), "expected closing parentheses");
1966       return MatchOperand_ParseFail;
1967     }
1968     Parser.Lex();
1969     Mods.Abs = true;
1970   }
1971 
1972   if (Negate) {
1973     Mods.Neg = true;
1974   } else if (Negate2) {
1975     if (getLexer().isNot(AsmToken::RParen)) {
1976       Error(Parser.getTok().getLoc(), "expected closing parentheses");
1977       return MatchOperand_ParseFail;
1978     }
1979     Parser.Lex();
1980     Mods.Neg = true;
1981   }
1982 
1983   if (Mods.hasFPModifiers()) {
1984     AMDGPUOperand &Op = static_cast<AMDGPUOperand &>(*Operands.back());
1985     Op.setModifiers(Mods);
1986   }
1987   return MatchOperand_Success;
1988 }
1989 
1990 OperandMatchResultTy
1991 AMDGPUAsmParser::parseRegOrImmWithIntInputMods(OperandVector &Operands,
1992                                                bool AllowImm) {
1993   bool Sext = false;
1994 
1995   if (getLexer().getKind() == AsmToken::Identifier &&
1996       Parser.getTok().getString() == "sext") {
1997     Parser.Lex();
1998     Sext = true;
1999     if (getLexer().isNot(AsmToken::LParen)) {
2000       Error(Parser.getTok().getLoc(), "expected left paren after sext");
2001       return MatchOperand_ParseFail;
2002     }
2003     Parser.Lex();
2004   }
2005 
2006   OperandMatchResultTy Res;
2007   if (AllowImm) {
2008     Res = parseRegOrImm(Operands);
2009   } else {
2010     Res = parseReg(Operands);
2011   }
2012   if (Res != MatchOperand_Success) {
2013     return Res;
2014   }
2015 
2016   AMDGPUOperand::Modifiers Mods;
2017   if (Sext) {
2018     if (getLexer().isNot(AsmToken::RParen)) {
2019       Error(Parser.getTok().getLoc(), "expected closing parentheses");
2020       return MatchOperand_ParseFail;
2021     }
2022     Parser.Lex();
2023     Mods.Sext = true;
2024   }
2025 
2026   if (Mods.hasIntModifiers()) {
2027     AMDGPUOperand &Op = static_cast<AMDGPUOperand &>(*Operands.back());
2028     Op.setModifiers(Mods);
2029   }
2030 
2031   return MatchOperand_Success;
2032 }
2033 
2034 OperandMatchResultTy
2035 AMDGPUAsmParser::parseRegWithFPInputMods(OperandVector &Operands) {
2036   return parseRegOrImmWithFPInputMods(Operands, false);
2037 }
2038 
2039 OperandMatchResultTy
2040 AMDGPUAsmParser::parseRegWithIntInputMods(OperandVector &Operands) {
2041   return parseRegOrImmWithIntInputMods(Operands, false);
2042 }
2043 
2044 OperandMatchResultTy AMDGPUAsmParser::parseVReg32OrOff(OperandVector &Operands) {
2045   std::unique_ptr<AMDGPUOperand> Reg = parseRegister();
2046   if (Reg) {
2047     Operands.push_back(std::move(Reg));
2048     return MatchOperand_Success;
2049   }
2050 
2051   const AsmToken &Tok = Parser.getTok();
2052   if (Tok.getString() == "off") {
2053     Operands.push_back(AMDGPUOperand::CreateImm(this, 0, Tok.getLoc(),
2054                                                 AMDGPUOperand::ImmTyOff, false));
2055     Parser.Lex();
2056     return MatchOperand_Success;
2057   }
2058 
2059   return MatchOperand_NoMatch;
2060 }
2061 
2062 unsigned AMDGPUAsmParser::checkTargetMatchPredicate(MCInst &Inst) {
2063   uint64_t TSFlags = MII.get(Inst.getOpcode()).TSFlags;
2064 
2065   if ((getForcedEncodingSize() == 32 && (TSFlags & SIInstrFlags::VOP3)) ||
2066       (getForcedEncodingSize() == 64 && !(TSFlags & SIInstrFlags::VOP3)) ||
2067       (isForcedDPP() && !(TSFlags & SIInstrFlags::DPP)) ||
2068       (isForcedSDWA() && !(TSFlags & SIInstrFlags::SDWA)) )
2069     return Match_InvalidOperand;
2070 
2071   if ((TSFlags & SIInstrFlags::VOP3) &&
2072       (TSFlags & SIInstrFlags::VOPAsmPrefer32Bit) &&
2073       getForcedEncodingSize() != 64)
2074     return Match_PreferE32;
2075 
2076   if (Inst.getOpcode() == AMDGPU::V_MAC_F32_sdwa_vi ||
2077       Inst.getOpcode() == AMDGPU::V_MAC_F16_sdwa_vi) {
2078     // v_mac_f32/16 allow only dst_sel == DWORD;
2079     auto OpNum =
2080         AMDGPU::getNamedOperandIdx(Inst.getOpcode(), AMDGPU::OpName::dst_sel);
2081     const auto &Op = Inst.getOperand(OpNum);
2082     if (!Op.isImm() || Op.getImm() != AMDGPU::SDWA::SdwaSel::DWORD) {
2083       return Match_InvalidOperand;
2084     }
2085   }
2086 
2087   if ((TSFlags & SIInstrFlags::FLAT) && !hasFlatOffsets()) {
2088     // FIXME: Produces error without correct column reported.
2089     auto OpNum =
2090         AMDGPU::getNamedOperandIdx(Inst.getOpcode(), AMDGPU::OpName::offset);
2091     const auto &Op = Inst.getOperand(OpNum);
2092     if (Op.getImm() != 0)
2093       return Match_InvalidOperand;
2094   }
2095 
2096   return Match_Success;
2097 }
2098 
2099 // What asm variants we should check
2100 ArrayRef<unsigned> AMDGPUAsmParser::getMatchedVariants() const {
2101   if (getForcedEncodingSize() == 32) {
2102     static const unsigned Variants[] = {AMDGPUAsmVariants::DEFAULT};
2103     return makeArrayRef(Variants);
2104   }
2105 
2106   if (isForcedVOP3()) {
2107     static const unsigned Variants[] = {AMDGPUAsmVariants::VOP3};
2108     return makeArrayRef(Variants);
2109   }
2110 
2111   if (isForcedSDWA()) {
2112     static const unsigned Variants[] = {AMDGPUAsmVariants::SDWA,
2113                                         AMDGPUAsmVariants::SDWA9};
2114     return makeArrayRef(Variants);
2115   }
2116 
2117   if (isForcedDPP()) {
2118     static const unsigned Variants[] = {AMDGPUAsmVariants::DPP};
2119     return makeArrayRef(Variants);
2120   }
2121 
2122   static const unsigned Variants[] = {
2123     AMDGPUAsmVariants::DEFAULT, AMDGPUAsmVariants::VOP3,
2124     AMDGPUAsmVariants::SDWA, AMDGPUAsmVariants::SDWA9, AMDGPUAsmVariants::DPP
2125   };
2126 
2127   return makeArrayRef(Variants);
2128 }
2129 
2130 unsigned AMDGPUAsmParser::findImplicitSGPRReadInVOP(const MCInst &Inst) const {
2131   const MCInstrDesc &Desc = MII.get(Inst.getOpcode());
2132   const unsigned Num = Desc.getNumImplicitUses();
2133   for (unsigned i = 0; i < Num; ++i) {
2134     unsigned Reg = Desc.ImplicitUses[i];
2135     switch (Reg) {
2136     case AMDGPU::FLAT_SCR:
2137     case AMDGPU::VCC:
2138     case AMDGPU::M0:
2139       return Reg;
2140     default:
2141       break;
2142     }
2143   }
2144   return AMDGPU::NoRegister;
2145 }
2146 
2147 // NB: This code is correct only when used to check constant
2148 // bus limitations because GFX7 support no f16 inline constants.
2149 // Note that there are no cases when a GFX7 opcode violates
2150 // constant bus limitations due to the use of an f16 constant.
2151 bool AMDGPUAsmParser::isInlineConstant(const MCInst &Inst,
2152                                        unsigned OpIdx) const {
2153   const MCInstrDesc &Desc = MII.get(Inst.getOpcode());
2154 
2155   if (!AMDGPU::isSISrcOperand(Desc, OpIdx)) {
2156     return false;
2157   }
2158 
2159   const MCOperand &MO = Inst.getOperand(OpIdx);
2160 
2161   int64_t Val = MO.getImm();
2162   auto OpSize = AMDGPU::getOperandSize(Desc, OpIdx);
2163 
2164   switch (OpSize) { // expected operand size
2165   case 8:
2166     return AMDGPU::isInlinableLiteral64(Val, hasInv2PiInlineImm());
2167   case 4:
2168     return AMDGPU::isInlinableLiteral32(Val, hasInv2PiInlineImm());
2169   case 2: {
2170     const unsigned OperandType = Desc.OpInfo[OpIdx].OperandType;
2171     if (OperandType == AMDGPU::OPERAND_REG_INLINE_C_V2INT16 ||
2172         OperandType == AMDGPU::OPERAND_REG_INLINE_C_V2FP16) {
2173       return AMDGPU::isInlinableLiteralV216(Val, hasInv2PiInlineImm());
2174     } else {
2175       return AMDGPU::isInlinableLiteral16(Val, hasInv2PiInlineImm());
2176     }
2177   }
2178   default:
2179     llvm_unreachable("invalid operand size");
2180   }
2181 }
2182 
2183 bool AMDGPUAsmParser::usesConstantBus(const MCInst &Inst, unsigned OpIdx) {
2184   const MCOperand &MO = Inst.getOperand(OpIdx);
2185   if (MO.isImm()) {
2186     return !isInlineConstant(Inst, OpIdx);
2187   }
2188   return !MO.isReg() ||
2189          isSGPR(mc2PseudoReg(MO.getReg()), getContext().getRegisterInfo());
2190 }
2191 
2192 bool AMDGPUAsmParser::validateConstantBusLimitations(const MCInst &Inst) {
2193   const unsigned Opcode = Inst.getOpcode();
2194   const MCInstrDesc &Desc = MII.get(Opcode);
2195   unsigned ConstantBusUseCount = 0;
2196 
2197   if (Desc.TSFlags &
2198       (SIInstrFlags::VOPC |
2199        SIInstrFlags::VOP1 | SIInstrFlags::VOP2 |
2200        SIInstrFlags::VOP3 | SIInstrFlags::VOP3P |
2201        SIInstrFlags::SDWA)) {
2202     // Check special imm operands (used by madmk, etc)
2203     if (AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::imm) != -1) {
2204       ++ConstantBusUseCount;
2205     }
2206 
2207     unsigned SGPRUsed = findImplicitSGPRReadInVOP(Inst);
2208     if (SGPRUsed != AMDGPU::NoRegister) {
2209       ++ConstantBusUseCount;
2210     }
2211 
2212     const int Src0Idx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::src0);
2213     const int Src1Idx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::src1);
2214     const int Src2Idx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::src2);
2215 
2216     const int OpIndices[] = { Src0Idx, Src1Idx, Src2Idx };
2217 
2218     for (int OpIdx : OpIndices) {
2219       if (OpIdx == -1) break;
2220 
2221       const MCOperand &MO = Inst.getOperand(OpIdx);
2222       if (usesConstantBus(Inst, OpIdx)) {
2223         if (MO.isReg()) {
2224           const unsigned Reg = mc2PseudoReg(MO.getReg());
2225           // Pairs of registers with a partial intersections like these
2226           //   s0, s[0:1]
2227           //   flat_scratch_lo, flat_scratch
2228           //   flat_scratch_lo, flat_scratch_hi
2229           // are theoretically valid but they are disabled anyway.
2230           // Note that this code mimics SIInstrInfo::verifyInstruction
2231           if (Reg != SGPRUsed) {
2232             ++ConstantBusUseCount;
2233           }
2234           SGPRUsed = Reg;
2235         } else { // Expression or a literal
2236           ++ConstantBusUseCount;
2237         }
2238       }
2239     }
2240   }
2241 
2242   return ConstantBusUseCount <= 1;
2243 }
2244 
2245 bool AMDGPUAsmParser::validateEarlyClobberLimitations(const MCInst &Inst) {
2246   const unsigned Opcode = Inst.getOpcode();
2247   const MCInstrDesc &Desc = MII.get(Opcode);
2248 
2249   const int DstIdx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::vdst);
2250   if (DstIdx == -1 ||
2251       Desc.getOperandConstraint(DstIdx, MCOI::EARLY_CLOBBER) == -1) {
2252     return true;
2253   }
2254 
2255   const MCRegisterInfo *TRI = getContext().getRegisterInfo();
2256 
2257   const int Src0Idx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::src0);
2258   const int Src1Idx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::src1);
2259   const int Src2Idx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::src2);
2260 
2261   assert(DstIdx != -1);
2262   const MCOperand &Dst = Inst.getOperand(DstIdx);
2263   assert(Dst.isReg());
2264   const unsigned DstReg = mc2PseudoReg(Dst.getReg());
2265 
2266   const int SrcIndices[] = { Src0Idx, Src1Idx, Src2Idx };
2267 
2268   for (int SrcIdx : SrcIndices) {
2269     if (SrcIdx == -1) break;
2270     const MCOperand &Src = Inst.getOperand(SrcIdx);
2271     if (Src.isReg()) {
2272       const unsigned SrcReg = mc2PseudoReg(Src.getReg());
2273       if (isRegIntersect(DstReg, SrcReg, TRI)) {
2274         return false;
2275       }
2276     }
2277   }
2278 
2279   return true;
2280 }
2281 
2282 bool AMDGPUAsmParser::validateIntClampSupported(const MCInst &Inst) {
2283 
2284   const unsigned Opc = Inst.getOpcode();
2285   const MCInstrDesc &Desc = MII.get(Opc);
2286 
2287   if ((Desc.TSFlags & SIInstrFlags::IntClamp) != 0 && !hasIntClamp()) {
2288     int ClampIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::clamp);
2289     assert(ClampIdx != -1);
2290     return Inst.getOperand(ClampIdx).getImm() == 0;
2291   }
2292 
2293   return true;
2294 }
2295 
2296 bool AMDGPUAsmParser::validateMIMGDataSize(const MCInst &Inst) {
2297 
2298   const unsigned Opc = Inst.getOpcode();
2299   const MCInstrDesc &Desc = MII.get(Opc);
2300 
2301   if ((Desc.TSFlags & SIInstrFlags::MIMG) == 0)
2302     return true;
2303 
2304   // Gather4 instructions do not need validation: dst size is hardcoded.
2305   if (Desc.TSFlags & SIInstrFlags::Gather4)
2306     return true;
2307 
2308   int VDataIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::vdata);
2309   int DMaskIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::dmask);
2310   int TFEIdx   = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::tfe);
2311 
2312   assert(VDataIdx != -1);
2313   assert(DMaskIdx != -1);
2314   assert(TFEIdx != -1);
2315 
2316   unsigned VDataSize = AMDGPU::getRegOperandSize(getMRI(), Desc, VDataIdx);
2317   unsigned TFESize = Inst.getOperand(TFEIdx).getImm()? 1 : 0;
2318   unsigned DMask = Inst.getOperand(DMaskIdx).getImm() & 0xf;
2319   if (DMask == 0)
2320     DMask = 1;
2321 
2322   unsigned DataSize = countPopulation(DMask);
2323   if ((Desc.TSFlags & SIInstrFlags::D16) != 0 && hasPackedD16()) {
2324     DataSize = (DataSize + 1) / 2;
2325   }
2326 
2327   return (VDataSize / 4) == DataSize + TFESize;
2328 }
2329 
2330 bool AMDGPUAsmParser::validateMIMGAtomicDMask(const MCInst &Inst) {
2331 
2332   const unsigned Opc = Inst.getOpcode();
2333   const MCInstrDesc &Desc = MII.get(Opc);
2334 
2335   if ((Desc.TSFlags & SIInstrFlags::MIMG) == 0)
2336     return true;
2337   if (!Desc.mayLoad() || !Desc.mayStore())
2338     return true; // Not atomic
2339 
2340   int DMaskIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::dmask);
2341   unsigned DMask = Inst.getOperand(DMaskIdx).getImm() & 0xf;
2342 
2343   // This is an incomplete check because image_atomic_cmpswap
2344   // may only use 0x3 and 0xf while other atomic operations
2345   // may use 0x1 and 0x3. However these limitations are
2346   // verified when we check that dmask matches dst size.
2347   return DMask == 0x1 || DMask == 0x3 || DMask == 0xf;
2348 }
2349 
2350 bool AMDGPUAsmParser::validateMIMGGatherDMask(const MCInst &Inst) {
2351 
2352   const unsigned Opc = Inst.getOpcode();
2353   const MCInstrDesc &Desc = MII.get(Opc);
2354 
2355   if ((Desc.TSFlags & SIInstrFlags::Gather4) == 0)
2356     return true;
2357 
2358   int DMaskIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::dmask);
2359   unsigned DMask = Inst.getOperand(DMaskIdx).getImm() & 0xf;
2360 
2361   // GATHER4 instructions use dmask in a different fashion compared to
2362   // other MIMG instructions. The only useful DMASK values are
2363   // 1=red, 2=green, 4=blue, 8=alpha. (e.g. 1 returns
2364   // (red,red,red,red) etc.) The ISA document doesn't mention
2365   // this.
2366   return DMask == 0x1 || DMask == 0x2 || DMask == 0x4 || DMask == 0x8;
2367 }
2368 
2369 bool AMDGPUAsmParser::validateMIMGR128(const MCInst &Inst) {
2370 
2371   const unsigned Opc = Inst.getOpcode();
2372   const MCInstrDesc &Desc = MII.get(Opc);
2373 
2374   if ((Desc.TSFlags & SIInstrFlags::MIMG) == 0)
2375     return true;
2376 
2377   int Idx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::r128);
2378   assert(Idx != -1);
2379 
2380   bool R128 = (Inst.getOperand(Idx).getImm() != 0);
2381 
2382   return !R128 || hasMIMG_R128();
2383 }
2384 
2385 bool AMDGPUAsmParser::validateMIMGD16(const MCInst &Inst) {
2386 
2387   const unsigned Opc = Inst.getOpcode();
2388   const MCInstrDesc &Desc = MII.get(Opc);
2389 
2390   if ((Desc.TSFlags & SIInstrFlags::MIMG) == 0)
2391     return true;
2392   if ((Desc.TSFlags & SIInstrFlags::D16) == 0)
2393     return true;
2394 
2395   return !isCI() && !isSI();
2396 }
2397 
2398 bool AMDGPUAsmParser::validateInstruction(const MCInst &Inst,
2399                                           const SMLoc &IDLoc) {
2400   if (!validateConstantBusLimitations(Inst)) {
2401     Error(IDLoc,
2402       "invalid operand (violates constant bus restrictions)");
2403     return false;
2404   }
2405   if (!validateEarlyClobberLimitations(Inst)) {
2406     Error(IDLoc,
2407       "destination must be different than all sources");
2408     return false;
2409   }
2410   if (!validateIntClampSupported(Inst)) {
2411     Error(IDLoc,
2412       "integer clamping is not supported on this GPU");
2413     return false;
2414   }
2415   if (!validateMIMGR128(Inst)) {
2416     Error(IDLoc,
2417       "r128 modifier is not supported on this GPU");
2418     return false;
2419   }
2420   // For MUBUF/MTBUF d16 is a part of opcode, so there is nothing to validate.
2421   if (!validateMIMGD16(Inst)) {
2422     Error(IDLoc,
2423       "d16 modifier is not supported on this GPU");
2424     return false;
2425   }
2426   if (!validateMIMGDataSize(Inst)) {
2427     Error(IDLoc,
2428       "image data size does not match dmask and tfe");
2429     return false;
2430   }
2431   if (!validateMIMGAtomicDMask(Inst)) {
2432     Error(IDLoc,
2433       "invalid atomic image dmask");
2434     return false;
2435   }
2436   if (!validateMIMGGatherDMask(Inst)) {
2437     Error(IDLoc,
2438       "invalid image_gather dmask: only one bit must be set");
2439     return false;
2440   }
2441 
2442   return true;
2443 }
2444 
2445 static std::string AMDGPUMnemonicSpellCheck(StringRef S, uint64_t FBS,
2446                                             unsigned VariantID = 0);
2447 
2448 bool AMDGPUAsmParser::MatchAndEmitInstruction(SMLoc IDLoc, unsigned &Opcode,
2449                                               OperandVector &Operands,
2450                                               MCStreamer &Out,
2451                                               uint64_t &ErrorInfo,
2452                                               bool MatchingInlineAsm) {
2453   MCInst Inst;
2454   unsigned Result = Match_Success;
2455   for (auto Variant : getMatchedVariants()) {
2456     uint64_t EI;
2457     auto R = MatchInstructionImpl(Operands, Inst, EI, MatchingInlineAsm,
2458                                   Variant);
2459     // We order match statuses from least to most specific. We use most specific
2460     // status as resulting
2461     // Match_MnemonicFail < Match_InvalidOperand < Match_MissingFeature < Match_PreferE32
2462     if ((R == Match_Success) ||
2463         (R == Match_PreferE32) ||
2464         (R == Match_MissingFeature && Result != Match_PreferE32) ||
2465         (R == Match_InvalidOperand && Result != Match_MissingFeature
2466                                    && Result != Match_PreferE32) ||
2467         (R == Match_MnemonicFail   && Result != Match_InvalidOperand
2468                                    && Result != Match_MissingFeature
2469                                    && Result != Match_PreferE32)) {
2470       Result = R;
2471       ErrorInfo = EI;
2472     }
2473     if (R == Match_Success)
2474       break;
2475   }
2476 
2477   switch (Result) {
2478   default: break;
2479   case Match_Success:
2480     if (!validateInstruction(Inst, IDLoc)) {
2481       return true;
2482     }
2483     Inst.setLoc(IDLoc);
2484     Out.EmitInstruction(Inst, getSTI());
2485     return false;
2486 
2487   case Match_MissingFeature:
2488     return Error(IDLoc, "instruction not supported on this GPU");
2489 
2490   case Match_MnemonicFail: {
2491     uint64_t FBS = ComputeAvailableFeatures(getSTI().getFeatureBits());
2492     std::string Suggestion = AMDGPUMnemonicSpellCheck(
2493         ((AMDGPUOperand &)*Operands[0]).getToken(), FBS);
2494     return Error(IDLoc, "invalid instruction" + Suggestion,
2495                  ((AMDGPUOperand &)*Operands[0]).getLocRange());
2496   }
2497 
2498   case Match_InvalidOperand: {
2499     SMLoc ErrorLoc = IDLoc;
2500     if (ErrorInfo != ~0ULL) {
2501       if (ErrorInfo >= Operands.size()) {
2502         return Error(IDLoc, "too few operands for instruction");
2503       }
2504       ErrorLoc = ((AMDGPUOperand &)*Operands[ErrorInfo]).getStartLoc();
2505       if (ErrorLoc == SMLoc())
2506         ErrorLoc = IDLoc;
2507     }
2508     return Error(ErrorLoc, "invalid operand for instruction");
2509   }
2510 
2511   case Match_PreferE32:
2512     return Error(IDLoc, "internal error: instruction without _e64 suffix "
2513                         "should be encoded as e32");
2514   }
2515   llvm_unreachable("Implement any new match types added!");
2516 }
2517 
2518 bool AMDGPUAsmParser::ParseAsAbsoluteExpression(uint32_t &Ret) {
2519   int64_t Tmp = -1;
2520   if (getLexer().isNot(AsmToken::Integer) && getLexer().isNot(AsmToken::Identifier)) {
2521     return true;
2522   }
2523   if (getParser().parseAbsoluteExpression(Tmp)) {
2524     return true;
2525   }
2526   Ret = static_cast<uint32_t>(Tmp);
2527   return false;
2528 }
2529 
2530 bool AMDGPUAsmParser::ParseDirectiveMajorMinor(uint32_t &Major,
2531                                                uint32_t &Minor) {
2532   if (ParseAsAbsoluteExpression(Major))
2533     return TokError("invalid major version");
2534 
2535   if (getLexer().isNot(AsmToken::Comma))
2536     return TokError("minor version number required, comma expected");
2537   Lex();
2538 
2539   if (ParseAsAbsoluteExpression(Minor))
2540     return TokError("invalid minor version");
2541 
2542   return false;
2543 }
2544 
2545 bool AMDGPUAsmParser::ParseDirectiveHSACodeObjectVersion() {
2546   uint32_t Major;
2547   uint32_t Minor;
2548 
2549   if (ParseDirectiveMajorMinor(Major, Minor))
2550     return true;
2551 
2552   getTargetStreamer().EmitDirectiveHSACodeObjectVersion(Major, Minor);
2553   return false;
2554 }
2555 
2556 bool AMDGPUAsmParser::ParseDirectiveHSACodeObjectISA() {
2557   uint32_t Major;
2558   uint32_t Minor;
2559   uint32_t Stepping;
2560   StringRef VendorName;
2561   StringRef ArchName;
2562 
2563   // If this directive has no arguments, then use the ISA version for the
2564   // targeted GPU.
2565   if (getLexer().is(AsmToken::EndOfStatement)) {
2566     AMDGPU::IsaInfo::IsaVersion ISA =
2567         AMDGPU::IsaInfo::getIsaVersion(getFeatureBits());
2568     getTargetStreamer().EmitDirectiveHSACodeObjectISA(ISA.Major, ISA.Minor,
2569                                                       ISA.Stepping,
2570                                                       "AMD", "AMDGPU");
2571     return false;
2572   }
2573 
2574   if (ParseDirectiveMajorMinor(Major, Minor))
2575     return true;
2576 
2577   if (getLexer().isNot(AsmToken::Comma))
2578     return TokError("stepping version number required, comma expected");
2579   Lex();
2580 
2581   if (ParseAsAbsoluteExpression(Stepping))
2582     return TokError("invalid stepping version");
2583 
2584   if (getLexer().isNot(AsmToken::Comma))
2585     return TokError("vendor name required, comma expected");
2586   Lex();
2587 
2588   if (getLexer().isNot(AsmToken::String))
2589     return TokError("invalid vendor name");
2590 
2591   VendorName = getLexer().getTok().getStringContents();
2592   Lex();
2593 
2594   if (getLexer().isNot(AsmToken::Comma))
2595     return TokError("arch name required, comma expected");
2596   Lex();
2597 
2598   if (getLexer().isNot(AsmToken::String))
2599     return TokError("invalid arch name");
2600 
2601   ArchName = getLexer().getTok().getStringContents();
2602   Lex();
2603 
2604   getTargetStreamer().EmitDirectiveHSACodeObjectISA(Major, Minor, Stepping,
2605                                                     VendorName, ArchName);
2606   return false;
2607 }
2608 
2609 bool AMDGPUAsmParser::ParseAMDKernelCodeTValue(StringRef ID,
2610                                                amd_kernel_code_t &Header) {
2611   SmallString<40> ErrStr;
2612   raw_svector_ostream Err(ErrStr);
2613   if (!parseAmdKernelCodeField(ID, getParser(), Header, Err)) {
2614     return TokError(Err.str());
2615   }
2616   Lex();
2617   return false;
2618 }
2619 
2620 bool AMDGPUAsmParser::ParseDirectiveAMDKernelCodeT() {
2621   amd_kernel_code_t Header;
2622   AMDGPU::initDefaultAMDKernelCodeT(Header, getFeatureBits());
2623 
2624   while (true) {
2625     // Lex EndOfStatement.  This is in a while loop, because lexing a comment
2626     // will set the current token to EndOfStatement.
2627     while(getLexer().is(AsmToken::EndOfStatement))
2628       Lex();
2629 
2630     if (getLexer().isNot(AsmToken::Identifier))
2631       return TokError("expected value identifier or .end_amd_kernel_code_t");
2632 
2633     StringRef ID = getLexer().getTok().getIdentifier();
2634     Lex();
2635 
2636     if (ID == ".end_amd_kernel_code_t")
2637       break;
2638 
2639     if (ParseAMDKernelCodeTValue(ID, Header))
2640       return true;
2641   }
2642 
2643   getTargetStreamer().EmitAMDKernelCodeT(Header);
2644 
2645   return false;
2646 }
2647 
2648 bool AMDGPUAsmParser::ParseDirectiveAMDGPUHsaKernel() {
2649   if (getLexer().isNot(AsmToken::Identifier))
2650     return TokError("expected symbol name");
2651 
2652   StringRef KernelName = Parser.getTok().getString();
2653 
2654   getTargetStreamer().EmitAMDGPUSymbolType(KernelName,
2655                                            ELF::STT_AMDGPU_HSA_KERNEL);
2656   Lex();
2657   KernelScope.initialize(getContext());
2658   return false;
2659 }
2660 
2661 bool AMDGPUAsmParser::ParseDirectiveISAVersion() {
2662   if (getSTI().getTargetTriple().getArch() != Triple::amdgcn) {
2663     return Error(getParser().getTok().getLoc(),
2664                  ".amd_amdgpu_isa directive is not available on non-amdgcn "
2665                  "architectures");
2666   }
2667 
2668   auto ISAVersionStringFromASM = getLexer().getTok().getStringContents();
2669 
2670   std::string ISAVersionStringFromSTI;
2671   raw_string_ostream ISAVersionStreamFromSTI(ISAVersionStringFromSTI);
2672   IsaInfo::streamIsaVersion(&getSTI(), ISAVersionStreamFromSTI);
2673 
2674   if (ISAVersionStringFromASM != ISAVersionStreamFromSTI.str()) {
2675     return Error(getParser().getTok().getLoc(),
2676                  ".amd_amdgpu_isa directive does not match triple and/or mcpu "
2677                  "arguments specified through the command line");
2678   }
2679 
2680   getTargetStreamer().EmitISAVersion(ISAVersionStreamFromSTI.str());
2681   Lex();
2682 
2683   return false;
2684 }
2685 
2686 bool AMDGPUAsmParser::ParseDirectiveHSAMetadata() {
2687   if (getSTI().getTargetTriple().getOS() != Triple::AMDHSA) {
2688     return Error(getParser().getTok().getLoc(),
2689                  (Twine(HSAMD::AssemblerDirectiveBegin) + Twine(" directive is "
2690                  "not available on non-amdhsa OSes")).str());
2691   }
2692 
2693   std::string HSAMetadataString;
2694   raw_string_ostream YamlStream(HSAMetadataString);
2695 
2696   getLexer().setSkipSpace(false);
2697 
2698   bool FoundEnd = false;
2699   while (!getLexer().is(AsmToken::Eof)) {
2700     while (getLexer().is(AsmToken::Space)) {
2701       YamlStream << getLexer().getTok().getString();
2702       Lex();
2703     }
2704 
2705     if (getLexer().is(AsmToken::Identifier)) {
2706       StringRef ID = getLexer().getTok().getIdentifier();
2707       if (ID == AMDGPU::HSAMD::AssemblerDirectiveEnd) {
2708         Lex();
2709         FoundEnd = true;
2710         break;
2711       }
2712     }
2713 
2714     YamlStream << Parser.parseStringToEndOfStatement()
2715                << getContext().getAsmInfo()->getSeparatorString();
2716 
2717     Parser.eatToEndOfStatement();
2718   }
2719 
2720   getLexer().setSkipSpace(true);
2721 
2722   if (getLexer().is(AsmToken::Eof) && !FoundEnd) {
2723     return TokError(Twine("expected directive ") +
2724                     Twine(HSAMD::AssemblerDirectiveEnd) + Twine(" not found"));
2725   }
2726 
2727   YamlStream.flush();
2728 
2729   if (!getTargetStreamer().EmitHSAMetadata(HSAMetadataString))
2730     return Error(getParser().getTok().getLoc(), "invalid HSA metadata");
2731 
2732   return false;
2733 }
2734 
2735 bool AMDGPUAsmParser::ParseDirectivePALMetadata() {
2736   if (getSTI().getTargetTriple().getOS() != Triple::AMDPAL) {
2737     return Error(getParser().getTok().getLoc(),
2738                  (Twine(PALMD::AssemblerDirective) + Twine(" directive is "
2739                  "not available on non-amdpal OSes")).str());
2740   }
2741 
2742   PALMD::Metadata PALMetadata;
2743   for (;;) {
2744     uint32_t Value;
2745     if (ParseAsAbsoluteExpression(Value)) {
2746       return TokError(Twine("invalid value in ") +
2747                       Twine(PALMD::AssemblerDirective));
2748     }
2749     PALMetadata.push_back(Value);
2750     if (getLexer().isNot(AsmToken::Comma))
2751       break;
2752     Lex();
2753   }
2754   getTargetStreamer().EmitPALMetadata(PALMetadata);
2755   return false;
2756 }
2757 
2758 bool AMDGPUAsmParser::ParseDirective(AsmToken DirectiveID) {
2759   StringRef IDVal = DirectiveID.getString();
2760 
2761   if (IDVal == ".hsa_code_object_version")
2762     return ParseDirectiveHSACodeObjectVersion();
2763 
2764   if (IDVal == ".hsa_code_object_isa")
2765     return ParseDirectiveHSACodeObjectISA();
2766 
2767   if (IDVal == ".amd_kernel_code_t")
2768     return ParseDirectiveAMDKernelCodeT();
2769 
2770   if (IDVal == ".amdgpu_hsa_kernel")
2771     return ParseDirectiveAMDGPUHsaKernel();
2772 
2773   if (IDVal == ".amd_amdgpu_isa")
2774     return ParseDirectiveISAVersion();
2775 
2776   if (IDVal == AMDGPU::HSAMD::AssemblerDirectiveBegin)
2777     return ParseDirectiveHSAMetadata();
2778 
2779   if (IDVal == PALMD::AssemblerDirective)
2780     return ParseDirectivePALMetadata();
2781 
2782   return true;
2783 }
2784 
2785 bool AMDGPUAsmParser::subtargetHasRegister(const MCRegisterInfo &MRI,
2786                                            unsigned RegNo) const {
2787 
2788   for (MCRegAliasIterator R(AMDGPU::TTMP12_TTMP13_TTMP14_TTMP15, &MRI, true);
2789        R.isValid(); ++R) {
2790     if (*R == RegNo)
2791       return isGFX9();
2792   }
2793 
2794   switch (RegNo) {
2795   case AMDGPU::TBA:
2796   case AMDGPU::TBA_LO:
2797   case AMDGPU::TBA_HI:
2798   case AMDGPU::TMA:
2799   case AMDGPU::TMA_LO:
2800   case AMDGPU::TMA_HI:
2801     return !isGFX9();
2802   case AMDGPU::XNACK_MASK:
2803   case AMDGPU::XNACK_MASK_LO:
2804   case AMDGPU::XNACK_MASK_HI:
2805     return !isCI() && !isSI() && hasXNACK();
2806   default:
2807     break;
2808   }
2809 
2810   if (isCI())
2811     return true;
2812 
2813   if (isSI()) {
2814     // No flat_scr
2815     switch (RegNo) {
2816     case AMDGPU::FLAT_SCR:
2817     case AMDGPU::FLAT_SCR_LO:
2818     case AMDGPU::FLAT_SCR_HI:
2819       return false;
2820     default:
2821       return true;
2822     }
2823   }
2824 
2825   // VI only has 102 SGPRs, so make sure we aren't trying to use the 2 more that
2826   // SI/CI have.
2827   for (MCRegAliasIterator R(AMDGPU::SGPR102_SGPR103, &MRI, true);
2828        R.isValid(); ++R) {
2829     if (*R == RegNo)
2830       return false;
2831   }
2832 
2833   return true;
2834 }
2835 
2836 OperandMatchResultTy
2837 AMDGPUAsmParser::parseOperand(OperandVector &Operands, StringRef Mnemonic) {
2838   // Try to parse with a custom parser
2839   OperandMatchResultTy ResTy = MatchOperandParserImpl(Operands, Mnemonic);
2840 
2841   // If we successfully parsed the operand or if there as an error parsing,
2842   // we are done.
2843   //
2844   // If we are parsing after we reach EndOfStatement then this means we
2845   // are appending default values to the Operands list.  This is only done
2846   // by custom parser, so we shouldn't continue on to the generic parsing.
2847   if (ResTy == MatchOperand_Success || ResTy == MatchOperand_ParseFail ||
2848       getLexer().is(AsmToken::EndOfStatement))
2849     return ResTy;
2850 
2851   ResTy = parseRegOrImm(Operands);
2852 
2853   if (ResTy == MatchOperand_Success)
2854     return ResTy;
2855 
2856   const auto &Tok = Parser.getTok();
2857   SMLoc S = Tok.getLoc();
2858 
2859   const MCExpr *Expr = nullptr;
2860   if (!Parser.parseExpression(Expr)) {
2861     Operands.push_back(AMDGPUOperand::CreateExpr(this, Expr, S));
2862     return MatchOperand_Success;
2863   }
2864 
2865   // Possibly this is an instruction flag like 'gds'.
2866   if (Tok.getKind() == AsmToken::Identifier) {
2867     Operands.push_back(AMDGPUOperand::CreateToken(this, Tok.getString(), S));
2868     Parser.Lex();
2869     return MatchOperand_Success;
2870   }
2871 
2872   return MatchOperand_NoMatch;
2873 }
2874 
2875 StringRef AMDGPUAsmParser::parseMnemonicSuffix(StringRef Name) {
2876   // Clear any forced encodings from the previous instruction.
2877   setForcedEncodingSize(0);
2878   setForcedDPP(false);
2879   setForcedSDWA(false);
2880 
2881   if (Name.endswith("_e64")) {
2882     setForcedEncodingSize(64);
2883     return Name.substr(0, Name.size() - 4);
2884   } else if (Name.endswith("_e32")) {
2885     setForcedEncodingSize(32);
2886     return Name.substr(0, Name.size() - 4);
2887   } else if (Name.endswith("_dpp")) {
2888     setForcedDPP(true);
2889     return Name.substr(0, Name.size() - 4);
2890   } else if (Name.endswith("_sdwa")) {
2891     setForcedSDWA(true);
2892     return Name.substr(0, Name.size() - 5);
2893   }
2894   return Name;
2895 }
2896 
2897 bool AMDGPUAsmParser::ParseInstruction(ParseInstructionInfo &Info,
2898                                        StringRef Name,
2899                                        SMLoc NameLoc, OperandVector &Operands) {
2900   // Add the instruction mnemonic
2901   Name = parseMnemonicSuffix(Name);
2902   Operands.push_back(AMDGPUOperand::CreateToken(this, Name, NameLoc));
2903 
2904   while (!getLexer().is(AsmToken::EndOfStatement)) {
2905     OperandMatchResultTy Res = parseOperand(Operands, Name);
2906 
2907     // Eat the comma or space if there is one.
2908     if (getLexer().is(AsmToken::Comma))
2909       Parser.Lex();
2910 
2911     switch (Res) {
2912       case MatchOperand_Success: break;
2913       case MatchOperand_ParseFail:
2914         Error(getLexer().getLoc(), "failed parsing operand.");
2915         while (!getLexer().is(AsmToken::EndOfStatement)) {
2916           Parser.Lex();
2917         }
2918         return true;
2919       case MatchOperand_NoMatch:
2920         Error(getLexer().getLoc(), "not a valid operand.");
2921         while (!getLexer().is(AsmToken::EndOfStatement)) {
2922           Parser.Lex();
2923         }
2924         return true;
2925     }
2926   }
2927 
2928   return false;
2929 }
2930 
2931 //===----------------------------------------------------------------------===//
2932 // Utility functions
2933 //===----------------------------------------------------------------------===//
2934 
2935 OperandMatchResultTy
2936 AMDGPUAsmParser::parseIntWithPrefix(const char *Prefix, int64_t &Int) {
2937   switch(getLexer().getKind()) {
2938     default: return MatchOperand_NoMatch;
2939     case AsmToken::Identifier: {
2940       StringRef Name = Parser.getTok().getString();
2941       if (!Name.equals(Prefix)) {
2942         return MatchOperand_NoMatch;
2943       }
2944 
2945       Parser.Lex();
2946       if (getLexer().isNot(AsmToken::Colon))
2947         return MatchOperand_ParseFail;
2948 
2949       Parser.Lex();
2950 
2951       bool IsMinus = false;
2952       if (getLexer().getKind() == AsmToken::Minus) {
2953         Parser.Lex();
2954         IsMinus = true;
2955       }
2956 
2957       if (getLexer().isNot(AsmToken::Integer))
2958         return MatchOperand_ParseFail;
2959 
2960       if (getParser().parseAbsoluteExpression(Int))
2961         return MatchOperand_ParseFail;
2962 
2963       if (IsMinus)
2964         Int = -Int;
2965       break;
2966     }
2967   }
2968   return MatchOperand_Success;
2969 }
2970 
2971 OperandMatchResultTy
2972 AMDGPUAsmParser::parseIntWithPrefix(const char *Prefix, OperandVector &Operands,
2973                                     AMDGPUOperand::ImmTy ImmTy,
2974                                     bool (*ConvertResult)(int64_t&)) {
2975   SMLoc S = Parser.getTok().getLoc();
2976   int64_t Value = 0;
2977 
2978   OperandMatchResultTy Res = parseIntWithPrefix(Prefix, Value);
2979   if (Res != MatchOperand_Success)
2980     return Res;
2981 
2982   if (ConvertResult && !ConvertResult(Value)) {
2983     return MatchOperand_ParseFail;
2984   }
2985 
2986   Operands.push_back(AMDGPUOperand::CreateImm(this, Value, S, ImmTy));
2987   return MatchOperand_Success;
2988 }
2989 
2990 OperandMatchResultTy AMDGPUAsmParser::parseOperandArrayWithPrefix(
2991   const char *Prefix,
2992   OperandVector &Operands,
2993   AMDGPUOperand::ImmTy ImmTy,
2994   bool (*ConvertResult)(int64_t&)) {
2995   StringRef Name = Parser.getTok().getString();
2996   if (!Name.equals(Prefix))
2997     return MatchOperand_NoMatch;
2998 
2999   Parser.Lex();
3000   if (getLexer().isNot(AsmToken::Colon))
3001     return MatchOperand_ParseFail;
3002 
3003   Parser.Lex();
3004   if (getLexer().isNot(AsmToken::LBrac))
3005     return MatchOperand_ParseFail;
3006   Parser.Lex();
3007 
3008   unsigned Val = 0;
3009   SMLoc S = Parser.getTok().getLoc();
3010 
3011   // FIXME: How to verify the number of elements matches the number of src
3012   // operands?
3013   for (int I = 0; I < 4; ++I) {
3014     if (I != 0) {
3015       if (getLexer().is(AsmToken::RBrac))
3016         break;
3017 
3018       if (getLexer().isNot(AsmToken::Comma))
3019         return MatchOperand_ParseFail;
3020       Parser.Lex();
3021     }
3022 
3023     if (getLexer().isNot(AsmToken::Integer))
3024       return MatchOperand_ParseFail;
3025 
3026     int64_t Op;
3027     if (getParser().parseAbsoluteExpression(Op))
3028       return MatchOperand_ParseFail;
3029 
3030     if (Op != 0 && Op != 1)
3031       return MatchOperand_ParseFail;
3032     Val |= (Op << I);
3033   }
3034 
3035   Parser.Lex();
3036   Operands.push_back(AMDGPUOperand::CreateImm(this, Val, S, ImmTy));
3037   return MatchOperand_Success;
3038 }
3039 
3040 OperandMatchResultTy
3041 AMDGPUAsmParser::parseNamedBit(const char *Name, OperandVector &Operands,
3042                                AMDGPUOperand::ImmTy ImmTy) {
3043   int64_t Bit = 0;
3044   SMLoc S = Parser.getTok().getLoc();
3045 
3046   // We are at the end of the statement, and this is a default argument, so
3047   // use a default value.
3048   if (getLexer().isNot(AsmToken::EndOfStatement)) {
3049     switch(getLexer().getKind()) {
3050       case AsmToken::Identifier: {
3051         StringRef Tok = Parser.getTok().getString();
3052         if (Tok == Name) {
3053           Bit = 1;
3054           Parser.Lex();
3055         } else if (Tok.startswith("no") && Tok.endswith(Name)) {
3056           Bit = 0;
3057           Parser.Lex();
3058         } else {
3059           return MatchOperand_NoMatch;
3060         }
3061         break;
3062       }
3063       default:
3064         return MatchOperand_NoMatch;
3065     }
3066   }
3067 
3068   Operands.push_back(AMDGPUOperand::CreateImm(this, Bit, S, ImmTy));
3069   return MatchOperand_Success;
3070 }
3071 
3072 static void addOptionalImmOperand(
3073   MCInst& Inst, const OperandVector& Operands,
3074   AMDGPUAsmParser::OptionalImmIndexMap& OptionalIdx,
3075   AMDGPUOperand::ImmTy ImmT,
3076   int64_t Default = 0) {
3077   auto i = OptionalIdx.find(ImmT);
3078   if (i != OptionalIdx.end()) {
3079     unsigned Idx = i->second;
3080     ((AMDGPUOperand &)*Operands[Idx]).addImmOperands(Inst, 1);
3081   } else {
3082     Inst.addOperand(MCOperand::createImm(Default));
3083   }
3084 }
3085 
3086 OperandMatchResultTy
3087 AMDGPUAsmParser::parseStringWithPrefix(StringRef Prefix, StringRef &Value) {
3088   if (getLexer().isNot(AsmToken::Identifier)) {
3089     return MatchOperand_NoMatch;
3090   }
3091   StringRef Tok = Parser.getTok().getString();
3092   if (Tok != Prefix) {
3093     return MatchOperand_NoMatch;
3094   }
3095 
3096   Parser.Lex();
3097   if (getLexer().isNot(AsmToken::Colon)) {
3098     return MatchOperand_ParseFail;
3099   }
3100 
3101   Parser.Lex();
3102   if (getLexer().isNot(AsmToken::Identifier)) {
3103     return MatchOperand_ParseFail;
3104   }
3105 
3106   Value = Parser.getTok().getString();
3107   return MatchOperand_Success;
3108 }
3109 
3110 //===----------------------------------------------------------------------===//
3111 // ds
3112 //===----------------------------------------------------------------------===//
3113 
3114 void AMDGPUAsmParser::cvtDSOffset01(MCInst &Inst,
3115                                     const OperandVector &Operands) {
3116   OptionalImmIndexMap OptionalIdx;
3117 
3118   for (unsigned i = 1, e = Operands.size(); i != e; ++i) {
3119     AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[i]);
3120 
3121     // Add the register arguments
3122     if (Op.isReg()) {
3123       Op.addRegOperands(Inst, 1);
3124       continue;
3125     }
3126 
3127     // Handle optional arguments
3128     OptionalIdx[Op.getImmTy()] = i;
3129   }
3130 
3131   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyOffset0);
3132   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyOffset1);
3133   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyGDS);
3134 
3135   Inst.addOperand(MCOperand::createReg(AMDGPU::M0)); // m0
3136 }
3137 
3138 void AMDGPUAsmParser::cvtDSImpl(MCInst &Inst, const OperandVector &Operands,
3139                                 bool IsGdsHardcoded) {
3140   OptionalImmIndexMap OptionalIdx;
3141 
3142   for (unsigned i = 1, e = Operands.size(); i != e; ++i) {
3143     AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[i]);
3144 
3145     // Add the register arguments
3146     if (Op.isReg()) {
3147       Op.addRegOperands(Inst, 1);
3148       continue;
3149     }
3150 
3151     if (Op.isToken() && Op.getToken() == "gds") {
3152       IsGdsHardcoded = true;
3153       continue;
3154     }
3155 
3156     // Handle optional arguments
3157     OptionalIdx[Op.getImmTy()] = i;
3158   }
3159 
3160   AMDGPUOperand::ImmTy OffsetType =
3161     (Inst.getOpcode() == AMDGPU::DS_SWIZZLE_B32_si ||
3162      Inst.getOpcode() == AMDGPU::DS_SWIZZLE_B32_vi) ? AMDGPUOperand::ImmTySwizzle :
3163                                                       AMDGPUOperand::ImmTyOffset;
3164 
3165   addOptionalImmOperand(Inst, Operands, OptionalIdx, OffsetType);
3166 
3167   if (!IsGdsHardcoded) {
3168     addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyGDS);
3169   }
3170   Inst.addOperand(MCOperand::createReg(AMDGPU::M0)); // m0
3171 }
3172 
3173 void AMDGPUAsmParser::cvtExp(MCInst &Inst, const OperandVector &Operands) {
3174   OptionalImmIndexMap OptionalIdx;
3175 
3176   unsigned OperandIdx[4];
3177   unsigned EnMask = 0;
3178   int SrcIdx = 0;
3179 
3180   for (unsigned i = 1, e = Operands.size(); i != e; ++i) {
3181     AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[i]);
3182 
3183     // Add the register arguments
3184     if (Op.isReg()) {
3185       assert(SrcIdx < 4);
3186       OperandIdx[SrcIdx] = Inst.size();
3187       Op.addRegOperands(Inst, 1);
3188       ++SrcIdx;
3189       continue;
3190     }
3191 
3192     if (Op.isOff()) {
3193       assert(SrcIdx < 4);
3194       OperandIdx[SrcIdx] = Inst.size();
3195       Inst.addOperand(MCOperand::createReg(AMDGPU::NoRegister));
3196       ++SrcIdx;
3197       continue;
3198     }
3199 
3200     if (Op.isImm() && Op.getImmTy() == AMDGPUOperand::ImmTyExpTgt) {
3201       Op.addImmOperands(Inst, 1);
3202       continue;
3203     }
3204 
3205     if (Op.isToken() && Op.getToken() == "done")
3206       continue;
3207 
3208     // Handle optional arguments
3209     OptionalIdx[Op.getImmTy()] = i;
3210   }
3211 
3212   assert(SrcIdx == 4);
3213 
3214   bool Compr = false;
3215   if (OptionalIdx.find(AMDGPUOperand::ImmTyExpCompr) != OptionalIdx.end()) {
3216     Compr = true;
3217     Inst.getOperand(OperandIdx[1]) = Inst.getOperand(OperandIdx[2]);
3218     Inst.getOperand(OperandIdx[2]).setReg(AMDGPU::NoRegister);
3219     Inst.getOperand(OperandIdx[3]).setReg(AMDGPU::NoRegister);
3220   }
3221 
3222   for (auto i = 0; i < SrcIdx; ++i) {
3223     if (Inst.getOperand(OperandIdx[i]).getReg() != AMDGPU::NoRegister) {
3224       EnMask |= Compr? (0x3 << i * 2) : (0x1 << i);
3225     }
3226   }
3227 
3228   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyExpVM);
3229   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyExpCompr);
3230 
3231   Inst.addOperand(MCOperand::createImm(EnMask));
3232 }
3233 
3234 //===----------------------------------------------------------------------===//
3235 // s_waitcnt
3236 //===----------------------------------------------------------------------===//
3237 
3238 static bool
3239 encodeCnt(
3240   const AMDGPU::IsaInfo::IsaVersion ISA,
3241   int64_t &IntVal,
3242   int64_t CntVal,
3243   bool Saturate,
3244   unsigned (*encode)(const IsaInfo::IsaVersion &Version, unsigned, unsigned),
3245   unsigned (*decode)(const IsaInfo::IsaVersion &Version, unsigned))
3246 {
3247   bool Failed = false;
3248 
3249   IntVal = encode(ISA, IntVal, CntVal);
3250   if (CntVal != decode(ISA, IntVal)) {
3251     if (Saturate) {
3252       IntVal = encode(ISA, IntVal, -1);
3253     } else {
3254       Failed = true;
3255     }
3256   }
3257   return Failed;
3258 }
3259 
3260 bool AMDGPUAsmParser::parseCnt(int64_t &IntVal) {
3261   StringRef CntName = Parser.getTok().getString();
3262   int64_t CntVal;
3263 
3264   Parser.Lex();
3265   if (getLexer().isNot(AsmToken::LParen))
3266     return true;
3267 
3268   Parser.Lex();
3269   if (getLexer().isNot(AsmToken::Integer))
3270     return true;
3271 
3272   SMLoc ValLoc = Parser.getTok().getLoc();
3273   if (getParser().parseAbsoluteExpression(CntVal))
3274     return true;
3275 
3276   AMDGPU::IsaInfo::IsaVersion ISA =
3277       AMDGPU::IsaInfo::getIsaVersion(getFeatureBits());
3278 
3279   bool Failed = true;
3280   bool Sat = CntName.endswith("_sat");
3281 
3282   if (CntName == "vmcnt" || CntName == "vmcnt_sat") {
3283     Failed = encodeCnt(ISA, IntVal, CntVal, Sat, encodeVmcnt, decodeVmcnt);
3284   } else if (CntName == "expcnt" || CntName == "expcnt_sat") {
3285     Failed = encodeCnt(ISA, IntVal, CntVal, Sat, encodeExpcnt, decodeExpcnt);
3286   } else if (CntName == "lgkmcnt" || CntName == "lgkmcnt_sat") {
3287     Failed = encodeCnt(ISA, IntVal, CntVal, Sat, encodeLgkmcnt, decodeLgkmcnt);
3288   }
3289 
3290   if (Failed) {
3291     Error(ValLoc, "too large value for " + CntName);
3292     return true;
3293   }
3294 
3295   if (getLexer().isNot(AsmToken::RParen)) {
3296     return true;
3297   }
3298 
3299   Parser.Lex();
3300   if (getLexer().is(AsmToken::Amp) || getLexer().is(AsmToken::Comma)) {
3301     const AsmToken NextToken = getLexer().peekTok();
3302     if (NextToken.is(AsmToken::Identifier)) {
3303       Parser.Lex();
3304     }
3305   }
3306 
3307   return false;
3308 }
3309 
3310 OperandMatchResultTy
3311 AMDGPUAsmParser::parseSWaitCntOps(OperandVector &Operands) {
3312   AMDGPU::IsaInfo::IsaVersion ISA =
3313       AMDGPU::IsaInfo::getIsaVersion(getFeatureBits());
3314   int64_t Waitcnt = getWaitcntBitMask(ISA);
3315   SMLoc S = Parser.getTok().getLoc();
3316 
3317   switch(getLexer().getKind()) {
3318     default: return MatchOperand_ParseFail;
3319     case AsmToken::Integer:
3320       // The operand can be an integer value.
3321       if (getParser().parseAbsoluteExpression(Waitcnt))
3322         return MatchOperand_ParseFail;
3323       break;
3324 
3325     case AsmToken::Identifier:
3326       do {
3327         if (parseCnt(Waitcnt))
3328           return MatchOperand_ParseFail;
3329       } while(getLexer().isNot(AsmToken::EndOfStatement));
3330       break;
3331   }
3332   Operands.push_back(AMDGPUOperand::CreateImm(this, Waitcnt, S));
3333   return MatchOperand_Success;
3334 }
3335 
3336 bool AMDGPUAsmParser::parseHwregConstruct(OperandInfoTy &HwReg, int64_t &Offset,
3337                                           int64_t &Width) {
3338   using namespace llvm::AMDGPU::Hwreg;
3339 
3340   if (Parser.getTok().getString() != "hwreg")
3341     return true;
3342   Parser.Lex();
3343 
3344   if (getLexer().isNot(AsmToken::LParen))
3345     return true;
3346   Parser.Lex();
3347 
3348   if (getLexer().is(AsmToken::Identifier)) {
3349     HwReg.IsSymbolic = true;
3350     HwReg.Id = ID_UNKNOWN_;
3351     const StringRef tok = Parser.getTok().getString();
3352     int Last = ID_SYMBOLIC_LAST_;
3353     if (isSI() || isCI() || isVI())
3354       Last = ID_SYMBOLIC_FIRST_GFX9_;
3355     for (int i = ID_SYMBOLIC_FIRST_; i < Last; ++i) {
3356       if (tok == IdSymbolic[i]) {
3357         HwReg.Id = i;
3358         break;
3359       }
3360     }
3361     Parser.Lex();
3362   } else {
3363     HwReg.IsSymbolic = false;
3364     if (getLexer().isNot(AsmToken::Integer))
3365       return true;
3366     if (getParser().parseAbsoluteExpression(HwReg.Id))
3367       return true;
3368   }
3369 
3370   if (getLexer().is(AsmToken::RParen)) {
3371     Parser.Lex();
3372     return false;
3373   }
3374 
3375   // optional params
3376   if (getLexer().isNot(AsmToken::Comma))
3377     return true;
3378   Parser.Lex();
3379 
3380   if (getLexer().isNot(AsmToken::Integer))
3381     return true;
3382   if (getParser().parseAbsoluteExpression(Offset))
3383     return true;
3384 
3385   if (getLexer().isNot(AsmToken::Comma))
3386     return true;
3387   Parser.Lex();
3388 
3389   if (getLexer().isNot(AsmToken::Integer))
3390     return true;
3391   if (getParser().parseAbsoluteExpression(Width))
3392     return true;
3393 
3394   if (getLexer().isNot(AsmToken::RParen))
3395     return true;
3396   Parser.Lex();
3397 
3398   return false;
3399 }
3400 
3401 OperandMatchResultTy AMDGPUAsmParser::parseHwreg(OperandVector &Operands) {
3402   using namespace llvm::AMDGPU::Hwreg;
3403 
3404   int64_t Imm16Val = 0;
3405   SMLoc S = Parser.getTok().getLoc();
3406 
3407   switch(getLexer().getKind()) {
3408     default: return MatchOperand_NoMatch;
3409     case AsmToken::Integer:
3410       // The operand can be an integer value.
3411       if (getParser().parseAbsoluteExpression(Imm16Val))
3412         return MatchOperand_NoMatch;
3413       if (Imm16Val < 0 || !isUInt<16>(Imm16Val)) {
3414         Error(S, "invalid immediate: only 16-bit values are legal");
3415         // Do not return error code, but create an imm operand anyway and proceed
3416         // to the next operand, if any. That avoids unneccessary error messages.
3417       }
3418       break;
3419 
3420     case AsmToken::Identifier: {
3421         OperandInfoTy HwReg(ID_UNKNOWN_);
3422         int64_t Offset = OFFSET_DEFAULT_;
3423         int64_t Width = WIDTH_M1_DEFAULT_ + 1;
3424         if (parseHwregConstruct(HwReg, Offset, Width))
3425           return MatchOperand_ParseFail;
3426         if (HwReg.Id < 0 || !isUInt<ID_WIDTH_>(HwReg.Id)) {
3427           if (HwReg.IsSymbolic)
3428             Error(S, "invalid symbolic name of hardware register");
3429           else
3430             Error(S, "invalid code of hardware register: only 6-bit values are legal");
3431         }
3432         if (Offset < 0 || !isUInt<OFFSET_WIDTH_>(Offset))
3433           Error(S, "invalid bit offset: only 5-bit values are legal");
3434         if ((Width-1) < 0 || !isUInt<WIDTH_M1_WIDTH_>(Width-1))
3435           Error(S, "invalid bitfield width: only values from 1 to 32 are legal");
3436         Imm16Val = (HwReg.Id << ID_SHIFT_) | (Offset << OFFSET_SHIFT_) | ((Width-1) << WIDTH_M1_SHIFT_);
3437       }
3438       break;
3439   }
3440   Operands.push_back(AMDGPUOperand::CreateImm(this, Imm16Val, S, AMDGPUOperand::ImmTyHwreg));
3441   return MatchOperand_Success;
3442 }
3443 
3444 bool AMDGPUOperand::isSWaitCnt() const {
3445   return isImm();
3446 }
3447 
3448 bool AMDGPUOperand::isHwreg() const {
3449   return isImmTy(ImmTyHwreg);
3450 }
3451 
3452 bool AMDGPUAsmParser::parseSendMsgConstruct(OperandInfoTy &Msg, OperandInfoTy &Operation, int64_t &StreamId) {
3453   using namespace llvm::AMDGPU::SendMsg;
3454 
3455   if (Parser.getTok().getString() != "sendmsg")
3456     return true;
3457   Parser.Lex();
3458 
3459   if (getLexer().isNot(AsmToken::LParen))
3460     return true;
3461   Parser.Lex();
3462 
3463   if (getLexer().is(AsmToken::Identifier)) {
3464     Msg.IsSymbolic = true;
3465     Msg.Id = ID_UNKNOWN_;
3466     const std::string tok = Parser.getTok().getString();
3467     for (int i = ID_GAPS_FIRST_; i < ID_GAPS_LAST_; ++i) {
3468       switch(i) {
3469         default: continue; // Omit gaps.
3470         case ID_INTERRUPT: case ID_GS: case ID_GS_DONE:  case ID_SYSMSG: break;
3471       }
3472       if (tok == IdSymbolic[i]) {
3473         Msg.Id = i;
3474         break;
3475       }
3476     }
3477     Parser.Lex();
3478   } else {
3479     Msg.IsSymbolic = false;
3480     if (getLexer().isNot(AsmToken::Integer))
3481       return true;
3482     if (getParser().parseAbsoluteExpression(Msg.Id))
3483       return true;
3484     if (getLexer().is(AsmToken::Integer))
3485       if (getParser().parseAbsoluteExpression(Msg.Id))
3486         Msg.Id = ID_UNKNOWN_;
3487   }
3488   if (Msg.Id == ID_UNKNOWN_) // Don't know how to parse the rest.
3489     return false;
3490 
3491   if (!(Msg.Id == ID_GS || Msg.Id == ID_GS_DONE || Msg.Id == ID_SYSMSG)) {
3492     if (getLexer().isNot(AsmToken::RParen))
3493       return true;
3494     Parser.Lex();
3495     return false;
3496   }
3497 
3498   if (getLexer().isNot(AsmToken::Comma))
3499     return true;
3500   Parser.Lex();
3501 
3502   assert(Msg.Id == ID_GS || Msg.Id == ID_GS_DONE || Msg.Id == ID_SYSMSG);
3503   Operation.Id = ID_UNKNOWN_;
3504   if (getLexer().is(AsmToken::Identifier)) {
3505     Operation.IsSymbolic = true;
3506     const char* const *S = (Msg.Id == ID_SYSMSG) ? OpSysSymbolic : OpGsSymbolic;
3507     const int F = (Msg.Id == ID_SYSMSG) ? OP_SYS_FIRST_ : OP_GS_FIRST_;
3508     const int L = (Msg.Id == ID_SYSMSG) ? OP_SYS_LAST_ : OP_GS_LAST_;
3509     const StringRef Tok = Parser.getTok().getString();
3510     for (int i = F; i < L; ++i) {
3511       if (Tok == S[i]) {
3512         Operation.Id = i;
3513         break;
3514       }
3515     }
3516     Parser.Lex();
3517   } else {
3518     Operation.IsSymbolic = false;
3519     if (getLexer().isNot(AsmToken::Integer))
3520       return true;
3521     if (getParser().parseAbsoluteExpression(Operation.Id))
3522       return true;
3523   }
3524 
3525   if ((Msg.Id == ID_GS || Msg.Id == ID_GS_DONE) && Operation.Id != OP_GS_NOP) {
3526     // Stream id is optional.
3527     if (getLexer().is(AsmToken::RParen)) {
3528       Parser.Lex();
3529       return false;
3530     }
3531 
3532     if (getLexer().isNot(AsmToken::Comma))
3533       return true;
3534     Parser.Lex();
3535 
3536     if (getLexer().isNot(AsmToken::Integer))
3537       return true;
3538     if (getParser().parseAbsoluteExpression(StreamId))
3539       return true;
3540   }
3541 
3542   if (getLexer().isNot(AsmToken::RParen))
3543     return true;
3544   Parser.Lex();
3545   return false;
3546 }
3547 
3548 OperandMatchResultTy AMDGPUAsmParser::parseInterpSlot(OperandVector &Operands) {
3549   if (getLexer().getKind() != AsmToken::Identifier)
3550     return MatchOperand_NoMatch;
3551 
3552   StringRef Str = Parser.getTok().getString();
3553   int Slot = StringSwitch<int>(Str)
3554     .Case("p10", 0)
3555     .Case("p20", 1)
3556     .Case("p0", 2)
3557     .Default(-1);
3558 
3559   SMLoc S = Parser.getTok().getLoc();
3560   if (Slot == -1)
3561     return MatchOperand_ParseFail;
3562 
3563   Parser.Lex();
3564   Operands.push_back(AMDGPUOperand::CreateImm(this, Slot, S,
3565                                               AMDGPUOperand::ImmTyInterpSlot));
3566   return MatchOperand_Success;
3567 }
3568 
3569 OperandMatchResultTy AMDGPUAsmParser::parseInterpAttr(OperandVector &Operands) {
3570   if (getLexer().getKind() != AsmToken::Identifier)
3571     return MatchOperand_NoMatch;
3572 
3573   StringRef Str = Parser.getTok().getString();
3574   if (!Str.startswith("attr"))
3575     return MatchOperand_NoMatch;
3576 
3577   StringRef Chan = Str.take_back(2);
3578   int AttrChan = StringSwitch<int>(Chan)
3579     .Case(".x", 0)
3580     .Case(".y", 1)
3581     .Case(".z", 2)
3582     .Case(".w", 3)
3583     .Default(-1);
3584   if (AttrChan == -1)
3585     return MatchOperand_ParseFail;
3586 
3587   Str = Str.drop_back(2).drop_front(4);
3588 
3589   uint8_t Attr;
3590   if (Str.getAsInteger(10, Attr))
3591     return MatchOperand_ParseFail;
3592 
3593   SMLoc S = Parser.getTok().getLoc();
3594   Parser.Lex();
3595   if (Attr > 63) {
3596     Error(S, "out of bounds attr");
3597     return MatchOperand_Success;
3598   }
3599 
3600   SMLoc SChan = SMLoc::getFromPointer(Chan.data());
3601 
3602   Operands.push_back(AMDGPUOperand::CreateImm(this, Attr, S,
3603                                               AMDGPUOperand::ImmTyInterpAttr));
3604   Operands.push_back(AMDGPUOperand::CreateImm(this, AttrChan, SChan,
3605                                               AMDGPUOperand::ImmTyAttrChan));
3606   return MatchOperand_Success;
3607 }
3608 
3609 void AMDGPUAsmParser::errorExpTgt() {
3610   Error(Parser.getTok().getLoc(), "invalid exp target");
3611 }
3612 
3613 OperandMatchResultTy AMDGPUAsmParser::parseExpTgtImpl(StringRef Str,
3614                                                       uint8_t &Val) {
3615   if (Str == "null") {
3616     Val = 9;
3617     return MatchOperand_Success;
3618   }
3619 
3620   if (Str.startswith("mrt")) {
3621     Str = Str.drop_front(3);
3622     if (Str == "z") { // == mrtz
3623       Val = 8;
3624       return MatchOperand_Success;
3625     }
3626 
3627     if (Str.getAsInteger(10, Val))
3628       return MatchOperand_ParseFail;
3629 
3630     if (Val > 7)
3631       errorExpTgt();
3632 
3633     return MatchOperand_Success;
3634   }
3635 
3636   if (Str.startswith("pos")) {
3637     Str = Str.drop_front(3);
3638     if (Str.getAsInteger(10, Val))
3639       return MatchOperand_ParseFail;
3640 
3641     if (Val > 3)
3642       errorExpTgt();
3643 
3644     Val += 12;
3645     return MatchOperand_Success;
3646   }
3647 
3648   if (Str.startswith("param")) {
3649     Str = Str.drop_front(5);
3650     if (Str.getAsInteger(10, Val))
3651       return MatchOperand_ParseFail;
3652 
3653     if (Val >= 32)
3654       errorExpTgt();
3655 
3656     Val += 32;
3657     return MatchOperand_Success;
3658   }
3659 
3660   if (Str.startswith("invalid_target_")) {
3661     Str = Str.drop_front(15);
3662     if (Str.getAsInteger(10, Val))
3663       return MatchOperand_ParseFail;
3664 
3665     errorExpTgt();
3666     return MatchOperand_Success;
3667   }
3668 
3669   return MatchOperand_NoMatch;
3670 }
3671 
3672 OperandMatchResultTy AMDGPUAsmParser::parseExpTgt(OperandVector &Operands) {
3673   uint8_t Val;
3674   StringRef Str = Parser.getTok().getString();
3675 
3676   auto Res = parseExpTgtImpl(Str, Val);
3677   if (Res != MatchOperand_Success)
3678     return Res;
3679 
3680   SMLoc S = Parser.getTok().getLoc();
3681   Parser.Lex();
3682 
3683   Operands.push_back(AMDGPUOperand::CreateImm(this, Val, S,
3684                                               AMDGPUOperand::ImmTyExpTgt));
3685   return MatchOperand_Success;
3686 }
3687 
3688 OperandMatchResultTy
3689 AMDGPUAsmParser::parseSendMsgOp(OperandVector &Operands) {
3690   using namespace llvm::AMDGPU::SendMsg;
3691 
3692   int64_t Imm16Val = 0;
3693   SMLoc S = Parser.getTok().getLoc();
3694 
3695   switch(getLexer().getKind()) {
3696   default:
3697     return MatchOperand_NoMatch;
3698   case AsmToken::Integer:
3699     // The operand can be an integer value.
3700     if (getParser().parseAbsoluteExpression(Imm16Val))
3701       return MatchOperand_NoMatch;
3702     if (Imm16Val < 0 || !isUInt<16>(Imm16Val)) {
3703       Error(S, "invalid immediate: only 16-bit values are legal");
3704       // Do not return error code, but create an imm operand anyway and proceed
3705       // to the next operand, if any. That avoids unneccessary error messages.
3706     }
3707     break;
3708   case AsmToken::Identifier: {
3709       OperandInfoTy Msg(ID_UNKNOWN_);
3710       OperandInfoTy Operation(OP_UNKNOWN_);
3711       int64_t StreamId = STREAM_ID_DEFAULT_;
3712       if (parseSendMsgConstruct(Msg, Operation, StreamId))
3713         return MatchOperand_ParseFail;
3714       do {
3715         // Validate and encode message ID.
3716         if (! ((ID_INTERRUPT <= Msg.Id && Msg.Id <= ID_GS_DONE)
3717                 || Msg.Id == ID_SYSMSG)) {
3718           if (Msg.IsSymbolic)
3719             Error(S, "invalid/unsupported symbolic name of message");
3720           else
3721             Error(S, "invalid/unsupported code of message");
3722           break;
3723         }
3724         Imm16Val = (Msg.Id << ID_SHIFT_);
3725         // Validate and encode operation ID.
3726         if (Msg.Id == ID_GS || Msg.Id == ID_GS_DONE) {
3727           if (! (OP_GS_FIRST_ <= Operation.Id && Operation.Id < OP_GS_LAST_)) {
3728             if (Operation.IsSymbolic)
3729               Error(S, "invalid symbolic name of GS_OP");
3730             else
3731               Error(S, "invalid code of GS_OP: only 2-bit values are legal");
3732             break;
3733           }
3734           if (Operation.Id == OP_GS_NOP
3735               && Msg.Id != ID_GS_DONE) {
3736             Error(S, "invalid GS_OP: NOP is for GS_DONE only");
3737             break;
3738           }
3739           Imm16Val |= (Operation.Id << OP_SHIFT_);
3740         }
3741         if (Msg.Id == ID_SYSMSG) {
3742           if (! (OP_SYS_FIRST_ <= Operation.Id && Operation.Id < OP_SYS_LAST_)) {
3743             if (Operation.IsSymbolic)
3744               Error(S, "invalid/unsupported symbolic name of SYSMSG_OP");
3745             else
3746               Error(S, "invalid/unsupported code of SYSMSG_OP");
3747             break;
3748           }
3749           Imm16Val |= (Operation.Id << OP_SHIFT_);
3750         }
3751         // Validate and encode stream ID.
3752         if ((Msg.Id == ID_GS || Msg.Id == ID_GS_DONE) && Operation.Id != OP_GS_NOP) {
3753           if (! (STREAM_ID_FIRST_ <= StreamId && StreamId < STREAM_ID_LAST_)) {
3754             Error(S, "invalid stream id: only 2-bit values are legal");
3755             break;
3756           }
3757           Imm16Val |= (StreamId << STREAM_ID_SHIFT_);
3758         }
3759       } while (false);
3760     }
3761     break;
3762   }
3763   Operands.push_back(AMDGPUOperand::CreateImm(this, Imm16Val, S, AMDGPUOperand::ImmTySendMsg));
3764   return MatchOperand_Success;
3765 }
3766 
3767 bool AMDGPUOperand::isSendMsg() const {
3768   return isImmTy(ImmTySendMsg);
3769 }
3770 
3771 //===----------------------------------------------------------------------===//
3772 // parser helpers
3773 //===----------------------------------------------------------------------===//
3774 
3775 bool
3776 AMDGPUAsmParser::trySkipId(const StringRef Id) {
3777   if (getLexer().getKind() == AsmToken::Identifier &&
3778       Parser.getTok().getString() == Id) {
3779     Parser.Lex();
3780     return true;
3781   }
3782   return false;
3783 }
3784 
3785 bool
3786 AMDGPUAsmParser::trySkipToken(const AsmToken::TokenKind Kind) {
3787   if (getLexer().getKind() == Kind) {
3788     Parser.Lex();
3789     return true;
3790   }
3791   return false;
3792 }
3793 
3794 bool
3795 AMDGPUAsmParser::skipToken(const AsmToken::TokenKind Kind,
3796                            const StringRef ErrMsg) {
3797   if (!trySkipToken(Kind)) {
3798     Error(Parser.getTok().getLoc(), ErrMsg);
3799     return false;
3800   }
3801   return true;
3802 }
3803 
3804 bool
3805 AMDGPUAsmParser::parseExpr(int64_t &Imm) {
3806   return !getParser().parseAbsoluteExpression(Imm);
3807 }
3808 
3809 bool
3810 AMDGPUAsmParser::parseString(StringRef &Val, const StringRef ErrMsg) {
3811   SMLoc S = Parser.getTok().getLoc();
3812   if (getLexer().getKind() == AsmToken::String) {
3813     Val = Parser.getTok().getStringContents();
3814     Parser.Lex();
3815     return true;
3816   } else {
3817     Error(S, ErrMsg);
3818     return false;
3819   }
3820 }
3821 
3822 //===----------------------------------------------------------------------===//
3823 // swizzle
3824 //===----------------------------------------------------------------------===//
3825 
3826 LLVM_READNONE
3827 static unsigned
3828 encodeBitmaskPerm(const unsigned AndMask,
3829                   const unsigned OrMask,
3830                   const unsigned XorMask) {
3831   using namespace llvm::AMDGPU::Swizzle;
3832 
3833   return BITMASK_PERM_ENC |
3834          (AndMask << BITMASK_AND_SHIFT) |
3835          (OrMask  << BITMASK_OR_SHIFT)  |
3836          (XorMask << BITMASK_XOR_SHIFT);
3837 }
3838 
3839 bool
3840 AMDGPUAsmParser::parseSwizzleOperands(const unsigned OpNum, int64_t* Op,
3841                                       const unsigned MinVal,
3842                                       const unsigned MaxVal,
3843                                       const StringRef ErrMsg) {
3844   for (unsigned i = 0; i < OpNum; ++i) {
3845     if (!skipToken(AsmToken::Comma, "expected a comma")){
3846       return false;
3847     }
3848     SMLoc ExprLoc = Parser.getTok().getLoc();
3849     if (!parseExpr(Op[i])) {
3850       return false;
3851     }
3852     if (Op[i] < MinVal || Op[i] > MaxVal) {
3853       Error(ExprLoc, ErrMsg);
3854       return false;
3855     }
3856   }
3857 
3858   return true;
3859 }
3860 
3861 bool
3862 AMDGPUAsmParser::parseSwizzleQuadPerm(int64_t &Imm) {
3863   using namespace llvm::AMDGPU::Swizzle;
3864 
3865   int64_t Lane[LANE_NUM];
3866   if (parseSwizzleOperands(LANE_NUM, Lane, 0, LANE_MAX,
3867                            "expected a 2-bit lane id")) {
3868     Imm = QUAD_PERM_ENC;
3869     for (auto i = 0; i < LANE_NUM; ++i) {
3870       Imm |= Lane[i] << (LANE_SHIFT * i);
3871     }
3872     return true;
3873   }
3874   return false;
3875 }
3876 
3877 bool
3878 AMDGPUAsmParser::parseSwizzleBroadcast(int64_t &Imm) {
3879   using namespace llvm::AMDGPU::Swizzle;
3880 
3881   SMLoc S = Parser.getTok().getLoc();
3882   int64_t GroupSize;
3883   int64_t LaneIdx;
3884 
3885   if (!parseSwizzleOperands(1, &GroupSize,
3886                             2, 32,
3887                             "group size must be in the interval [2,32]")) {
3888     return false;
3889   }
3890   if (!isPowerOf2_64(GroupSize)) {
3891     Error(S, "group size must be a power of two");
3892     return false;
3893   }
3894   if (parseSwizzleOperands(1, &LaneIdx,
3895                            0, GroupSize - 1,
3896                            "lane id must be in the interval [0,group size - 1]")) {
3897     Imm = encodeBitmaskPerm(BITMASK_MAX - GroupSize + 1, LaneIdx, 0);
3898     return true;
3899   }
3900   return false;
3901 }
3902 
3903 bool
3904 AMDGPUAsmParser::parseSwizzleReverse(int64_t &Imm) {
3905   using namespace llvm::AMDGPU::Swizzle;
3906 
3907   SMLoc S = Parser.getTok().getLoc();
3908   int64_t GroupSize;
3909 
3910   if (!parseSwizzleOperands(1, &GroupSize,
3911       2, 32, "group size must be in the interval [2,32]")) {
3912     return false;
3913   }
3914   if (!isPowerOf2_64(GroupSize)) {
3915     Error(S, "group size must be a power of two");
3916     return false;
3917   }
3918 
3919   Imm = encodeBitmaskPerm(BITMASK_MAX, 0, GroupSize - 1);
3920   return true;
3921 }
3922 
3923 bool
3924 AMDGPUAsmParser::parseSwizzleSwap(int64_t &Imm) {
3925   using namespace llvm::AMDGPU::Swizzle;
3926 
3927   SMLoc S = Parser.getTok().getLoc();
3928   int64_t GroupSize;
3929 
3930   if (!parseSwizzleOperands(1, &GroupSize,
3931       1, 16, "group size must be in the interval [1,16]")) {
3932     return false;
3933   }
3934   if (!isPowerOf2_64(GroupSize)) {
3935     Error(S, "group size must be a power of two");
3936     return false;
3937   }
3938 
3939   Imm = encodeBitmaskPerm(BITMASK_MAX, 0, GroupSize);
3940   return true;
3941 }
3942 
3943 bool
3944 AMDGPUAsmParser::parseSwizzleBitmaskPerm(int64_t &Imm) {
3945   using namespace llvm::AMDGPU::Swizzle;
3946 
3947   if (!skipToken(AsmToken::Comma, "expected a comma")) {
3948     return false;
3949   }
3950 
3951   StringRef Ctl;
3952   SMLoc StrLoc = Parser.getTok().getLoc();
3953   if (!parseString(Ctl)) {
3954     return false;
3955   }
3956   if (Ctl.size() != BITMASK_WIDTH) {
3957     Error(StrLoc, "expected a 5-character mask");
3958     return false;
3959   }
3960 
3961   unsigned AndMask = 0;
3962   unsigned OrMask = 0;
3963   unsigned XorMask = 0;
3964 
3965   for (size_t i = 0; i < Ctl.size(); ++i) {
3966     unsigned Mask = 1 << (BITMASK_WIDTH - 1 - i);
3967     switch(Ctl[i]) {
3968     default:
3969       Error(StrLoc, "invalid mask");
3970       return false;
3971     case '0':
3972       break;
3973     case '1':
3974       OrMask |= Mask;
3975       break;
3976     case 'p':
3977       AndMask |= Mask;
3978       break;
3979     case 'i':
3980       AndMask |= Mask;
3981       XorMask |= Mask;
3982       break;
3983     }
3984   }
3985 
3986   Imm = encodeBitmaskPerm(AndMask, OrMask, XorMask);
3987   return true;
3988 }
3989 
3990 bool
3991 AMDGPUAsmParser::parseSwizzleOffset(int64_t &Imm) {
3992 
3993   SMLoc OffsetLoc = Parser.getTok().getLoc();
3994 
3995   if (!parseExpr(Imm)) {
3996     return false;
3997   }
3998   if (!isUInt<16>(Imm)) {
3999     Error(OffsetLoc, "expected a 16-bit offset");
4000     return false;
4001   }
4002   return true;
4003 }
4004 
4005 bool
4006 AMDGPUAsmParser::parseSwizzleMacro(int64_t &Imm) {
4007   using namespace llvm::AMDGPU::Swizzle;
4008 
4009   if (skipToken(AsmToken::LParen, "expected a left parentheses")) {
4010 
4011     SMLoc ModeLoc = Parser.getTok().getLoc();
4012     bool Ok = false;
4013 
4014     if (trySkipId(IdSymbolic[ID_QUAD_PERM])) {
4015       Ok = parseSwizzleQuadPerm(Imm);
4016     } else if (trySkipId(IdSymbolic[ID_BITMASK_PERM])) {
4017       Ok = parseSwizzleBitmaskPerm(Imm);
4018     } else if (trySkipId(IdSymbolic[ID_BROADCAST])) {
4019       Ok = parseSwizzleBroadcast(Imm);
4020     } else if (trySkipId(IdSymbolic[ID_SWAP])) {
4021       Ok = parseSwizzleSwap(Imm);
4022     } else if (trySkipId(IdSymbolic[ID_REVERSE])) {
4023       Ok = parseSwizzleReverse(Imm);
4024     } else {
4025       Error(ModeLoc, "expected a swizzle mode");
4026     }
4027 
4028     return Ok && skipToken(AsmToken::RParen, "expected a closing parentheses");
4029   }
4030 
4031   return false;
4032 }
4033 
4034 OperandMatchResultTy
4035 AMDGPUAsmParser::parseSwizzleOp(OperandVector &Operands) {
4036   SMLoc S = Parser.getTok().getLoc();
4037   int64_t Imm = 0;
4038 
4039   if (trySkipId("offset")) {
4040 
4041     bool Ok = false;
4042     if (skipToken(AsmToken::Colon, "expected a colon")) {
4043       if (trySkipId("swizzle")) {
4044         Ok = parseSwizzleMacro(Imm);
4045       } else {
4046         Ok = parseSwizzleOffset(Imm);
4047       }
4048     }
4049 
4050     Operands.push_back(AMDGPUOperand::CreateImm(this, Imm, S, AMDGPUOperand::ImmTySwizzle));
4051 
4052     return Ok? MatchOperand_Success : MatchOperand_ParseFail;
4053   } else {
4054     // Swizzle "offset" operand is optional.
4055     // If it is omitted, try parsing other optional operands.
4056     return parseOptionalOpr(Operands);
4057   }
4058 }
4059 
4060 bool
4061 AMDGPUOperand::isSwizzle() const {
4062   return isImmTy(ImmTySwizzle);
4063 }
4064 
4065 //===----------------------------------------------------------------------===//
4066 // sopp branch targets
4067 //===----------------------------------------------------------------------===//
4068 
4069 OperandMatchResultTy
4070 AMDGPUAsmParser::parseSOppBrTarget(OperandVector &Operands) {
4071   SMLoc S = Parser.getTok().getLoc();
4072 
4073   switch (getLexer().getKind()) {
4074     default: return MatchOperand_ParseFail;
4075     case AsmToken::Integer: {
4076       int64_t Imm;
4077       if (getParser().parseAbsoluteExpression(Imm))
4078         return MatchOperand_ParseFail;
4079       Operands.push_back(AMDGPUOperand::CreateImm(this, Imm, S));
4080       return MatchOperand_Success;
4081     }
4082 
4083     case AsmToken::Identifier:
4084       Operands.push_back(AMDGPUOperand::CreateExpr(this,
4085           MCSymbolRefExpr::create(getContext().getOrCreateSymbol(
4086                                   Parser.getTok().getString()), getContext()), S));
4087       Parser.Lex();
4088       return MatchOperand_Success;
4089   }
4090 }
4091 
4092 //===----------------------------------------------------------------------===//
4093 // mubuf
4094 //===----------------------------------------------------------------------===//
4095 
4096 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultGLC() const {
4097   return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTyGLC);
4098 }
4099 
4100 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultSLC() const {
4101   return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTySLC);
4102 }
4103 
4104 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultTFE() const {
4105   return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTyTFE);
4106 }
4107 
4108 void AMDGPUAsmParser::cvtMubufImpl(MCInst &Inst,
4109                                const OperandVector &Operands,
4110                                bool IsAtomic,
4111                                bool IsAtomicReturn,
4112                                bool IsLds) {
4113   bool IsLdsOpcode = IsLds;
4114   bool HasLdsModifier = false;
4115   OptionalImmIndexMap OptionalIdx;
4116   assert(IsAtomicReturn ? IsAtomic : true);
4117 
4118   for (unsigned i = 1, e = Operands.size(); i != e; ++i) {
4119     AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[i]);
4120 
4121     // Add the register arguments
4122     if (Op.isReg()) {
4123       Op.addRegOperands(Inst, 1);
4124       continue;
4125     }
4126 
4127     // Handle the case where soffset is an immediate
4128     if (Op.isImm() && Op.getImmTy() == AMDGPUOperand::ImmTyNone) {
4129       Op.addImmOperands(Inst, 1);
4130       continue;
4131     }
4132 
4133     HasLdsModifier = Op.isLDS();
4134 
4135     // Handle tokens like 'offen' which are sometimes hard-coded into the
4136     // asm string.  There are no MCInst operands for these.
4137     if (Op.isToken()) {
4138       continue;
4139     }
4140     assert(Op.isImm());
4141 
4142     // Handle optional arguments
4143     OptionalIdx[Op.getImmTy()] = i;
4144   }
4145 
4146   // This is a workaround for an llvm quirk which may result in an
4147   // incorrect instruction selection. Lds and non-lds versions of
4148   // MUBUF instructions are identical except that lds versions
4149   // have mandatory 'lds' modifier. However this modifier follows
4150   // optional modifiers and llvm asm matcher regards this 'lds'
4151   // modifier as an optional one. As a result, an lds version
4152   // of opcode may be selected even if it has no 'lds' modifier.
4153   if (IsLdsOpcode && !HasLdsModifier) {
4154     int NoLdsOpcode = AMDGPU::getMUBUFNoLdsInst(Inst.getOpcode());
4155     if (NoLdsOpcode != -1) { // Got lds version - correct it.
4156       Inst.setOpcode(NoLdsOpcode);
4157       IsLdsOpcode = false;
4158     }
4159   }
4160 
4161   // Copy $vdata_in operand and insert as $vdata for MUBUF_Atomic RTN insns.
4162   if (IsAtomicReturn) {
4163     MCInst::iterator I = Inst.begin(); // $vdata_in is always at the beginning.
4164     Inst.insert(I, *I);
4165   }
4166 
4167   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyOffset);
4168   if (!IsAtomic) { // glc is hard-coded.
4169     addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyGLC);
4170   }
4171   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySLC);
4172 
4173   if (!IsLdsOpcode) { // tfe is not legal with lds opcodes
4174     addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyTFE);
4175   }
4176 }
4177 
4178 void AMDGPUAsmParser::cvtMtbuf(MCInst &Inst, const OperandVector &Operands) {
4179   OptionalImmIndexMap OptionalIdx;
4180 
4181   for (unsigned i = 1, e = Operands.size(); i != e; ++i) {
4182     AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[i]);
4183 
4184     // Add the register arguments
4185     if (Op.isReg()) {
4186       Op.addRegOperands(Inst, 1);
4187       continue;
4188     }
4189 
4190     // Handle the case where soffset is an immediate
4191     if (Op.isImm() && Op.getImmTy() == AMDGPUOperand::ImmTyNone) {
4192       Op.addImmOperands(Inst, 1);
4193       continue;
4194     }
4195 
4196     // Handle tokens like 'offen' which are sometimes hard-coded into the
4197     // asm string.  There are no MCInst operands for these.
4198     if (Op.isToken()) {
4199       continue;
4200     }
4201     assert(Op.isImm());
4202 
4203     // Handle optional arguments
4204     OptionalIdx[Op.getImmTy()] = i;
4205   }
4206 
4207   addOptionalImmOperand(Inst, Operands, OptionalIdx,
4208                         AMDGPUOperand::ImmTyOffset);
4209   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyDFMT);
4210   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyNFMT);
4211   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyGLC);
4212   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySLC);
4213   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyTFE);
4214 }
4215 
4216 //===----------------------------------------------------------------------===//
4217 // mimg
4218 //===----------------------------------------------------------------------===//
4219 
4220 void AMDGPUAsmParser::cvtMIMG(MCInst &Inst, const OperandVector &Operands,
4221                               bool IsAtomic) {
4222   unsigned I = 1;
4223   const MCInstrDesc &Desc = MII.get(Inst.getOpcode());
4224   for (unsigned J = 0; J < Desc.getNumDefs(); ++J) {
4225     ((AMDGPUOperand &)*Operands[I++]).addRegOperands(Inst, 1);
4226   }
4227 
4228   if (IsAtomic) {
4229     // Add src, same as dst
4230     assert(Desc.getNumDefs() == 1);
4231     ((AMDGPUOperand &)*Operands[I - 1]).addRegOperands(Inst, 1);
4232   }
4233 
4234   OptionalImmIndexMap OptionalIdx;
4235 
4236   for (unsigned E = Operands.size(); I != E; ++I) {
4237     AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[I]);
4238 
4239     // Add the register arguments
4240     if (Op.isReg()) {
4241       Op.addRegOperands(Inst, 1);
4242     } else if (Op.isImmModifier()) {
4243       OptionalIdx[Op.getImmTy()] = I;
4244     } else {
4245       llvm_unreachable("unexpected operand type");
4246     }
4247   }
4248 
4249   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyDMask);
4250   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyUNorm);
4251   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyGLC);
4252   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySLC);
4253   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyR128);
4254   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyTFE);
4255   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyLWE);
4256   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyDA);
4257 }
4258 
4259 void AMDGPUAsmParser::cvtMIMGAtomic(MCInst &Inst, const OperandVector &Operands) {
4260   cvtMIMG(Inst, Operands, true);
4261 }
4262 
4263 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultDMask() const {
4264   return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTyDMask);
4265 }
4266 
4267 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultUNorm() const {
4268   return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTyUNorm);
4269 }
4270 
4271 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultDA() const {
4272   return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTyDA);
4273 }
4274 
4275 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultR128() const {
4276   return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTyR128);
4277 }
4278 
4279 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultLWE() const {
4280   return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTyLWE);
4281 }
4282 
4283 //===----------------------------------------------------------------------===//
4284 // smrd
4285 //===----------------------------------------------------------------------===//
4286 
4287 bool AMDGPUOperand::isSMRDOffset8() const {
4288   return isImm() && isUInt<8>(getImm());
4289 }
4290 
4291 bool AMDGPUOperand::isSMRDOffset20() const {
4292   return isImm() && isUInt<20>(getImm());
4293 }
4294 
4295 bool AMDGPUOperand::isSMRDLiteralOffset() const {
4296   // 32-bit literals are only supported on CI and we only want to use them
4297   // when the offset is > 8-bits.
4298   return isImm() && !isUInt<8>(getImm()) && isUInt<32>(getImm());
4299 }
4300 
4301 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultSMRDOffset8() const {
4302   return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTyOffset);
4303 }
4304 
4305 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultSMRDOffset20() const {
4306   return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTyOffset);
4307 }
4308 
4309 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultSMRDLiteralOffset() const {
4310   return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTyOffset);
4311 }
4312 
4313 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultOffsetU12() const {
4314   return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTyOffset);
4315 }
4316 
4317 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultOffsetS13() const {
4318   return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTyOffset);
4319 }
4320 
4321 //===----------------------------------------------------------------------===//
4322 // vop3
4323 //===----------------------------------------------------------------------===//
4324 
4325 static bool ConvertOmodMul(int64_t &Mul) {
4326   if (Mul != 1 && Mul != 2 && Mul != 4)
4327     return false;
4328 
4329   Mul >>= 1;
4330   return true;
4331 }
4332 
4333 static bool ConvertOmodDiv(int64_t &Div) {
4334   if (Div == 1) {
4335     Div = 0;
4336     return true;
4337   }
4338 
4339   if (Div == 2) {
4340     Div = 3;
4341     return true;
4342   }
4343 
4344   return false;
4345 }
4346 
4347 static bool ConvertBoundCtrl(int64_t &BoundCtrl) {
4348   if (BoundCtrl == 0) {
4349     BoundCtrl = 1;
4350     return true;
4351   }
4352 
4353   if (BoundCtrl == -1) {
4354     BoundCtrl = 0;
4355     return true;
4356   }
4357 
4358   return false;
4359 }
4360 
4361 // Note: the order in this table matches the order of operands in AsmString.
4362 static const OptionalOperand AMDGPUOptionalOperandTable[] = {
4363   {"offen",   AMDGPUOperand::ImmTyOffen, true, nullptr},
4364   {"idxen",   AMDGPUOperand::ImmTyIdxen, true, nullptr},
4365   {"addr64",  AMDGPUOperand::ImmTyAddr64, true, nullptr},
4366   {"offset0", AMDGPUOperand::ImmTyOffset0, false, nullptr},
4367   {"offset1", AMDGPUOperand::ImmTyOffset1, false, nullptr},
4368   {"gds",     AMDGPUOperand::ImmTyGDS, true, nullptr},
4369   {"lds",     AMDGPUOperand::ImmTyLDS, true, nullptr},
4370   {"offset",  AMDGPUOperand::ImmTyOffset, false, nullptr},
4371   {"inst_offset", AMDGPUOperand::ImmTyInstOffset, false, nullptr},
4372   {"dfmt",    AMDGPUOperand::ImmTyDFMT, false, nullptr},
4373   {"nfmt",    AMDGPUOperand::ImmTyNFMT, false, nullptr},
4374   {"glc",     AMDGPUOperand::ImmTyGLC, true, nullptr},
4375   {"slc",     AMDGPUOperand::ImmTySLC, true, nullptr},
4376   {"tfe",     AMDGPUOperand::ImmTyTFE, true, nullptr},
4377   {"d16",     AMDGPUOperand::ImmTyD16, true, nullptr},
4378   {"high",    AMDGPUOperand::ImmTyHigh, true, nullptr},
4379   {"clamp",   AMDGPUOperand::ImmTyClampSI, true, nullptr},
4380   {"omod",    AMDGPUOperand::ImmTyOModSI, false, ConvertOmodMul},
4381   {"unorm",   AMDGPUOperand::ImmTyUNorm, true, nullptr},
4382   {"da",      AMDGPUOperand::ImmTyDA,    true, nullptr},
4383   {"r128",    AMDGPUOperand::ImmTyR128,  true, nullptr},
4384   {"lwe",     AMDGPUOperand::ImmTyLWE,   true, nullptr},
4385   {"dmask",   AMDGPUOperand::ImmTyDMask, false, nullptr},
4386   {"row_mask",   AMDGPUOperand::ImmTyDppRowMask, false, nullptr},
4387   {"bank_mask",  AMDGPUOperand::ImmTyDppBankMask, false, nullptr},
4388   {"bound_ctrl", AMDGPUOperand::ImmTyDppBoundCtrl, false, ConvertBoundCtrl},
4389   {"dst_sel",    AMDGPUOperand::ImmTySdwaDstSel, false, nullptr},
4390   {"src0_sel",   AMDGPUOperand::ImmTySdwaSrc0Sel, false, nullptr},
4391   {"src1_sel",   AMDGPUOperand::ImmTySdwaSrc1Sel, false, nullptr},
4392   {"dst_unused", AMDGPUOperand::ImmTySdwaDstUnused, false, nullptr},
4393   {"compr", AMDGPUOperand::ImmTyExpCompr, true, nullptr },
4394   {"vm", AMDGPUOperand::ImmTyExpVM, true, nullptr},
4395   {"op_sel", AMDGPUOperand::ImmTyOpSel, false, nullptr},
4396   {"op_sel_hi", AMDGPUOperand::ImmTyOpSelHi, false, nullptr},
4397   {"neg_lo", AMDGPUOperand::ImmTyNegLo, false, nullptr},
4398   {"neg_hi", AMDGPUOperand::ImmTyNegHi, false, nullptr}
4399 };
4400 
4401 OperandMatchResultTy AMDGPUAsmParser::parseOptionalOperand(OperandVector &Operands) {
4402   unsigned size = Operands.size();
4403   assert(size > 0);
4404 
4405   OperandMatchResultTy res = parseOptionalOpr(Operands);
4406 
4407   // This is a hack to enable hardcoded mandatory operands which follow
4408   // optional operands.
4409   //
4410   // Current design assumes that all operands after the first optional operand
4411   // are also optional. However implementation of some instructions violates
4412   // this rule (see e.g. flat/global atomic which have hardcoded 'glc' operands).
4413   //
4414   // To alleviate this problem, we have to (implicitly) parse extra operands
4415   // to make sure autogenerated parser of custom operands never hit hardcoded
4416   // mandatory operands.
4417 
4418   if (size == 1 || ((AMDGPUOperand &)*Operands[size - 1]).isRegKind()) {
4419 
4420     // We have parsed the first optional operand.
4421     // Parse as many operands as necessary to skip all mandatory operands.
4422 
4423     for (unsigned i = 0; i < MAX_OPR_LOOKAHEAD; ++i) {
4424       if (res != MatchOperand_Success ||
4425           getLexer().is(AsmToken::EndOfStatement)) break;
4426       if (getLexer().is(AsmToken::Comma)) Parser.Lex();
4427       res = parseOptionalOpr(Operands);
4428     }
4429   }
4430 
4431   return res;
4432 }
4433 
4434 OperandMatchResultTy AMDGPUAsmParser::parseOptionalOpr(OperandVector &Operands) {
4435   OperandMatchResultTy res;
4436   for (const OptionalOperand &Op : AMDGPUOptionalOperandTable) {
4437     // try to parse any optional operand here
4438     if (Op.IsBit) {
4439       res = parseNamedBit(Op.Name, Operands, Op.Type);
4440     } else if (Op.Type == AMDGPUOperand::ImmTyOModSI) {
4441       res = parseOModOperand(Operands);
4442     } else if (Op.Type == AMDGPUOperand::ImmTySdwaDstSel ||
4443                Op.Type == AMDGPUOperand::ImmTySdwaSrc0Sel ||
4444                Op.Type == AMDGPUOperand::ImmTySdwaSrc1Sel) {
4445       res = parseSDWASel(Operands, Op.Name, Op.Type);
4446     } else if (Op.Type == AMDGPUOperand::ImmTySdwaDstUnused) {
4447       res = parseSDWADstUnused(Operands);
4448     } else if (Op.Type == AMDGPUOperand::ImmTyOpSel ||
4449                Op.Type == AMDGPUOperand::ImmTyOpSelHi ||
4450                Op.Type == AMDGPUOperand::ImmTyNegLo ||
4451                Op.Type == AMDGPUOperand::ImmTyNegHi) {
4452       res = parseOperandArrayWithPrefix(Op.Name, Operands, Op.Type,
4453                                         Op.ConvertResult);
4454     } else {
4455       res = parseIntWithPrefix(Op.Name, Operands, Op.Type, Op.ConvertResult);
4456     }
4457     if (res != MatchOperand_NoMatch) {
4458       return res;
4459     }
4460   }
4461   return MatchOperand_NoMatch;
4462 }
4463 
4464 OperandMatchResultTy AMDGPUAsmParser::parseOModOperand(OperandVector &Operands) {
4465   StringRef Name = Parser.getTok().getString();
4466   if (Name == "mul") {
4467     return parseIntWithPrefix("mul", Operands,
4468                               AMDGPUOperand::ImmTyOModSI, ConvertOmodMul);
4469   }
4470 
4471   if (Name == "div") {
4472     return parseIntWithPrefix("div", Operands,
4473                               AMDGPUOperand::ImmTyOModSI, ConvertOmodDiv);
4474   }
4475 
4476   return MatchOperand_NoMatch;
4477 }
4478 
4479 void AMDGPUAsmParser::cvtVOP3OpSel(MCInst &Inst, const OperandVector &Operands) {
4480   cvtVOP3P(Inst, Operands);
4481 
4482   int Opc = Inst.getOpcode();
4483 
4484   int SrcNum;
4485   const int Ops[] = { AMDGPU::OpName::src0,
4486                       AMDGPU::OpName::src1,
4487                       AMDGPU::OpName::src2 };
4488   for (SrcNum = 0;
4489        SrcNum < 3 && AMDGPU::getNamedOperandIdx(Opc, Ops[SrcNum]) != -1;
4490        ++SrcNum);
4491   assert(SrcNum > 0);
4492 
4493   int OpSelIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::op_sel);
4494   unsigned OpSel = Inst.getOperand(OpSelIdx).getImm();
4495 
4496   if ((OpSel & (1 << SrcNum)) != 0) {
4497     int ModIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src0_modifiers);
4498     uint32_t ModVal = Inst.getOperand(ModIdx).getImm();
4499     Inst.getOperand(ModIdx).setImm(ModVal | SISrcMods::DST_OP_SEL);
4500   }
4501 }
4502 
4503 static bool isRegOrImmWithInputMods(const MCInstrDesc &Desc, unsigned OpNum) {
4504       // 1. This operand is input modifiers
4505   return Desc.OpInfo[OpNum].OperandType == AMDGPU::OPERAND_INPUT_MODS
4506       // 2. This is not last operand
4507       && Desc.NumOperands > (OpNum + 1)
4508       // 3. Next operand is register class
4509       && Desc.OpInfo[OpNum + 1].RegClass != -1
4510       // 4. Next register is not tied to any other operand
4511       && Desc.getOperandConstraint(OpNum + 1, MCOI::OperandConstraint::TIED_TO) == -1;
4512 }
4513 
4514 void AMDGPUAsmParser::cvtVOP3Interp(MCInst &Inst, const OperandVector &Operands)
4515 {
4516   OptionalImmIndexMap OptionalIdx;
4517   unsigned Opc = Inst.getOpcode();
4518 
4519   unsigned I = 1;
4520   const MCInstrDesc &Desc = MII.get(Inst.getOpcode());
4521   for (unsigned J = 0; J < Desc.getNumDefs(); ++J) {
4522     ((AMDGPUOperand &)*Operands[I++]).addRegOperands(Inst, 1);
4523   }
4524 
4525   for (unsigned E = Operands.size(); I != E; ++I) {
4526     AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[I]);
4527     if (isRegOrImmWithInputMods(Desc, Inst.getNumOperands())) {
4528       Op.addRegOrImmWithFPInputModsOperands(Inst, 2);
4529     } else if (Op.isInterpSlot() ||
4530                Op.isInterpAttr() ||
4531                Op.isAttrChan()) {
4532       Inst.addOperand(MCOperand::createImm(Op.Imm.Val));
4533     } else if (Op.isImmModifier()) {
4534       OptionalIdx[Op.getImmTy()] = I;
4535     } else {
4536       llvm_unreachable("unhandled operand type");
4537     }
4538   }
4539 
4540   if (AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::high) != -1) {
4541     addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyHigh);
4542   }
4543 
4544   if (AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::clamp) != -1) {
4545     addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyClampSI);
4546   }
4547 
4548   if (AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::omod) != -1) {
4549     addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyOModSI);
4550   }
4551 }
4552 
4553 void AMDGPUAsmParser::cvtVOP3(MCInst &Inst, const OperandVector &Operands,
4554                               OptionalImmIndexMap &OptionalIdx) {
4555   unsigned Opc = Inst.getOpcode();
4556 
4557   unsigned I = 1;
4558   const MCInstrDesc &Desc = MII.get(Inst.getOpcode());
4559   for (unsigned J = 0; J < Desc.getNumDefs(); ++J) {
4560     ((AMDGPUOperand &)*Operands[I++]).addRegOperands(Inst, 1);
4561   }
4562 
4563   if (AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src0_modifiers) != -1) {
4564     // This instruction has src modifiers
4565     for (unsigned E = Operands.size(); I != E; ++I) {
4566       AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[I]);
4567       if (isRegOrImmWithInputMods(Desc, Inst.getNumOperands())) {
4568         Op.addRegOrImmWithFPInputModsOperands(Inst, 2);
4569       } else if (Op.isImmModifier()) {
4570         OptionalIdx[Op.getImmTy()] = I;
4571       } else if (Op.isRegOrImm()) {
4572         Op.addRegOrImmOperands(Inst, 1);
4573       } else {
4574         llvm_unreachable("unhandled operand type");
4575       }
4576     }
4577   } else {
4578     // No src modifiers
4579     for (unsigned E = Operands.size(); I != E; ++I) {
4580       AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[I]);
4581       if (Op.isMod()) {
4582         OptionalIdx[Op.getImmTy()] = I;
4583       } else {
4584         Op.addRegOrImmOperands(Inst, 1);
4585       }
4586     }
4587   }
4588 
4589   if (AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::clamp) != -1) {
4590     addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyClampSI);
4591   }
4592 
4593   if (AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::omod) != -1) {
4594     addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyOModSI);
4595   }
4596 
4597   // special case v_mac_{f16, f32}:
4598   // it has src2 register operand that is tied to dst operand
4599   // we don't allow modifiers for this operand in assembler so src2_modifiers
4600   // should be 0
4601   if (Opc == AMDGPU::V_MAC_F32_e64_si || Opc == AMDGPU::V_MAC_F32_e64_vi ||
4602       Opc == AMDGPU::V_MAC_F16_e64_vi) {
4603     auto it = Inst.begin();
4604     std::advance(it, AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src2_modifiers));
4605     it = Inst.insert(it, MCOperand::createImm(0)); // no modifiers for src2
4606     ++it;
4607     Inst.insert(it, Inst.getOperand(0)); // src2 = dst
4608   }
4609 }
4610 
4611 void AMDGPUAsmParser::cvtVOP3(MCInst &Inst, const OperandVector &Operands) {
4612   OptionalImmIndexMap OptionalIdx;
4613   cvtVOP3(Inst, Operands, OptionalIdx);
4614 }
4615 
4616 void AMDGPUAsmParser::cvtVOP3P(MCInst &Inst,
4617                                const OperandVector &Operands) {
4618   OptionalImmIndexMap OptIdx;
4619   const int Opc = Inst.getOpcode();
4620   const MCInstrDesc &Desc = MII.get(Opc);
4621 
4622   const bool IsPacked = (Desc.TSFlags & SIInstrFlags::IsPacked) != 0;
4623 
4624   cvtVOP3(Inst, Operands, OptIdx);
4625 
4626   if (AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::vdst_in) != -1) {
4627     assert(!IsPacked);
4628     Inst.addOperand(Inst.getOperand(0));
4629   }
4630 
4631   // FIXME: This is messy. Parse the modifiers as if it was a normal VOP3
4632   // instruction, and then figure out where to actually put the modifiers
4633 
4634   addOptionalImmOperand(Inst, Operands, OptIdx, AMDGPUOperand::ImmTyOpSel);
4635 
4636   int OpSelHiIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::op_sel_hi);
4637   if (OpSelHiIdx != -1) {
4638     int DefaultVal = IsPacked ? -1 : 0;
4639     addOptionalImmOperand(Inst, Operands, OptIdx, AMDGPUOperand::ImmTyOpSelHi,
4640                           DefaultVal);
4641   }
4642 
4643   int NegLoIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::neg_lo);
4644   if (NegLoIdx != -1) {
4645     assert(IsPacked);
4646     addOptionalImmOperand(Inst, Operands, OptIdx, AMDGPUOperand::ImmTyNegLo);
4647     addOptionalImmOperand(Inst, Operands, OptIdx, AMDGPUOperand::ImmTyNegHi);
4648   }
4649 
4650   const int Ops[] = { AMDGPU::OpName::src0,
4651                       AMDGPU::OpName::src1,
4652                       AMDGPU::OpName::src2 };
4653   const int ModOps[] = { AMDGPU::OpName::src0_modifiers,
4654                          AMDGPU::OpName::src1_modifiers,
4655                          AMDGPU::OpName::src2_modifiers };
4656 
4657   int OpSelIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::op_sel);
4658 
4659   unsigned OpSel = Inst.getOperand(OpSelIdx).getImm();
4660   unsigned OpSelHi = 0;
4661   unsigned NegLo = 0;
4662   unsigned NegHi = 0;
4663 
4664   if (OpSelHiIdx != -1) {
4665     OpSelHi = Inst.getOperand(OpSelHiIdx).getImm();
4666   }
4667 
4668   if (NegLoIdx != -1) {
4669     int NegHiIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::neg_hi);
4670     NegLo = Inst.getOperand(NegLoIdx).getImm();
4671     NegHi = Inst.getOperand(NegHiIdx).getImm();
4672   }
4673 
4674   for (int J = 0; J < 3; ++J) {
4675     int OpIdx = AMDGPU::getNamedOperandIdx(Opc, Ops[J]);
4676     if (OpIdx == -1)
4677       break;
4678 
4679     uint32_t ModVal = 0;
4680 
4681     if ((OpSel & (1 << J)) != 0)
4682       ModVal |= SISrcMods::OP_SEL_0;
4683 
4684     if ((OpSelHi & (1 << J)) != 0)
4685       ModVal |= SISrcMods::OP_SEL_1;
4686 
4687     if ((NegLo & (1 << J)) != 0)
4688       ModVal |= SISrcMods::NEG;
4689 
4690     if ((NegHi & (1 << J)) != 0)
4691       ModVal |= SISrcMods::NEG_HI;
4692 
4693     int ModIdx = AMDGPU::getNamedOperandIdx(Opc, ModOps[J]);
4694 
4695     Inst.getOperand(ModIdx).setImm(Inst.getOperand(ModIdx).getImm() | ModVal);
4696   }
4697 }
4698 
4699 //===----------------------------------------------------------------------===//
4700 // dpp
4701 //===----------------------------------------------------------------------===//
4702 
4703 bool AMDGPUOperand::isDPPCtrl() const {
4704   bool result = isImm() && getImmTy() == ImmTyDppCtrl && isUInt<9>(getImm());
4705   if (result) {
4706     int64_t Imm = getImm();
4707     return ((Imm >= 0x000) && (Imm <= 0x0ff)) ||
4708            ((Imm >= 0x101) && (Imm <= 0x10f)) ||
4709            ((Imm >= 0x111) && (Imm <= 0x11f)) ||
4710            ((Imm >= 0x121) && (Imm <= 0x12f)) ||
4711            (Imm == 0x130) ||
4712            (Imm == 0x134) ||
4713            (Imm == 0x138) ||
4714            (Imm == 0x13c) ||
4715            (Imm == 0x140) ||
4716            (Imm == 0x141) ||
4717            (Imm == 0x142) ||
4718            (Imm == 0x143);
4719   }
4720   return false;
4721 }
4722 
4723 bool AMDGPUOperand::isGPRIdxMode() const {
4724   return isImm() && isUInt<4>(getImm());
4725 }
4726 
4727 bool AMDGPUOperand::isS16Imm() const {
4728   return isImm() && (isInt<16>(getImm()) || isUInt<16>(getImm()));
4729 }
4730 
4731 bool AMDGPUOperand::isU16Imm() const {
4732   return isImm() && isUInt<16>(getImm());
4733 }
4734 
4735 OperandMatchResultTy
4736 AMDGPUAsmParser::parseDPPCtrl(OperandVector &Operands) {
4737   SMLoc S = Parser.getTok().getLoc();
4738   StringRef Prefix;
4739   int64_t Int;
4740 
4741   if (getLexer().getKind() == AsmToken::Identifier) {
4742     Prefix = Parser.getTok().getString();
4743   } else {
4744     return MatchOperand_NoMatch;
4745   }
4746 
4747   if (Prefix == "row_mirror") {
4748     Int = 0x140;
4749     Parser.Lex();
4750   } else if (Prefix == "row_half_mirror") {
4751     Int = 0x141;
4752     Parser.Lex();
4753   } else {
4754     // Check to prevent parseDPPCtrlOps from eating invalid tokens
4755     if (Prefix != "quad_perm"
4756         && Prefix != "row_shl"
4757         && Prefix != "row_shr"
4758         && Prefix != "row_ror"
4759         && Prefix != "wave_shl"
4760         && Prefix != "wave_rol"
4761         && Prefix != "wave_shr"
4762         && Prefix != "wave_ror"
4763         && Prefix != "row_bcast") {
4764       return MatchOperand_NoMatch;
4765     }
4766 
4767     Parser.Lex();
4768     if (getLexer().isNot(AsmToken::Colon))
4769       return MatchOperand_ParseFail;
4770 
4771     if (Prefix == "quad_perm") {
4772       // quad_perm:[%d,%d,%d,%d]
4773       Parser.Lex();
4774       if (getLexer().isNot(AsmToken::LBrac))
4775         return MatchOperand_ParseFail;
4776       Parser.Lex();
4777 
4778       if (getParser().parseAbsoluteExpression(Int) || !(0 <= Int && Int <=3))
4779         return MatchOperand_ParseFail;
4780 
4781       for (int i = 0; i < 3; ++i) {
4782         if (getLexer().isNot(AsmToken::Comma))
4783           return MatchOperand_ParseFail;
4784         Parser.Lex();
4785 
4786         int64_t Temp;
4787         if (getParser().parseAbsoluteExpression(Temp) || !(0 <= Temp && Temp <=3))
4788           return MatchOperand_ParseFail;
4789         const int shift = i*2 + 2;
4790         Int += (Temp << shift);
4791       }
4792 
4793       if (getLexer().isNot(AsmToken::RBrac))
4794         return MatchOperand_ParseFail;
4795       Parser.Lex();
4796     } else {
4797       // sel:%d
4798       Parser.Lex();
4799       if (getParser().parseAbsoluteExpression(Int))
4800         return MatchOperand_ParseFail;
4801 
4802       if (Prefix == "row_shl" && 1 <= Int && Int <= 15) {
4803         Int |= 0x100;
4804       } else if (Prefix == "row_shr" && 1 <= Int && Int <= 15) {
4805         Int |= 0x110;
4806       } else if (Prefix == "row_ror" && 1 <= Int && Int <= 15) {
4807         Int |= 0x120;
4808       } else if (Prefix == "wave_shl" && 1 == Int) {
4809         Int = 0x130;
4810       } else if (Prefix == "wave_rol" && 1 == Int) {
4811         Int = 0x134;
4812       } else if (Prefix == "wave_shr" && 1 == Int) {
4813         Int = 0x138;
4814       } else if (Prefix == "wave_ror" && 1 == Int) {
4815         Int = 0x13C;
4816       } else if (Prefix == "row_bcast") {
4817         if (Int == 15) {
4818           Int = 0x142;
4819         } else if (Int == 31) {
4820           Int = 0x143;
4821         } else {
4822           return MatchOperand_ParseFail;
4823         }
4824       } else {
4825         return MatchOperand_ParseFail;
4826       }
4827     }
4828   }
4829 
4830   Operands.push_back(AMDGPUOperand::CreateImm(this, Int, S, AMDGPUOperand::ImmTyDppCtrl));
4831   return MatchOperand_Success;
4832 }
4833 
4834 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultRowMask() const {
4835   return AMDGPUOperand::CreateImm(this, 0xf, SMLoc(), AMDGPUOperand::ImmTyDppRowMask);
4836 }
4837 
4838 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultBankMask() const {
4839   return AMDGPUOperand::CreateImm(this, 0xf, SMLoc(), AMDGPUOperand::ImmTyDppBankMask);
4840 }
4841 
4842 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultBoundCtrl() const {
4843   return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTyDppBoundCtrl);
4844 }
4845 
4846 void AMDGPUAsmParser::cvtDPP(MCInst &Inst, const OperandVector &Operands) {
4847   OptionalImmIndexMap OptionalIdx;
4848 
4849   unsigned I = 1;
4850   const MCInstrDesc &Desc = MII.get(Inst.getOpcode());
4851   for (unsigned J = 0; J < Desc.getNumDefs(); ++J) {
4852     ((AMDGPUOperand &)*Operands[I++]).addRegOperands(Inst, 1);
4853   }
4854 
4855   // All DPP instructions with at least one source operand have a fake "old"
4856   // source at the beginning that's tied to the dst operand. Handle it here.
4857   if (Desc.getNumOperands() >= 2)
4858     Inst.addOperand(Inst.getOperand(0));
4859 
4860   for (unsigned E = Operands.size(); I != E; ++I) {
4861     AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[I]);
4862     // Add the register arguments
4863     if (Op.isReg() && Op.Reg.RegNo == AMDGPU::VCC) {
4864       // VOP2b (v_add_u32, v_sub_u32 ...) dpp use "vcc" token.
4865       // Skip it.
4866       continue;
4867     } if (isRegOrImmWithInputMods(Desc, Inst.getNumOperands())) {
4868       Op.addRegWithFPInputModsOperands(Inst, 2);
4869     } else if (Op.isDPPCtrl()) {
4870       Op.addImmOperands(Inst, 1);
4871     } else if (Op.isImm()) {
4872       // Handle optional arguments
4873       OptionalIdx[Op.getImmTy()] = I;
4874     } else {
4875       llvm_unreachable("Invalid operand type");
4876     }
4877   }
4878 
4879   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyDppRowMask, 0xf);
4880   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyDppBankMask, 0xf);
4881   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyDppBoundCtrl);
4882 }
4883 
4884 //===----------------------------------------------------------------------===//
4885 // sdwa
4886 //===----------------------------------------------------------------------===//
4887 
4888 OperandMatchResultTy
4889 AMDGPUAsmParser::parseSDWASel(OperandVector &Operands, StringRef Prefix,
4890                               AMDGPUOperand::ImmTy Type) {
4891   using namespace llvm::AMDGPU::SDWA;
4892 
4893   SMLoc S = Parser.getTok().getLoc();
4894   StringRef Value;
4895   OperandMatchResultTy res;
4896 
4897   res = parseStringWithPrefix(Prefix, Value);
4898   if (res != MatchOperand_Success) {
4899     return res;
4900   }
4901 
4902   int64_t Int;
4903   Int = StringSwitch<int64_t>(Value)
4904         .Case("BYTE_0", SdwaSel::BYTE_0)
4905         .Case("BYTE_1", SdwaSel::BYTE_1)
4906         .Case("BYTE_2", SdwaSel::BYTE_2)
4907         .Case("BYTE_3", SdwaSel::BYTE_3)
4908         .Case("WORD_0", SdwaSel::WORD_0)
4909         .Case("WORD_1", SdwaSel::WORD_1)
4910         .Case("DWORD", SdwaSel::DWORD)
4911         .Default(0xffffffff);
4912   Parser.Lex(); // eat last token
4913 
4914   if (Int == 0xffffffff) {
4915     return MatchOperand_ParseFail;
4916   }
4917 
4918   Operands.push_back(AMDGPUOperand::CreateImm(this, Int, S, Type));
4919   return MatchOperand_Success;
4920 }
4921 
4922 OperandMatchResultTy
4923 AMDGPUAsmParser::parseSDWADstUnused(OperandVector &Operands) {
4924   using namespace llvm::AMDGPU::SDWA;
4925 
4926   SMLoc S = Parser.getTok().getLoc();
4927   StringRef Value;
4928   OperandMatchResultTy res;
4929 
4930   res = parseStringWithPrefix("dst_unused", Value);
4931   if (res != MatchOperand_Success) {
4932     return res;
4933   }
4934 
4935   int64_t Int;
4936   Int = StringSwitch<int64_t>(Value)
4937         .Case("UNUSED_PAD", DstUnused::UNUSED_PAD)
4938         .Case("UNUSED_SEXT", DstUnused::UNUSED_SEXT)
4939         .Case("UNUSED_PRESERVE", DstUnused::UNUSED_PRESERVE)
4940         .Default(0xffffffff);
4941   Parser.Lex(); // eat last token
4942 
4943   if (Int == 0xffffffff) {
4944     return MatchOperand_ParseFail;
4945   }
4946 
4947   Operands.push_back(AMDGPUOperand::CreateImm(this, Int, S, AMDGPUOperand::ImmTySdwaDstUnused));
4948   return MatchOperand_Success;
4949 }
4950 
4951 void AMDGPUAsmParser::cvtSdwaVOP1(MCInst &Inst, const OperandVector &Operands) {
4952   cvtSDWA(Inst, Operands, SIInstrFlags::VOP1);
4953 }
4954 
4955 void AMDGPUAsmParser::cvtSdwaVOP2(MCInst &Inst, const OperandVector &Operands) {
4956   cvtSDWA(Inst, Operands, SIInstrFlags::VOP2);
4957 }
4958 
4959 void AMDGPUAsmParser::cvtSdwaVOP2b(MCInst &Inst, const OperandVector &Operands) {
4960   cvtSDWA(Inst, Operands, SIInstrFlags::VOP2, true);
4961 }
4962 
4963 void AMDGPUAsmParser::cvtSdwaVOPC(MCInst &Inst, const OperandVector &Operands) {
4964   cvtSDWA(Inst, Operands, SIInstrFlags::VOPC, isVI());
4965 }
4966 
4967 void AMDGPUAsmParser::cvtSDWA(MCInst &Inst, const OperandVector &Operands,
4968                               uint64_t BasicInstType, bool skipVcc) {
4969   using namespace llvm::AMDGPU::SDWA;
4970 
4971   OptionalImmIndexMap OptionalIdx;
4972   bool skippedVcc = false;
4973 
4974   unsigned I = 1;
4975   const MCInstrDesc &Desc = MII.get(Inst.getOpcode());
4976   for (unsigned J = 0; J < Desc.getNumDefs(); ++J) {
4977     ((AMDGPUOperand &)*Operands[I++]).addRegOperands(Inst, 1);
4978   }
4979 
4980   for (unsigned E = Operands.size(); I != E; ++I) {
4981     AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[I]);
4982     if (skipVcc && !skippedVcc && Op.isReg() && Op.Reg.RegNo == AMDGPU::VCC) {
4983       // VOP2b (v_add_u32, v_sub_u32 ...) sdwa use "vcc" token as dst.
4984       // Skip it if it's 2nd (e.g. v_add_i32_sdwa v1, vcc, v2, v3)
4985       // or 4th (v_addc_u32_sdwa v1, vcc, v2, v3, vcc) operand.
4986       // Skip VCC only if we didn't skip it on previous iteration.
4987       if (BasicInstType == SIInstrFlags::VOP2 &&
4988           (Inst.getNumOperands() == 1 || Inst.getNumOperands() == 5)) {
4989         skippedVcc = true;
4990         continue;
4991       } else if (BasicInstType == SIInstrFlags::VOPC &&
4992                  Inst.getNumOperands() == 0) {
4993         skippedVcc = true;
4994         continue;
4995       }
4996     }
4997     if (isRegOrImmWithInputMods(Desc, Inst.getNumOperands())) {
4998       Op.addRegOrImmWithInputModsOperands(Inst, 2);
4999     } else if (Op.isImm()) {
5000       // Handle optional arguments
5001       OptionalIdx[Op.getImmTy()] = I;
5002     } else {
5003       llvm_unreachable("Invalid operand type");
5004     }
5005     skippedVcc = false;
5006   }
5007 
5008   if (Inst.getOpcode() != AMDGPU::V_NOP_sdwa_gfx9 &&
5009       Inst.getOpcode() != AMDGPU::V_NOP_sdwa_vi) {
5010     // v_nop_sdwa_sdwa_vi/gfx9 has no optional sdwa arguments
5011     switch (BasicInstType) {
5012     case SIInstrFlags::VOP1:
5013       addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyClampSI, 0);
5014       if (AMDGPU::getNamedOperandIdx(Inst.getOpcode(), AMDGPU::OpName::omod) != -1) {
5015         addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyOModSI, 0);
5016       }
5017       addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySdwaDstSel, SdwaSel::DWORD);
5018       addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySdwaDstUnused, DstUnused::UNUSED_PRESERVE);
5019       addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySdwaSrc0Sel, SdwaSel::DWORD);
5020       break;
5021 
5022     case SIInstrFlags::VOP2:
5023       addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyClampSI, 0);
5024       if (AMDGPU::getNamedOperandIdx(Inst.getOpcode(), AMDGPU::OpName::omod) != -1) {
5025         addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyOModSI, 0);
5026       }
5027       addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySdwaDstSel, SdwaSel::DWORD);
5028       addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySdwaDstUnused, DstUnused::UNUSED_PRESERVE);
5029       addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySdwaSrc0Sel, SdwaSel::DWORD);
5030       addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySdwaSrc1Sel, SdwaSel::DWORD);
5031       break;
5032 
5033     case SIInstrFlags::VOPC:
5034       addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyClampSI, 0);
5035       addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySdwaSrc0Sel, SdwaSel::DWORD);
5036       addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySdwaSrc1Sel, SdwaSel::DWORD);
5037       break;
5038 
5039     default:
5040       llvm_unreachable("Invalid instruction type. Only VOP1, VOP2 and VOPC allowed");
5041     }
5042   }
5043 
5044   // special case v_mac_{f16, f32}:
5045   // it has src2 register operand that is tied to dst operand
5046   if (Inst.getOpcode() == AMDGPU::V_MAC_F32_sdwa_vi ||
5047       Inst.getOpcode() == AMDGPU::V_MAC_F16_sdwa_vi)  {
5048     auto it = Inst.begin();
5049     std::advance(
5050       it, AMDGPU::getNamedOperandIdx(Inst.getOpcode(), AMDGPU::OpName::src2));
5051     Inst.insert(it, Inst.getOperand(0)); // src2 = dst
5052   }
5053 }
5054 
5055 /// Force static initialization.
5056 extern "C" void LLVMInitializeAMDGPUAsmParser() {
5057   RegisterMCAsmParser<AMDGPUAsmParser> A(getTheAMDGPUTarget());
5058   RegisterMCAsmParser<AMDGPUAsmParser> B(getTheGCNTarget());
5059 }
5060 
5061 #define GET_REGISTER_MATCHER
5062 #define GET_MATCHER_IMPLEMENTATION
5063 #define GET_MNEMONIC_SPELL_CHECKER
5064 #include "AMDGPUGenAsmMatcher.inc"
5065 
5066 // This fuction should be defined after auto-generated include so that we have
5067 // MatchClassKind enum defined
5068 unsigned AMDGPUAsmParser::validateTargetOperandClass(MCParsedAsmOperand &Op,
5069                                                      unsigned Kind) {
5070   // Tokens like "glc" would be parsed as immediate operands in ParseOperand().
5071   // But MatchInstructionImpl() expects to meet token and fails to validate
5072   // operand. This method checks if we are given immediate operand but expect to
5073   // get corresponding token.
5074   AMDGPUOperand &Operand = (AMDGPUOperand&)Op;
5075   switch (Kind) {
5076   case MCK_addr64:
5077     return Operand.isAddr64() ? Match_Success : Match_InvalidOperand;
5078   case MCK_gds:
5079     return Operand.isGDS() ? Match_Success : Match_InvalidOperand;
5080   case MCK_lds:
5081     return Operand.isLDS() ? Match_Success : Match_InvalidOperand;
5082   case MCK_glc:
5083     return Operand.isGLC() ? Match_Success : Match_InvalidOperand;
5084   case MCK_d16:
5085     return Operand.isD16() ? Match_Success : Match_InvalidOperand;
5086   case MCK_idxen:
5087     return Operand.isIdxen() ? Match_Success : Match_InvalidOperand;
5088   case MCK_offen:
5089     return Operand.isOffen() ? Match_Success : Match_InvalidOperand;
5090   case MCK_SSrcB32:
5091     // When operands have expression values, they will return true for isToken,
5092     // because it is not possible to distinguish between a token and an
5093     // expression at parse time. MatchInstructionImpl() will always try to
5094     // match an operand as a token, when isToken returns true, and when the
5095     // name of the expression is not a valid token, the match will fail,
5096     // so we need to handle it here.
5097     return Operand.isSSrcB32() ? Match_Success : Match_InvalidOperand;
5098   case MCK_SSrcF32:
5099     return Operand.isSSrcF32() ? Match_Success : Match_InvalidOperand;
5100   case MCK_SoppBrTarget:
5101     return Operand.isSoppBrTarget() ? Match_Success : Match_InvalidOperand;
5102   case MCK_VReg32OrOff:
5103     return Operand.isVReg32OrOff() ? Match_Success : Match_InvalidOperand;
5104   case MCK_InterpSlot:
5105     return Operand.isInterpSlot() ? Match_Success : Match_InvalidOperand;
5106   case MCK_Attr:
5107     return Operand.isInterpAttr() ? Match_Success : Match_InvalidOperand;
5108   case MCK_AttrChan:
5109     return Operand.isAttrChan() ? Match_Success : Match_InvalidOperand;
5110   default:
5111     return Match_InvalidOperand;
5112   }
5113 }
5114