1 //===- AMDGPUAsmParser.cpp - Parse SI asm to MCInst instructions ----------===//
2 //
3 //                     The LLVM Compiler Infrastructure
4 //
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
7 //
8 //===----------------------------------------------------------------------===//
9 
10 #include "AMDGPU.h"
11 #include "AMDKernelCodeT.h"
12 #include "MCTargetDesc/AMDGPUMCTargetDesc.h"
13 #include "MCTargetDesc/AMDGPUTargetStreamer.h"
14 #include "SIDefines.h"
15 #include "Utils/AMDGPUAsmUtils.h"
16 #include "Utils/AMDGPUBaseInfo.h"
17 #include "Utils/AMDKernelCodeTUtils.h"
18 #include "llvm/ADT/APFloat.h"
19 #include "llvm/ADT/APInt.h"
20 #include "llvm/ADT/ArrayRef.h"
21 #include "llvm/ADT/STLExtras.h"
22 #include "llvm/ADT/SmallBitVector.h"
23 #include "llvm/ADT/SmallString.h"
24 #include "llvm/ADT/StringRef.h"
25 #include "llvm/ADT/StringSwitch.h"
26 #include "llvm/ADT/Twine.h"
27 #include "llvm/BinaryFormat/ELF.h"
28 #include "llvm/CodeGen/MachineValueType.h"
29 #include "llvm/MC/MCAsmInfo.h"
30 #include "llvm/MC/MCContext.h"
31 #include "llvm/MC/MCExpr.h"
32 #include "llvm/MC/MCInst.h"
33 #include "llvm/MC/MCInstrDesc.h"
34 #include "llvm/MC/MCInstrInfo.h"
35 #include "llvm/MC/MCParser/MCAsmLexer.h"
36 #include "llvm/MC/MCParser/MCAsmParser.h"
37 #include "llvm/MC/MCParser/MCAsmParserExtension.h"
38 #include "llvm/MC/MCParser/MCParsedAsmOperand.h"
39 #include "llvm/MC/MCParser/MCTargetAsmParser.h"
40 #include "llvm/MC/MCRegisterInfo.h"
41 #include "llvm/MC/MCStreamer.h"
42 #include "llvm/MC/MCSubtargetInfo.h"
43 #include "llvm/MC/MCSymbol.h"
44 #include "llvm/Support/AMDGPUCodeObjectMetadata.h"
45 #include "llvm/Support/Casting.h"
46 #include "llvm/Support/Compiler.h"
47 #include "llvm/Support/ErrorHandling.h"
48 #include "llvm/Support/MathExtras.h"
49 #include "llvm/Support/SMLoc.h"
50 #include "llvm/Support/TargetRegistry.h"
51 #include "llvm/Support/raw_ostream.h"
52 #include <algorithm>
53 #include <cassert>
54 #include <cstdint>
55 #include <cstring>
56 #include <iterator>
57 #include <map>
58 #include <memory>
59 #include <string>
60 
61 using namespace llvm;
62 using namespace llvm::AMDGPU;
63 
64 namespace {
65 
66 class AMDGPUAsmParser;
67 
68 enum RegisterKind { IS_UNKNOWN, IS_VGPR, IS_SGPR, IS_TTMP, IS_SPECIAL };
69 
70 //===----------------------------------------------------------------------===//
71 // Operand
72 //===----------------------------------------------------------------------===//
73 
74 class AMDGPUOperand : public MCParsedAsmOperand {
75   enum KindTy {
76     Token,
77     Immediate,
78     Register,
79     Expression
80   } Kind;
81 
82   SMLoc StartLoc, EndLoc;
83   const AMDGPUAsmParser *AsmParser;
84 
85 public:
86   AMDGPUOperand(KindTy Kind_, const AMDGPUAsmParser *AsmParser_)
87     : MCParsedAsmOperand(), Kind(Kind_), AsmParser(AsmParser_) {}
88 
89   using Ptr = std::unique_ptr<AMDGPUOperand>;
90 
91   struct Modifiers {
92     bool Abs = false;
93     bool Neg = false;
94     bool Sext = false;
95 
96     bool hasFPModifiers() const { return Abs || Neg; }
97     bool hasIntModifiers() const { return Sext; }
98     bool hasModifiers() const { return hasFPModifiers() || hasIntModifiers(); }
99 
100     int64_t getFPModifiersOperand() const {
101       int64_t Operand = 0;
102       Operand |= Abs ? SISrcMods::ABS : 0;
103       Operand |= Neg ? SISrcMods::NEG : 0;
104       return Operand;
105     }
106 
107     int64_t getIntModifiersOperand() const {
108       int64_t Operand = 0;
109       Operand |= Sext ? SISrcMods::SEXT : 0;
110       return Operand;
111     }
112 
113     int64_t getModifiersOperand() const {
114       assert(!(hasFPModifiers() && hasIntModifiers())
115            && "fp and int modifiers should not be used simultaneously");
116       if (hasFPModifiers()) {
117         return getFPModifiersOperand();
118       } else if (hasIntModifiers()) {
119         return getIntModifiersOperand();
120       } else {
121         return 0;
122       }
123     }
124 
125     friend raw_ostream &operator <<(raw_ostream &OS, AMDGPUOperand::Modifiers Mods);
126   };
127 
128   enum ImmTy {
129     ImmTyNone,
130     ImmTyGDS,
131     ImmTyOffen,
132     ImmTyIdxen,
133     ImmTyAddr64,
134     ImmTyOffset,
135     ImmTyOffset0,
136     ImmTyOffset1,
137     ImmTyGLC,
138     ImmTySLC,
139     ImmTyTFE,
140     ImmTyClampSI,
141     ImmTyOModSI,
142     ImmTyDppCtrl,
143     ImmTyDppRowMask,
144     ImmTyDppBankMask,
145     ImmTyDppBoundCtrl,
146     ImmTySdwaDstSel,
147     ImmTySdwaSrc0Sel,
148     ImmTySdwaSrc1Sel,
149     ImmTySdwaDstUnused,
150     ImmTyDMask,
151     ImmTyUNorm,
152     ImmTyDA,
153     ImmTyR128,
154     ImmTyLWE,
155     ImmTyExpTgt,
156     ImmTyExpCompr,
157     ImmTyExpVM,
158     ImmTyDFMT,
159     ImmTyNFMT,
160     ImmTyHwreg,
161     ImmTyOff,
162     ImmTySendMsg,
163     ImmTyInterpSlot,
164     ImmTyInterpAttr,
165     ImmTyAttrChan,
166     ImmTyOpSel,
167     ImmTyOpSelHi,
168     ImmTyNegLo,
169     ImmTyNegHi,
170     ImmTySwizzle,
171     ImmTyHigh
172   };
173 
174   struct TokOp {
175     const char *Data;
176     unsigned Length;
177   };
178 
179   struct ImmOp {
180     int64_t Val;
181     ImmTy Type;
182     bool IsFPImm;
183     Modifiers Mods;
184   };
185 
186   struct RegOp {
187     unsigned RegNo;
188     bool IsForcedVOP3;
189     Modifiers Mods;
190   };
191 
192   union {
193     TokOp Tok;
194     ImmOp Imm;
195     RegOp Reg;
196     const MCExpr *Expr;
197   };
198 
199   bool isToken() const override {
200     if (Kind == Token)
201       return true;
202 
203     if (Kind != Expression || !Expr)
204       return false;
205 
206     // When parsing operands, we can't always tell if something was meant to be
207     // a token, like 'gds', or an expression that references a global variable.
208     // In this case, we assume the string is an expression, and if we need to
209     // interpret is a token, then we treat the symbol name as the token.
210     return isa<MCSymbolRefExpr>(Expr);
211   }
212 
213   bool isImm() const override {
214     return Kind == Immediate;
215   }
216 
217   bool isInlinableImm(MVT type) const;
218   bool isLiteralImm(MVT type) const;
219 
220   bool isRegKind() const {
221     return Kind == Register;
222   }
223 
224   bool isReg() const override {
225     return isRegKind() && !hasModifiers();
226   }
227 
228   bool isRegOrImmWithInputMods(MVT type) const {
229     return isRegKind() || isInlinableImm(type);
230   }
231 
232   bool isRegOrImmWithInt16InputMods() const {
233     return isRegOrImmWithInputMods(MVT::i16);
234   }
235 
236   bool isRegOrImmWithInt32InputMods() const {
237     return isRegOrImmWithInputMods(MVT::i32);
238   }
239 
240   bool isRegOrImmWithInt64InputMods() const {
241     return isRegOrImmWithInputMods(MVT::i64);
242   }
243 
244   bool isRegOrImmWithFP16InputMods() const {
245     return isRegOrImmWithInputMods(MVT::f16);
246   }
247 
248   bool isRegOrImmWithFP32InputMods() const {
249     return isRegOrImmWithInputMods(MVT::f32);
250   }
251 
252   bool isRegOrImmWithFP64InputMods() const {
253     return isRegOrImmWithInputMods(MVT::f64);
254   }
255 
256   bool isVReg() const {
257     return isRegClass(AMDGPU::VGPR_32RegClassID) ||
258            isRegClass(AMDGPU::VReg_64RegClassID) ||
259            isRegClass(AMDGPU::VReg_96RegClassID) ||
260            isRegClass(AMDGPU::VReg_128RegClassID) ||
261            isRegClass(AMDGPU::VReg_256RegClassID) ||
262            isRegClass(AMDGPU::VReg_512RegClassID);
263   }
264 
265   bool isVReg32OrOff() const {
266     return isOff() || isRegClass(AMDGPU::VGPR_32RegClassID);
267   }
268 
269   bool isSDWARegKind() const;
270 
271   bool isImmTy(ImmTy ImmT) const {
272     return isImm() && Imm.Type == ImmT;
273   }
274 
275   bool isImmModifier() const {
276     return isImm() && Imm.Type != ImmTyNone;
277   }
278 
279   bool isClampSI() const { return isImmTy(ImmTyClampSI); }
280   bool isOModSI() const { return isImmTy(ImmTyOModSI); }
281   bool isDMask() const { return isImmTy(ImmTyDMask); }
282   bool isUNorm() const { return isImmTy(ImmTyUNorm); }
283   bool isDA() const { return isImmTy(ImmTyDA); }
284   bool isR128() const { return isImmTy(ImmTyUNorm); }
285   bool isLWE() const { return isImmTy(ImmTyLWE); }
286   bool isOff() const { return isImmTy(ImmTyOff); }
287   bool isExpTgt() const { return isImmTy(ImmTyExpTgt); }
288   bool isExpVM() const { return isImmTy(ImmTyExpVM); }
289   bool isExpCompr() const { return isImmTy(ImmTyExpCompr); }
290   bool isOffen() const { return isImmTy(ImmTyOffen); }
291   bool isIdxen() const { return isImmTy(ImmTyIdxen); }
292   bool isAddr64() const { return isImmTy(ImmTyAddr64); }
293   bool isOffset() const { return isImmTy(ImmTyOffset) && isUInt<16>(getImm()); }
294   bool isOffset0() const { return isImmTy(ImmTyOffset0) && isUInt<16>(getImm()); }
295   bool isOffset1() const { return isImmTy(ImmTyOffset1) && isUInt<8>(getImm()); }
296 
297   bool isOffsetU12() const { return isImmTy(ImmTyOffset) && isUInt<12>(getImm()); }
298   bool isOffsetS13() const { return isImmTy(ImmTyOffset) && isInt<13>(getImm()); }
299   bool isGDS() const { return isImmTy(ImmTyGDS); }
300   bool isGLC() const { return isImmTy(ImmTyGLC); }
301   bool isSLC() const { return isImmTy(ImmTySLC); }
302   bool isTFE() const { return isImmTy(ImmTyTFE); }
303   bool isDFMT() const { return isImmTy(ImmTyDFMT) && isUInt<8>(getImm()); }
304   bool isNFMT() const { return isImmTy(ImmTyNFMT) && isUInt<8>(getImm()); }
305   bool isBankMask() const { return isImmTy(ImmTyDppBankMask); }
306   bool isRowMask() const { return isImmTy(ImmTyDppRowMask); }
307   bool isBoundCtrl() const { return isImmTy(ImmTyDppBoundCtrl); }
308   bool isSDWADstSel() const { return isImmTy(ImmTySdwaDstSel); }
309   bool isSDWASrc0Sel() const { return isImmTy(ImmTySdwaSrc0Sel); }
310   bool isSDWASrc1Sel() const { return isImmTy(ImmTySdwaSrc1Sel); }
311   bool isSDWADstUnused() const { return isImmTy(ImmTySdwaDstUnused); }
312   bool isInterpSlot() const { return isImmTy(ImmTyInterpSlot); }
313   bool isInterpAttr() const { return isImmTy(ImmTyInterpAttr); }
314   bool isAttrChan() const { return isImmTy(ImmTyAttrChan); }
315   bool isOpSel() const { return isImmTy(ImmTyOpSel); }
316   bool isOpSelHi() const { return isImmTy(ImmTyOpSelHi); }
317   bool isNegLo() const { return isImmTy(ImmTyNegLo); }
318   bool isNegHi() const { return isImmTy(ImmTyNegHi); }
319   bool isHigh() const { return isImmTy(ImmTyHigh); }
320 
321   bool isMod() const {
322     return isClampSI() || isOModSI();
323   }
324 
325   bool isRegOrImm() const {
326     return isReg() || isImm();
327   }
328 
329   bool isRegClass(unsigned RCID) const;
330 
331   bool isRegOrInlineNoMods(unsigned RCID, MVT type) const {
332     return (isRegClass(RCID) || isInlinableImm(type)) && !hasModifiers();
333   }
334 
335   bool isSCSrcB16() const {
336     return isRegOrInlineNoMods(AMDGPU::SReg_32RegClassID, MVT::i16);
337   }
338 
339   bool isSCSrcV2B16() const {
340     return isSCSrcB16();
341   }
342 
343   bool isSCSrcB32() const {
344     return isRegOrInlineNoMods(AMDGPU::SReg_32RegClassID, MVT::i32);
345   }
346 
347   bool isSCSrcB64() const {
348     return isRegOrInlineNoMods(AMDGPU::SReg_64RegClassID, MVT::i64);
349   }
350 
351   bool isSCSrcF16() const {
352     return isRegOrInlineNoMods(AMDGPU::SReg_32RegClassID, MVT::f16);
353   }
354 
355   bool isSCSrcV2F16() const {
356     return isSCSrcF16();
357   }
358 
359   bool isSCSrcF32() const {
360     return isRegOrInlineNoMods(AMDGPU::SReg_32RegClassID, MVT::f32);
361   }
362 
363   bool isSCSrcF64() const {
364     return isRegOrInlineNoMods(AMDGPU::SReg_64RegClassID, MVT::f64);
365   }
366 
367   bool isSSrcB32() const {
368     return isSCSrcB32() || isLiteralImm(MVT::i32) || isExpr();
369   }
370 
371   bool isSSrcB16() const {
372     return isSCSrcB16() || isLiteralImm(MVT::i16);
373   }
374 
375   bool isSSrcV2B16() const {
376     llvm_unreachable("cannot happen");
377     return isSSrcB16();
378   }
379 
380   bool isSSrcB64() const {
381     // TODO: Find out how SALU supports extension of 32-bit literals to 64 bits.
382     // See isVSrc64().
383     return isSCSrcB64() || isLiteralImm(MVT::i64);
384   }
385 
386   bool isSSrcF32() const {
387     return isSCSrcB32() || isLiteralImm(MVT::f32) || isExpr();
388   }
389 
390   bool isSSrcF64() const {
391     return isSCSrcB64() || isLiteralImm(MVT::f64);
392   }
393 
394   bool isSSrcF16() const {
395     return isSCSrcB16() || isLiteralImm(MVT::f16);
396   }
397 
398   bool isSSrcV2F16() const {
399     llvm_unreachable("cannot happen");
400     return isSSrcF16();
401   }
402 
403   bool isVCSrcB32() const {
404     return isRegOrInlineNoMods(AMDGPU::VS_32RegClassID, MVT::i32);
405   }
406 
407   bool isVCSrcB64() const {
408     return isRegOrInlineNoMods(AMDGPU::VS_64RegClassID, MVT::i64);
409   }
410 
411   bool isVCSrcB16() const {
412     return isRegOrInlineNoMods(AMDGPU::VS_32RegClassID, MVT::i16);
413   }
414 
415   bool isVCSrcV2B16() const {
416     return isVCSrcB16();
417   }
418 
419   bool isVCSrcF32() const {
420     return isRegOrInlineNoMods(AMDGPU::VS_32RegClassID, MVT::f32);
421   }
422 
423   bool isVCSrcF64() const {
424     return isRegOrInlineNoMods(AMDGPU::VS_64RegClassID, MVT::f64);
425   }
426 
427   bool isVCSrcF16() const {
428     return isRegOrInlineNoMods(AMDGPU::VS_32RegClassID, MVT::f16);
429   }
430 
431   bool isVCSrcV2F16() const {
432     return isVCSrcF16();
433   }
434 
435   bool isVSrcB32() const {
436     return isVCSrcF32() || isLiteralImm(MVT::i32);
437   }
438 
439   bool isVSrcB64() const {
440     return isVCSrcF64() || isLiteralImm(MVT::i64);
441   }
442 
443   bool isVSrcB16() const {
444     return isVCSrcF16() || isLiteralImm(MVT::i16);
445   }
446 
447   bool isVSrcV2B16() const {
448     llvm_unreachable("cannot happen");
449     return isVSrcB16();
450   }
451 
452   bool isVSrcF32() const {
453     return isVCSrcF32() || isLiteralImm(MVT::f32);
454   }
455 
456   bool isVSrcF64() const {
457     return isVCSrcF64() || isLiteralImm(MVT::f64);
458   }
459 
460   bool isVSrcF16() const {
461     return isVCSrcF16() || isLiteralImm(MVT::f16);
462   }
463 
464   bool isVSrcV2F16() const {
465     llvm_unreachable("cannot happen");
466     return isVSrcF16();
467   }
468 
469   bool isKImmFP32() const {
470     return isLiteralImm(MVT::f32);
471   }
472 
473   bool isKImmFP16() const {
474     return isLiteralImm(MVT::f16);
475   }
476 
477   bool isMem() const override {
478     return false;
479   }
480 
481   bool isExpr() const {
482     return Kind == Expression;
483   }
484 
485   bool isSoppBrTarget() const {
486     return isExpr() || isImm();
487   }
488 
489   bool isSWaitCnt() const;
490   bool isHwreg() const;
491   bool isSendMsg() const;
492   bool isSwizzle() const;
493   bool isSMRDOffset8() const;
494   bool isSMRDOffset20() const;
495   bool isSMRDLiteralOffset() const;
496   bool isDPPCtrl() const;
497   bool isGPRIdxMode() const;
498   bool isS16Imm() const;
499   bool isU16Imm() const;
500 
501   StringRef getExpressionAsToken() const {
502     assert(isExpr());
503     const MCSymbolRefExpr *S = cast<MCSymbolRefExpr>(Expr);
504     return S->getSymbol().getName();
505   }
506 
507   StringRef getToken() const {
508     assert(isToken());
509 
510     if (Kind == Expression)
511       return getExpressionAsToken();
512 
513     return StringRef(Tok.Data, Tok.Length);
514   }
515 
516   int64_t getImm() const {
517     assert(isImm());
518     return Imm.Val;
519   }
520 
521   ImmTy getImmTy() const {
522     assert(isImm());
523     return Imm.Type;
524   }
525 
526   unsigned getReg() const override {
527     return Reg.RegNo;
528   }
529 
530   SMLoc getStartLoc() const override {
531     return StartLoc;
532   }
533 
534   SMLoc getEndLoc() const override {
535     return EndLoc;
536   }
537 
538   Modifiers getModifiers() const {
539     assert(isRegKind() || isImmTy(ImmTyNone));
540     return isRegKind() ? Reg.Mods : Imm.Mods;
541   }
542 
543   void setModifiers(Modifiers Mods) {
544     assert(isRegKind() || isImmTy(ImmTyNone));
545     if (isRegKind())
546       Reg.Mods = Mods;
547     else
548       Imm.Mods = Mods;
549   }
550 
551   bool hasModifiers() const {
552     return getModifiers().hasModifiers();
553   }
554 
555   bool hasFPModifiers() const {
556     return getModifiers().hasFPModifiers();
557   }
558 
559   bool hasIntModifiers() const {
560     return getModifiers().hasIntModifiers();
561   }
562 
563   uint64_t applyInputFPModifiers(uint64_t Val, unsigned Size) const;
564 
565   void addImmOperands(MCInst &Inst, unsigned N, bool ApplyModifiers = true) const;
566 
567   void addLiteralImmOperand(MCInst &Inst, int64_t Val, bool ApplyModifiers) const;
568 
569   template <unsigned Bitwidth>
570   void addKImmFPOperands(MCInst &Inst, unsigned N) const;
571 
572   void addKImmFP16Operands(MCInst &Inst, unsigned N) const {
573     addKImmFPOperands<16>(Inst, N);
574   }
575 
576   void addKImmFP32Operands(MCInst &Inst, unsigned N) const {
577     addKImmFPOperands<32>(Inst, N);
578   }
579 
580   void addRegOperands(MCInst &Inst, unsigned N) const;
581 
582   void addRegOrImmOperands(MCInst &Inst, unsigned N) const {
583     if (isRegKind())
584       addRegOperands(Inst, N);
585     else if (isExpr())
586       Inst.addOperand(MCOperand::createExpr(Expr));
587     else
588       addImmOperands(Inst, N);
589   }
590 
591   void addRegOrImmWithInputModsOperands(MCInst &Inst, unsigned N) const {
592     Modifiers Mods = getModifiers();
593     Inst.addOperand(MCOperand::createImm(Mods.getModifiersOperand()));
594     if (isRegKind()) {
595       addRegOperands(Inst, N);
596     } else {
597       addImmOperands(Inst, N, false);
598     }
599   }
600 
601   void addRegOrImmWithFPInputModsOperands(MCInst &Inst, unsigned N) const {
602     assert(!hasIntModifiers());
603     addRegOrImmWithInputModsOperands(Inst, N);
604   }
605 
606   void addRegOrImmWithIntInputModsOperands(MCInst &Inst, unsigned N) const {
607     assert(!hasFPModifiers());
608     addRegOrImmWithInputModsOperands(Inst, N);
609   }
610 
611   void addRegWithInputModsOperands(MCInst &Inst, unsigned N) const {
612     Modifiers Mods = getModifiers();
613     Inst.addOperand(MCOperand::createImm(Mods.getModifiersOperand()));
614     assert(isRegKind());
615     addRegOperands(Inst, N);
616   }
617 
618   void addRegWithFPInputModsOperands(MCInst &Inst, unsigned N) const {
619     assert(!hasIntModifiers());
620     addRegWithInputModsOperands(Inst, N);
621   }
622 
623   void addRegWithIntInputModsOperands(MCInst &Inst, unsigned N) const {
624     assert(!hasFPModifiers());
625     addRegWithInputModsOperands(Inst, N);
626   }
627 
628   void addSoppBrTargetOperands(MCInst &Inst, unsigned N) const {
629     if (isImm())
630       addImmOperands(Inst, N);
631     else {
632       assert(isExpr());
633       Inst.addOperand(MCOperand::createExpr(Expr));
634     }
635   }
636 
637   static void printImmTy(raw_ostream& OS, ImmTy Type) {
638     switch (Type) {
639     case ImmTyNone: OS << "None"; break;
640     case ImmTyGDS: OS << "GDS"; break;
641     case ImmTyOffen: OS << "Offen"; break;
642     case ImmTyIdxen: OS << "Idxen"; break;
643     case ImmTyAddr64: OS << "Addr64"; break;
644     case ImmTyOffset: OS << "Offset"; break;
645     case ImmTyOffset0: OS << "Offset0"; break;
646     case ImmTyOffset1: OS << "Offset1"; break;
647     case ImmTyGLC: OS << "GLC"; break;
648     case ImmTySLC: OS << "SLC"; break;
649     case ImmTyTFE: OS << "TFE"; break;
650     case ImmTyDFMT: OS << "DFMT"; break;
651     case ImmTyNFMT: OS << "NFMT"; break;
652     case ImmTyClampSI: OS << "ClampSI"; break;
653     case ImmTyOModSI: OS << "OModSI"; break;
654     case ImmTyDppCtrl: OS << "DppCtrl"; break;
655     case ImmTyDppRowMask: OS << "DppRowMask"; break;
656     case ImmTyDppBankMask: OS << "DppBankMask"; break;
657     case ImmTyDppBoundCtrl: OS << "DppBoundCtrl"; break;
658     case ImmTySdwaDstSel: OS << "SdwaDstSel"; break;
659     case ImmTySdwaSrc0Sel: OS << "SdwaSrc0Sel"; break;
660     case ImmTySdwaSrc1Sel: OS << "SdwaSrc1Sel"; break;
661     case ImmTySdwaDstUnused: OS << "SdwaDstUnused"; break;
662     case ImmTyDMask: OS << "DMask"; break;
663     case ImmTyUNorm: OS << "UNorm"; break;
664     case ImmTyDA: OS << "DA"; break;
665     case ImmTyR128: OS << "R128"; break;
666     case ImmTyLWE: OS << "LWE"; break;
667     case ImmTyOff: OS << "Off"; break;
668     case ImmTyExpTgt: OS << "ExpTgt"; break;
669     case ImmTyExpCompr: OS << "ExpCompr"; break;
670     case ImmTyExpVM: OS << "ExpVM"; break;
671     case ImmTyHwreg: OS << "Hwreg"; break;
672     case ImmTySendMsg: OS << "SendMsg"; break;
673     case ImmTyInterpSlot: OS << "InterpSlot"; break;
674     case ImmTyInterpAttr: OS << "InterpAttr"; break;
675     case ImmTyAttrChan: OS << "AttrChan"; break;
676     case ImmTyOpSel: OS << "OpSel"; break;
677     case ImmTyOpSelHi: OS << "OpSelHi"; break;
678     case ImmTyNegLo: OS << "NegLo"; break;
679     case ImmTyNegHi: OS << "NegHi"; break;
680     case ImmTySwizzle: OS << "Swizzle"; break;
681     case ImmTyHigh: OS << "High"; break;
682     }
683   }
684 
685   void print(raw_ostream &OS) const override {
686     switch (Kind) {
687     case Register:
688       OS << "<register " << getReg() << " mods: " << Reg.Mods << '>';
689       break;
690     case Immediate:
691       OS << '<' << getImm();
692       if (getImmTy() != ImmTyNone) {
693         OS << " type: "; printImmTy(OS, getImmTy());
694       }
695       OS << " mods: " << Imm.Mods << '>';
696       break;
697     case Token:
698       OS << '\'' << getToken() << '\'';
699       break;
700     case Expression:
701       OS << "<expr " << *Expr << '>';
702       break;
703     }
704   }
705 
706   static AMDGPUOperand::Ptr CreateImm(const AMDGPUAsmParser *AsmParser,
707                                       int64_t Val, SMLoc Loc,
708                                       ImmTy Type = ImmTyNone,
709                                       bool IsFPImm = false) {
710     auto Op = llvm::make_unique<AMDGPUOperand>(Immediate, AsmParser);
711     Op->Imm.Val = Val;
712     Op->Imm.IsFPImm = IsFPImm;
713     Op->Imm.Type = Type;
714     Op->Imm.Mods = Modifiers();
715     Op->StartLoc = Loc;
716     Op->EndLoc = Loc;
717     return Op;
718   }
719 
720   static AMDGPUOperand::Ptr CreateToken(const AMDGPUAsmParser *AsmParser,
721                                         StringRef Str, SMLoc Loc,
722                                         bool HasExplicitEncodingSize = true) {
723     auto Res = llvm::make_unique<AMDGPUOperand>(Token, AsmParser);
724     Res->Tok.Data = Str.data();
725     Res->Tok.Length = Str.size();
726     Res->StartLoc = Loc;
727     Res->EndLoc = Loc;
728     return Res;
729   }
730 
731   static AMDGPUOperand::Ptr CreateReg(const AMDGPUAsmParser *AsmParser,
732                                       unsigned RegNo, SMLoc S,
733                                       SMLoc E,
734                                       bool ForceVOP3) {
735     auto Op = llvm::make_unique<AMDGPUOperand>(Register, AsmParser);
736     Op->Reg.RegNo = RegNo;
737     Op->Reg.Mods = Modifiers();
738     Op->Reg.IsForcedVOP3 = ForceVOP3;
739     Op->StartLoc = S;
740     Op->EndLoc = E;
741     return Op;
742   }
743 
744   static AMDGPUOperand::Ptr CreateExpr(const AMDGPUAsmParser *AsmParser,
745                                        const class MCExpr *Expr, SMLoc S) {
746     auto Op = llvm::make_unique<AMDGPUOperand>(Expression, AsmParser);
747     Op->Expr = Expr;
748     Op->StartLoc = S;
749     Op->EndLoc = S;
750     return Op;
751   }
752 };
753 
754 raw_ostream &operator <<(raw_ostream &OS, AMDGPUOperand::Modifiers Mods) {
755   OS << "abs:" << Mods.Abs << " neg: " << Mods.Neg << " sext:" << Mods.Sext;
756   return OS;
757 }
758 
759 //===----------------------------------------------------------------------===//
760 // AsmParser
761 //===----------------------------------------------------------------------===//
762 
763 // Holds info related to the current kernel, e.g. count of SGPRs used.
764 // Kernel scope begins at .amdgpu_hsa_kernel directive, ends at next
765 // .amdgpu_hsa_kernel or at EOF.
766 class KernelScopeInfo {
767   int SgprIndexUnusedMin = -1;
768   int VgprIndexUnusedMin = -1;
769   MCContext *Ctx = nullptr;
770 
771   void usesSgprAt(int i) {
772     if (i >= SgprIndexUnusedMin) {
773       SgprIndexUnusedMin = ++i;
774       if (Ctx) {
775         MCSymbol * const Sym = Ctx->getOrCreateSymbol(Twine(".kernel.sgpr_count"));
776         Sym->setVariableValue(MCConstantExpr::create(SgprIndexUnusedMin, *Ctx));
777       }
778     }
779   }
780 
781   void usesVgprAt(int i) {
782     if (i >= VgprIndexUnusedMin) {
783       VgprIndexUnusedMin = ++i;
784       if (Ctx) {
785         MCSymbol * const Sym = Ctx->getOrCreateSymbol(Twine(".kernel.vgpr_count"));
786         Sym->setVariableValue(MCConstantExpr::create(VgprIndexUnusedMin, *Ctx));
787       }
788     }
789   }
790 
791 public:
792   KernelScopeInfo() = default;
793 
794   void initialize(MCContext &Context) {
795     Ctx = &Context;
796     usesSgprAt(SgprIndexUnusedMin = -1);
797     usesVgprAt(VgprIndexUnusedMin = -1);
798   }
799 
800   void usesRegister(RegisterKind RegKind, unsigned DwordRegIndex, unsigned RegWidth) {
801     switch (RegKind) {
802       case IS_SGPR: usesSgprAt(DwordRegIndex + RegWidth - 1); break;
803       case IS_VGPR: usesVgprAt(DwordRegIndex + RegWidth - 1); break;
804       default: break;
805     }
806   }
807 };
808 
809 class AMDGPUAsmParser : public MCTargetAsmParser {
810   const MCInstrInfo &MII;
811   MCAsmParser &Parser;
812 
813   unsigned ForcedEncodingSize = 0;
814   bool ForcedDPP = false;
815   bool ForcedSDWA = false;
816   KernelScopeInfo KernelScope;
817 
818   /// @name Auto-generated Match Functions
819   /// {
820 
821 #define GET_ASSEMBLER_HEADER
822 #include "AMDGPUGenAsmMatcher.inc"
823 
824   /// }
825 
826 private:
827   bool ParseAsAbsoluteExpression(uint32_t &Ret);
828   bool ParseDirectiveMajorMinor(uint32_t &Major, uint32_t &Minor);
829   bool ParseDirectiveHSACodeObjectVersion();
830   bool ParseDirectiveHSACodeObjectISA();
831   bool ParseDirectiveCodeObjectMetadata();
832   bool ParseAMDKernelCodeTValue(StringRef ID, amd_kernel_code_t &Header);
833   bool ParseDirectiveAMDKernelCodeT();
834   bool subtargetHasRegister(const MCRegisterInfo &MRI, unsigned RegNo) const;
835   bool ParseDirectiveAMDGPUHsaKernel();
836   bool AddNextRegisterToList(unsigned& Reg, unsigned& RegWidth,
837                              RegisterKind RegKind, unsigned Reg1,
838                              unsigned RegNum);
839   bool ParseAMDGPURegister(RegisterKind& RegKind, unsigned& Reg,
840                            unsigned& RegNum, unsigned& RegWidth,
841                            unsigned *DwordRegIndex);
842   void cvtMubufImpl(MCInst &Inst, const OperandVector &Operands,
843                     bool IsAtomic, bool IsAtomicReturn);
844   void cvtDSImpl(MCInst &Inst, const OperandVector &Operands,
845                  bool IsGdsHardcoded);
846 
847 public:
848   enum AMDGPUMatchResultTy {
849     Match_PreferE32 = FIRST_TARGET_MATCH_RESULT_TY
850   };
851 
852   using OptionalImmIndexMap = std::map<AMDGPUOperand::ImmTy, unsigned>;
853 
854   AMDGPUAsmParser(const MCSubtargetInfo &STI, MCAsmParser &_Parser,
855                const MCInstrInfo &MII,
856                const MCTargetOptions &Options)
857       : MCTargetAsmParser(Options, STI), MII(MII), Parser(_Parser) {
858     MCAsmParserExtension::Initialize(Parser);
859 
860     if (getFeatureBits().none()) {
861       // Set default features.
862       copySTI().ToggleFeature("SOUTHERN_ISLANDS");
863     }
864 
865     setAvailableFeatures(ComputeAvailableFeatures(getFeatureBits()));
866 
867     {
868       // TODO: make those pre-defined variables read-only.
869       // Currently there is none suitable machinery in the core llvm-mc for this.
870       // MCSymbol::isRedefinable is intended for another purpose, and
871       // AsmParser::parseDirectiveSet() cannot be specialized for specific target.
872       AMDGPU::IsaInfo::IsaVersion ISA =
873           AMDGPU::IsaInfo::getIsaVersion(getFeatureBits());
874       MCContext &Ctx = getContext();
875       MCSymbol *Sym =
876           Ctx.getOrCreateSymbol(Twine(".option.machine_version_major"));
877       Sym->setVariableValue(MCConstantExpr::create(ISA.Major, Ctx));
878       Sym = Ctx.getOrCreateSymbol(Twine(".option.machine_version_minor"));
879       Sym->setVariableValue(MCConstantExpr::create(ISA.Minor, Ctx));
880       Sym = Ctx.getOrCreateSymbol(Twine(".option.machine_version_stepping"));
881       Sym->setVariableValue(MCConstantExpr::create(ISA.Stepping, Ctx));
882     }
883     KernelScope.initialize(getContext());
884   }
885 
886   bool isSI() const {
887     return AMDGPU::isSI(getSTI());
888   }
889 
890   bool isCI() const {
891     return AMDGPU::isCI(getSTI());
892   }
893 
894   bool isVI() const {
895     return AMDGPU::isVI(getSTI());
896   }
897 
898   bool isGFX9() const {
899     return AMDGPU::isGFX9(getSTI());
900   }
901 
902   bool hasInv2PiInlineImm() const {
903     return getFeatureBits()[AMDGPU::FeatureInv2PiInlineImm];
904   }
905 
906   bool hasFlatOffsets() const {
907     return getFeatureBits()[AMDGPU::FeatureFlatInstOffsets];
908   }
909 
910   bool hasSGPR102_SGPR103() const {
911     return !isVI();
912   }
913 
914   AMDGPUTargetStreamer &getTargetStreamer() {
915     MCTargetStreamer &TS = *getParser().getStreamer().getTargetStreamer();
916     return static_cast<AMDGPUTargetStreamer &>(TS);
917   }
918 
919   const MCRegisterInfo *getMRI() const {
920     // We need this const_cast because for some reason getContext() is not const
921     // in MCAsmParser.
922     return const_cast<AMDGPUAsmParser*>(this)->getContext().getRegisterInfo();
923   }
924 
925   const MCInstrInfo *getMII() const {
926     return &MII;
927   }
928 
929   const FeatureBitset &getFeatureBits() const {
930     return getSTI().getFeatureBits();
931   }
932 
933   void setForcedEncodingSize(unsigned Size) { ForcedEncodingSize = Size; }
934   void setForcedDPP(bool ForceDPP_) { ForcedDPP = ForceDPP_; }
935   void setForcedSDWA(bool ForceSDWA_) { ForcedSDWA = ForceSDWA_; }
936 
937   unsigned getForcedEncodingSize() const { return ForcedEncodingSize; }
938   bool isForcedVOP3() const { return ForcedEncodingSize == 64; }
939   bool isForcedDPP() const { return ForcedDPP; }
940   bool isForcedSDWA() const { return ForcedSDWA; }
941   ArrayRef<unsigned> getMatchedVariants() const;
942 
943   std::unique_ptr<AMDGPUOperand> parseRegister();
944   bool ParseRegister(unsigned &RegNo, SMLoc &StartLoc, SMLoc &EndLoc) override;
945   unsigned checkTargetMatchPredicate(MCInst &Inst) override;
946   unsigned validateTargetOperandClass(MCParsedAsmOperand &Op,
947                                       unsigned Kind) override;
948   bool MatchAndEmitInstruction(SMLoc IDLoc, unsigned &Opcode,
949                                OperandVector &Operands, MCStreamer &Out,
950                                uint64_t &ErrorInfo,
951                                bool MatchingInlineAsm) override;
952   bool ParseDirective(AsmToken DirectiveID) override;
953   OperandMatchResultTy parseOperand(OperandVector &Operands, StringRef Mnemonic);
954   StringRef parseMnemonicSuffix(StringRef Name);
955   bool ParseInstruction(ParseInstructionInfo &Info, StringRef Name,
956                         SMLoc NameLoc, OperandVector &Operands) override;
957   //bool ProcessInstruction(MCInst &Inst);
958 
959   OperandMatchResultTy parseIntWithPrefix(const char *Prefix, int64_t &Int);
960 
961   OperandMatchResultTy
962   parseIntWithPrefix(const char *Prefix, OperandVector &Operands,
963                      AMDGPUOperand::ImmTy ImmTy = AMDGPUOperand::ImmTyNone,
964                      bool (*ConvertResult)(int64_t &) = nullptr);
965 
966   OperandMatchResultTy parseOperandArrayWithPrefix(
967     const char *Prefix,
968     OperandVector &Operands,
969     AMDGPUOperand::ImmTy ImmTy = AMDGPUOperand::ImmTyNone,
970     bool (*ConvertResult)(int64_t&) = nullptr);
971 
972   OperandMatchResultTy
973   parseNamedBit(const char *Name, OperandVector &Operands,
974                 AMDGPUOperand::ImmTy ImmTy = AMDGPUOperand::ImmTyNone);
975   OperandMatchResultTy parseStringWithPrefix(StringRef Prefix,
976                                              StringRef &Value);
977 
978   bool parseAbsoluteExpr(int64_t &Val, bool AbsMod = false);
979   OperandMatchResultTy parseImm(OperandVector &Operands, bool AbsMod = false);
980   OperandMatchResultTy parseReg(OperandVector &Operands);
981   OperandMatchResultTy parseRegOrImm(OperandVector &Operands, bool AbsMod = false);
982   OperandMatchResultTy parseRegOrImmWithFPInputMods(OperandVector &Operands, bool AllowImm = true);
983   OperandMatchResultTy parseRegOrImmWithIntInputMods(OperandVector &Operands, bool AllowImm = true);
984   OperandMatchResultTy parseRegWithFPInputMods(OperandVector &Operands);
985   OperandMatchResultTy parseRegWithIntInputMods(OperandVector &Operands);
986   OperandMatchResultTy parseVReg32OrOff(OperandVector &Operands);
987 
988   void cvtDSOffset01(MCInst &Inst, const OperandVector &Operands);
989   void cvtDS(MCInst &Inst, const OperandVector &Operands) { cvtDSImpl(Inst, Operands, false); }
990   void cvtDSGds(MCInst &Inst, const OperandVector &Operands) { cvtDSImpl(Inst, Operands, true); }
991   void cvtExp(MCInst &Inst, const OperandVector &Operands);
992 
993   bool parseCnt(int64_t &IntVal);
994   OperandMatchResultTy parseSWaitCntOps(OperandVector &Operands);
995   OperandMatchResultTy parseHwreg(OperandVector &Operands);
996 
997 private:
998   struct OperandInfoTy {
999     int64_t Id;
1000     bool IsSymbolic = false;
1001 
1002     OperandInfoTy(int64_t Id_) : Id(Id_) {}
1003   };
1004 
1005   bool parseSendMsgConstruct(OperandInfoTy &Msg, OperandInfoTy &Operation, int64_t &StreamId);
1006   bool parseHwregConstruct(OperandInfoTy &HwReg, int64_t &Offset, int64_t &Width);
1007 
1008   void errorExpTgt();
1009   OperandMatchResultTy parseExpTgtImpl(StringRef Str, uint8_t &Val);
1010 
1011   bool validateInstruction(const MCInst &Inst, const SMLoc &IDLoc);
1012   bool validateConstantBusLimitations(const MCInst &Inst);
1013   bool validateEarlyClobberLimitations(const MCInst &Inst);
1014   bool usesConstantBus(const MCInst &Inst, unsigned OpIdx);
1015   bool isInlineConstant(const MCInst &Inst, unsigned OpIdx) const;
1016   unsigned findImplicitSGPRReadInVOP(const MCInst &Inst) const;
1017 
1018   bool trySkipId(const StringRef Id);
1019   bool trySkipToken(const AsmToken::TokenKind Kind);
1020   bool skipToken(const AsmToken::TokenKind Kind, const StringRef ErrMsg);
1021   bool parseString(StringRef &Val, const StringRef ErrMsg = "expected a string");
1022   bool parseExpr(int64_t &Imm);
1023 
1024 public:
1025   OperandMatchResultTy parseOptionalOperand(OperandVector &Operands);
1026 
1027   OperandMatchResultTy parseExpTgt(OperandVector &Operands);
1028   OperandMatchResultTy parseSendMsgOp(OperandVector &Operands);
1029   OperandMatchResultTy parseInterpSlot(OperandVector &Operands);
1030   OperandMatchResultTy parseInterpAttr(OperandVector &Operands);
1031   OperandMatchResultTy parseSOppBrTarget(OperandVector &Operands);
1032 
1033   bool parseSwizzleOperands(const unsigned OpNum, int64_t* Op,
1034                             const unsigned MinVal,
1035                             const unsigned MaxVal,
1036                             const StringRef ErrMsg);
1037   OperandMatchResultTy parseSwizzleOp(OperandVector &Operands);
1038   bool parseSwizzleOffset(int64_t &Imm);
1039   bool parseSwizzleMacro(int64_t &Imm);
1040   bool parseSwizzleQuadPerm(int64_t &Imm);
1041   bool parseSwizzleBitmaskPerm(int64_t &Imm);
1042   bool parseSwizzleBroadcast(int64_t &Imm);
1043   bool parseSwizzleSwap(int64_t &Imm);
1044   bool parseSwizzleReverse(int64_t &Imm);
1045 
1046   void cvtMubuf(MCInst &Inst, const OperandVector &Operands) { cvtMubufImpl(Inst, Operands, false, false); }
1047   void cvtMubufAtomic(MCInst &Inst, const OperandVector &Operands) { cvtMubufImpl(Inst, Operands, true, false); }
1048   void cvtMubufAtomicReturn(MCInst &Inst, const OperandVector &Operands) { cvtMubufImpl(Inst, Operands, true, true); }
1049   void cvtMtbuf(MCInst &Inst, const OperandVector &Operands);
1050 
1051   AMDGPUOperand::Ptr defaultGLC() const;
1052   AMDGPUOperand::Ptr defaultSLC() const;
1053   AMDGPUOperand::Ptr defaultTFE() const;
1054 
1055   AMDGPUOperand::Ptr defaultDMask() const;
1056   AMDGPUOperand::Ptr defaultUNorm() const;
1057   AMDGPUOperand::Ptr defaultDA() const;
1058   AMDGPUOperand::Ptr defaultR128() const;
1059   AMDGPUOperand::Ptr defaultLWE() const;
1060   AMDGPUOperand::Ptr defaultSMRDOffset8() const;
1061   AMDGPUOperand::Ptr defaultSMRDOffset20() const;
1062   AMDGPUOperand::Ptr defaultSMRDLiteralOffset() const;
1063   AMDGPUOperand::Ptr defaultOffsetU12() const;
1064   AMDGPUOperand::Ptr defaultOffsetS13() const;
1065 
1066   OperandMatchResultTy parseOModOperand(OperandVector &Operands);
1067 
1068   void cvtVOP3(MCInst &Inst, const OperandVector &Operands,
1069                OptionalImmIndexMap &OptionalIdx);
1070   void cvtVOP3OpSel(MCInst &Inst, const OperandVector &Operands);
1071   void cvtVOP3(MCInst &Inst, const OperandVector &Operands);
1072   void cvtVOP3P(MCInst &Inst, const OperandVector &Operands);
1073 
1074   void cvtVOP3Interp(MCInst &Inst, const OperandVector &Operands);
1075 
1076   void cvtMIMG(MCInst &Inst, const OperandVector &Operands,
1077                bool IsAtomic = false);
1078   void cvtMIMGAtomic(MCInst &Inst, const OperandVector &Operands);
1079 
1080   OperandMatchResultTy parseDPPCtrl(OperandVector &Operands);
1081   AMDGPUOperand::Ptr defaultRowMask() const;
1082   AMDGPUOperand::Ptr defaultBankMask() const;
1083   AMDGPUOperand::Ptr defaultBoundCtrl() const;
1084   void cvtDPP(MCInst &Inst, const OperandVector &Operands);
1085 
1086   OperandMatchResultTy parseSDWASel(OperandVector &Operands, StringRef Prefix,
1087                                     AMDGPUOperand::ImmTy Type);
1088   OperandMatchResultTy parseSDWADstUnused(OperandVector &Operands);
1089   void cvtSdwaVOP1(MCInst &Inst, const OperandVector &Operands);
1090   void cvtSdwaVOP2(MCInst &Inst, const OperandVector &Operands);
1091   void cvtSdwaVOP2b(MCInst &Inst, const OperandVector &Operands);
1092   void cvtSdwaVOPC(MCInst &Inst, const OperandVector &Operands);
1093   void cvtSDWA(MCInst &Inst, const OperandVector &Operands,
1094                 uint64_t BasicInstType, bool skipVcc = false);
1095 };
1096 
1097 struct OptionalOperand {
1098   const char *Name;
1099   AMDGPUOperand::ImmTy Type;
1100   bool IsBit;
1101   bool (*ConvertResult)(int64_t&);
1102 };
1103 
1104 } // end anonymous namespace
1105 
1106 // May be called with integer type with equivalent bitwidth.
1107 static const fltSemantics *getFltSemantics(unsigned Size) {
1108   switch (Size) {
1109   case 4:
1110     return &APFloat::IEEEsingle();
1111   case 8:
1112     return &APFloat::IEEEdouble();
1113   case 2:
1114     return &APFloat::IEEEhalf();
1115   default:
1116     llvm_unreachable("unsupported fp type");
1117   }
1118 }
1119 
1120 static const fltSemantics *getFltSemantics(MVT VT) {
1121   return getFltSemantics(VT.getSizeInBits() / 8);
1122 }
1123 
1124 static const fltSemantics *getOpFltSemantics(uint8_t OperandType) {
1125   switch (OperandType) {
1126   case AMDGPU::OPERAND_REG_IMM_INT32:
1127   case AMDGPU::OPERAND_REG_IMM_FP32:
1128   case AMDGPU::OPERAND_REG_INLINE_C_INT32:
1129   case AMDGPU::OPERAND_REG_INLINE_C_FP32:
1130     return &APFloat::IEEEsingle();
1131   case AMDGPU::OPERAND_REG_IMM_INT64:
1132   case AMDGPU::OPERAND_REG_IMM_FP64:
1133   case AMDGPU::OPERAND_REG_INLINE_C_INT64:
1134   case AMDGPU::OPERAND_REG_INLINE_C_FP64:
1135     return &APFloat::IEEEdouble();
1136   case AMDGPU::OPERAND_REG_IMM_INT16:
1137   case AMDGPU::OPERAND_REG_IMM_FP16:
1138   case AMDGPU::OPERAND_REG_INLINE_C_INT16:
1139   case AMDGPU::OPERAND_REG_INLINE_C_FP16:
1140   case AMDGPU::OPERAND_REG_INLINE_C_V2INT16:
1141   case AMDGPU::OPERAND_REG_INLINE_C_V2FP16:
1142     return &APFloat::IEEEhalf();
1143   default:
1144     llvm_unreachable("unsupported fp type");
1145   }
1146 }
1147 
1148 //===----------------------------------------------------------------------===//
1149 // Operand
1150 //===----------------------------------------------------------------------===//
1151 
1152 static bool canLosslesslyConvertToFPType(APFloat &FPLiteral, MVT VT) {
1153   bool Lost;
1154 
1155   // Convert literal to single precision
1156   APFloat::opStatus Status = FPLiteral.convert(*getFltSemantics(VT),
1157                                                APFloat::rmNearestTiesToEven,
1158                                                &Lost);
1159   // We allow precision lost but not overflow or underflow
1160   if (Status != APFloat::opOK &&
1161       Lost &&
1162       ((Status & APFloat::opOverflow)  != 0 ||
1163        (Status & APFloat::opUnderflow) != 0)) {
1164     return false;
1165   }
1166 
1167   return true;
1168 }
1169 
1170 bool AMDGPUOperand::isInlinableImm(MVT type) const {
1171   if (!isImmTy(ImmTyNone)) {
1172     // Only plain immediates are inlinable (e.g. "clamp" attribute is not)
1173     return false;
1174   }
1175   // TODO: We should avoid using host float here. It would be better to
1176   // check the float bit values which is what a few other places do.
1177   // We've had bot failures before due to weird NaN support on mips hosts.
1178 
1179   APInt Literal(64, Imm.Val);
1180 
1181   if (Imm.IsFPImm) { // We got fp literal token
1182     if (type == MVT::f64 || type == MVT::i64) { // Expected 64-bit operand
1183       return AMDGPU::isInlinableLiteral64(Imm.Val,
1184                                           AsmParser->hasInv2PiInlineImm());
1185     }
1186 
1187     APFloat FPLiteral(APFloat::IEEEdouble(), APInt(64, Imm.Val));
1188     if (!canLosslesslyConvertToFPType(FPLiteral, type))
1189       return false;
1190 
1191     if (type.getScalarSizeInBits() == 16) {
1192       return AMDGPU::isInlinableLiteral16(
1193         static_cast<int16_t>(FPLiteral.bitcastToAPInt().getZExtValue()),
1194         AsmParser->hasInv2PiInlineImm());
1195     }
1196 
1197     // Check if single precision literal is inlinable
1198     return AMDGPU::isInlinableLiteral32(
1199       static_cast<int32_t>(FPLiteral.bitcastToAPInt().getZExtValue()),
1200       AsmParser->hasInv2PiInlineImm());
1201   }
1202 
1203   // We got int literal token.
1204   if (type == MVT::f64 || type == MVT::i64) { // Expected 64-bit operand
1205     return AMDGPU::isInlinableLiteral64(Imm.Val,
1206                                         AsmParser->hasInv2PiInlineImm());
1207   }
1208 
1209   if (type.getScalarSizeInBits() == 16) {
1210     return AMDGPU::isInlinableLiteral16(
1211       static_cast<int16_t>(Literal.getLoBits(16).getSExtValue()),
1212       AsmParser->hasInv2PiInlineImm());
1213   }
1214 
1215   return AMDGPU::isInlinableLiteral32(
1216     static_cast<int32_t>(Literal.getLoBits(32).getZExtValue()),
1217     AsmParser->hasInv2PiInlineImm());
1218 }
1219 
1220 bool AMDGPUOperand::isLiteralImm(MVT type) const {
1221   // Check that this immediate can be added as literal
1222   if (!isImmTy(ImmTyNone)) {
1223     return false;
1224   }
1225 
1226   if (!Imm.IsFPImm) {
1227     // We got int literal token.
1228 
1229     if (type == MVT::f64 && hasFPModifiers()) {
1230       // Cannot apply fp modifiers to int literals preserving the same semantics
1231       // for VOP1/2/C and VOP3 because of integer truncation. To avoid ambiguity,
1232       // disable these cases.
1233       return false;
1234     }
1235 
1236     unsigned Size = type.getSizeInBits();
1237     if (Size == 64)
1238       Size = 32;
1239 
1240     // FIXME: 64-bit operands can zero extend, sign extend, or pad zeroes for FP
1241     // types.
1242     return isUIntN(Size, Imm.Val) || isIntN(Size, Imm.Val);
1243   }
1244 
1245   // We got fp literal token
1246   if (type == MVT::f64) { // Expected 64-bit fp operand
1247     // We would set low 64-bits of literal to zeroes but we accept this literals
1248     return true;
1249   }
1250 
1251   if (type == MVT::i64) { // Expected 64-bit int operand
1252     // We don't allow fp literals in 64-bit integer instructions. It is
1253     // unclear how we should encode them.
1254     return false;
1255   }
1256 
1257   APFloat FPLiteral(APFloat::IEEEdouble(), APInt(64, Imm.Val));
1258   return canLosslesslyConvertToFPType(FPLiteral, type);
1259 }
1260 
1261 bool AMDGPUOperand::isRegClass(unsigned RCID) const {
1262   return isRegKind() && AsmParser->getMRI()->getRegClass(RCID).contains(getReg());
1263 }
1264 
1265 bool AMDGPUOperand::isSDWARegKind() const {
1266   if (AsmParser->isVI())
1267     return isVReg();
1268   else if (AsmParser->isGFX9())
1269     return isRegKind();
1270   else
1271     return false;
1272 }
1273 
1274 uint64_t AMDGPUOperand::applyInputFPModifiers(uint64_t Val, unsigned Size) const
1275 {
1276   assert(isImmTy(ImmTyNone) && Imm.Mods.hasFPModifiers());
1277   assert(Size == 2 || Size == 4 || Size == 8);
1278 
1279   const uint64_t FpSignMask = (1ULL << (Size * 8 - 1));
1280 
1281   if (Imm.Mods.Abs) {
1282     Val &= ~FpSignMask;
1283   }
1284   if (Imm.Mods.Neg) {
1285     Val ^= FpSignMask;
1286   }
1287 
1288   return Val;
1289 }
1290 
1291 void AMDGPUOperand::addImmOperands(MCInst &Inst, unsigned N, bool ApplyModifiers) const {
1292   if (AMDGPU::isSISrcOperand(AsmParser->getMII()->get(Inst.getOpcode()),
1293                              Inst.getNumOperands())) {
1294     addLiteralImmOperand(Inst, Imm.Val,
1295                          ApplyModifiers &
1296                          isImmTy(ImmTyNone) && Imm.Mods.hasFPModifiers());
1297   } else {
1298     assert(!isImmTy(ImmTyNone) || !hasModifiers());
1299     Inst.addOperand(MCOperand::createImm(Imm.Val));
1300   }
1301 }
1302 
1303 void AMDGPUOperand::addLiteralImmOperand(MCInst &Inst, int64_t Val, bool ApplyModifiers) const {
1304   const auto& InstDesc = AsmParser->getMII()->get(Inst.getOpcode());
1305   auto OpNum = Inst.getNumOperands();
1306   // Check that this operand accepts literals
1307   assert(AMDGPU::isSISrcOperand(InstDesc, OpNum));
1308 
1309   if (ApplyModifiers) {
1310     assert(AMDGPU::isSISrcFPOperand(InstDesc, OpNum));
1311     const unsigned Size = Imm.IsFPImm ? sizeof(double) : getOperandSize(InstDesc, OpNum);
1312     Val = applyInputFPModifiers(Val, Size);
1313   }
1314 
1315   APInt Literal(64, Val);
1316   uint8_t OpTy = InstDesc.OpInfo[OpNum].OperandType;
1317 
1318   if (Imm.IsFPImm) { // We got fp literal token
1319     switch (OpTy) {
1320     case AMDGPU::OPERAND_REG_IMM_INT64:
1321     case AMDGPU::OPERAND_REG_IMM_FP64:
1322     case AMDGPU::OPERAND_REG_INLINE_C_INT64:
1323     case AMDGPU::OPERAND_REG_INLINE_C_FP64:
1324       if (AMDGPU::isInlinableLiteral64(Literal.getZExtValue(),
1325                                        AsmParser->hasInv2PiInlineImm())) {
1326         Inst.addOperand(MCOperand::createImm(Literal.getZExtValue()));
1327         return;
1328       }
1329 
1330       // Non-inlineable
1331       if (AMDGPU::isSISrcFPOperand(InstDesc, OpNum)) { // Expected 64-bit fp operand
1332         // For fp operands we check if low 32 bits are zeros
1333         if (Literal.getLoBits(32) != 0) {
1334           const_cast<AMDGPUAsmParser *>(AsmParser)->Warning(Inst.getLoc(),
1335           "Can't encode literal as exact 64-bit floating-point operand. "
1336           "Low 32-bits will be set to zero");
1337         }
1338 
1339         Inst.addOperand(MCOperand::createImm(Literal.lshr(32).getZExtValue()));
1340         return;
1341       }
1342 
1343       // We don't allow fp literals in 64-bit integer instructions. It is
1344       // unclear how we should encode them. This case should be checked earlier
1345       // in predicate methods (isLiteralImm())
1346       llvm_unreachable("fp literal in 64-bit integer instruction.");
1347 
1348     case AMDGPU::OPERAND_REG_IMM_INT32:
1349     case AMDGPU::OPERAND_REG_IMM_FP32:
1350     case AMDGPU::OPERAND_REG_INLINE_C_INT32:
1351     case AMDGPU::OPERAND_REG_INLINE_C_FP32:
1352     case AMDGPU::OPERAND_REG_IMM_INT16:
1353     case AMDGPU::OPERAND_REG_IMM_FP16:
1354     case AMDGPU::OPERAND_REG_INLINE_C_INT16:
1355     case AMDGPU::OPERAND_REG_INLINE_C_FP16:
1356     case AMDGPU::OPERAND_REG_INLINE_C_V2INT16:
1357     case AMDGPU::OPERAND_REG_INLINE_C_V2FP16: {
1358       bool lost;
1359       APFloat FPLiteral(APFloat::IEEEdouble(), Literal);
1360       // Convert literal to single precision
1361       FPLiteral.convert(*getOpFltSemantics(OpTy),
1362                         APFloat::rmNearestTiesToEven, &lost);
1363       // We allow precision lost but not overflow or underflow. This should be
1364       // checked earlier in isLiteralImm()
1365 
1366       uint64_t ImmVal = FPLiteral.bitcastToAPInt().getZExtValue();
1367       if (OpTy == AMDGPU::OPERAND_REG_INLINE_C_V2INT16 ||
1368           OpTy == AMDGPU::OPERAND_REG_INLINE_C_V2FP16) {
1369         ImmVal |= (ImmVal << 16);
1370       }
1371 
1372       Inst.addOperand(MCOperand::createImm(ImmVal));
1373       return;
1374     }
1375     default:
1376       llvm_unreachable("invalid operand size");
1377     }
1378 
1379     return;
1380   }
1381 
1382    // We got int literal token.
1383   // Only sign extend inline immediates.
1384   // FIXME: No errors on truncation
1385   switch (OpTy) {
1386   case AMDGPU::OPERAND_REG_IMM_INT32:
1387   case AMDGPU::OPERAND_REG_IMM_FP32:
1388   case AMDGPU::OPERAND_REG_INLINE_C_INT32:
1389   case AMDGPU::OPERAND_REG_INLINE_C_FP32:
1390     if (isInt<32>(Val) &&
1391         AMDGPU::isInlinableLiteral32(static_cast<int32_t>(Val),
1392                                      AsmParser->hasInv2PiInlineImm())) {
1393       Inst.addOperand(MCOperand::createImm(Val));
1394       return;
1395     }
1396 
1397     Inst.addOperand(MCOperand::createImm(Val & 0xffffffff));
1398     return;
1399 
1400   case AMDGPU::OPERAND_REG_IMM_INT64:
1401   case AMDGPU::OPERAND_REG_IMM_FP64:
1402   case AMDGPU::OPERAND_REG_INLINE_C_INT64:
1403   case AMDGPU::OPERAND_REG_INLINE_C_FP64:
1404     if (AMDGPU::isInlinableLiteral64(Val, AsmParser->hasInv2PiInlineImm())) {
1405       Inst.addOperand(MCOperand::createImm(Val));
1406       return;
1407     }
1408 
1409     Inst.addOperand(MCOperand::createImm(Lo_32(Val)));
1410     return;
1411 
1412   case AMDGPU::OPERAND_REG_IMM_INT16:
1413   case AMDGPU::OPERAND_REG_IMM_FP16:
1414   case AMDGPU::OPERAND_REG_INLINE_C_INT16:
1415   case AMDGPU::OPERAND_REG_INLINE_C_FP16:
1416     if (isInt<16>(Val) &&
1417         AMDGPU::isInlinableLiteral16(static_cast<int16_t>(Val),
1418                                      AsmParser->hasInv2PiInlineImm())) {
1419       Inst.addOperand(MCOperand::createImm(Val));
1420       return;
1421     }
1422 
1423     Inst.addOperand(MCOperand::createImm(Val & 0xffff));
1424     return;
1425 
1426   case AMDGPU::OPERAND_REG_INLINE_C_V2INT16:
1427   case AMDGPU::OPERAND_REG_INLINE_C_V2FP16: {
1428     auto LiteralVal = static_cast<uint16_t>(Literal.getLoBits(16).getZExtValue());
1429     assert(AMDGPU::isInlinableLiteral16(LiteralVal,
1430                                         AsmParser->hasInv2PiInlineImm()));
1431 
1432     uint32_t ImmVal = static_cast<uint32_t>(LiteralVal) << 16 |
1433                       static_cast<uint32_t>(LiteralVal);
1434     Inst.addOperand(MCOperand::createImm(ImmVal));
1435     return;
1436   }
1437   default:
1438     llvm_unreachable("invalid operand size");
1439   }
1440 }
1441 
1442 template <unsigned Bitwidth>
1443 void AMDGPUOperand::addKImmFPOperands(MCInst &Inst, unsigned N) const {
1444   APInt Literal(64, Imm.Val);
1445 
1446   if (!Imm.IsFPImm) {
1447     // We got int literal token.
1448     Inst.addOperand(MCOperand::createImm(Literal.getLoBits(Bitwidth).getZExtValue()));
1449     return;
1450   }
1451 
1452   bool Lost;
1453   APFloat FPLiteral(APFloat::IEEEdouble(), Literal);
1454   FPLiteral.convert(*getFltSemantics(Bitwidth / 8),
1455                     APFloat::rmNearestTiesToEven, &Lost);
1456   Inst.addOperand(MCOperand::createImm(FPLiteral.bitcastToAPInt().getZExtValue()));
1457 }
1458 
1459 void AMDGPUOperand::addRegOperands(MCInst &Inst, unsigned N) const {
1460   Inst.addOperand(MCOperand::createReg(AMDGPU::getMCReg(getReg(), AsmParser->getSTI())));
1461 }
1462 
1463 //===----------------------------------------------------------------------===//
1464 // AsmParser
1465 //===----------------------------------------------------------------------===//
1466 
1467 static int getRegClass(RegisterKind Is, unsigned RegWidth) {
1468   if (Is == IS_VGPR) {
1469     switch (RegWidth) {
1470       default: return -1;
1471       case 1: return AMDGPU::VGPR_32RegClassID;
1472       case 2: return AMDGPU::VReg_64RegClassID;
1473       case 3: return AMDGPU::VReg_96RegClassID;
1474       case 4: return AMDGPU::VReg_128RegClassID;
1475       case 8: return AMDGPU::VReg_256RegClassID;
1476       case 16: return AMDGPU::VReg_512RegClassID;
1477     }
1478   } else if (Is == IS_TTMP) {
1479     switch (RegWidth) {
1480       default: return -1;
1481       case 1: return AMDGPU::TTMP_32RegClassID;
1482       case 2: return AMDGPU::TTMP_64RegClassID;
1483       case 4: return AMDGPU::TTMP_128RegClassID;
1484     }
1485   } else if (Is == IS_SGPR) {
1486     switch (RegWidth) {
1487       default: return -1;
1488       case 1: return AMDGPU::SGPR_32RegClassID;
1489       case 2: return AMDGPU::SGPR_64RegClassID;
1490       case 4: return AMDGPU::SGPR_128RegClassID;
1491       case 8: return AMDGPU::SReg_256RegClassID;
1492       case 16: return AMDGPU::SReg_512RegClassID;
1493     }
1494   }
1495   return -1;
1496 }
1497 
1498 static unsigned getSpecialRegForName(StringRef RegName) {
1499   return StringSwitch<unsigned>(RegName)
1500     .Case("exec", AMDGPU::EXEC)
1501     .Case("vcc", AMDGPU::VCC)
1502     .Case("flat_scratch", AMDGPU::FLAT_SCR)
1503     .Case("m0", AMDGPU::M0)
1504     .Case("scc", AMDGPU::SCC)
1505     .Case("tba", AMDGPU::TBA)
1506     .Case("tma", AMDGPU::TMA)
1507     .Case("flat_scratch_lo", AMDGPU::FLAT_SCR_LO)
1508     .Case("flat_scratch_hi", AMDGPU::FLAT_SCR_HI)
1509     .Case("vcc_lo", AMDGPU::VCC_LO)
1510     .Case("vcc_hi", AMDGPU::VCC_HI)
1511     .Case("exec_lo", AMDGPU::EXEC_LO)
1512     .Case("exec_hi", AMDGPU::EXEC_HI)
1513     .Case("tma_lo", AMDGPU::TMA_LO)
1514     .Case("tma_hi", AMDGPU::TMA_HI)
1515     .Case("tba_lo", AMDGPU::TBA_LO)
1516     .Case("tba_hi", AMDGPU::TBA_HI)
1517     .Default(0);
1518 }
1519 
1520 bool AMDGPUAsmParser::ParseRegister(unsigned &RegNo, SMLoc &StartLoc,
1521                                     SMLoc &EndLoc) {
1522   auto R = parseRegister();
1523   if (!R) return true;
1524   assert(R->isReg());
1525   RegNo = R->getReg();
1526   StartLoc = R->getStartLoc();
1527   EndLoc = R->getEndLoc();
1528   return false;
1529 }
1530 
1531 bool AMDGPUAsmParser::AddNextRegisterToList(unsigned &Reg, unsigned &RegWidth,
1532                                             RegisterKind RegKind, unsigned Reg1,
1533                                             unsigned RegNum) {
1534   switch (RegKind) {
1535   case IS_SPECIAL:
1536     if (Reg == AMDGPU::EXEC_LO && Reg1 == AMDGPU::EXEC_HI) {
1537       Reg = AMDGPU::EXEC;
1538       RegWidth = 2;
1539       return true;
1540     }
1541     if (Reg == AMDGPU::FLAT_SCR_LO && Reg1 == AMDGPU::FLAT_SCR_HI) {
1542       Reg = AMDGPU::FLAT_SCR;
1543       RegWidth = 2;
1544       return true;
1545     }
1546     if (Reg == AMDGPU::VCC_LO && Reg1 == AMDGPU::VCC_HI) {
1547       Reg = AMDGPU::VCC;
1548       RegWidth = 2;
1549       return true;
1550     }
1551     if (Reg == AMDGPU::TBA_LO && Reg1 == AMDGPU::TBA_HI) {
1552       Reg = AMDGPU::TBA;
1553       RegWidth = 2;
1554       return true;
1555     }
1556     if (Reg == AMDGPU::TMA_LO && Reg1 == AMDGPU::TMA_HI) {
1557       Reg = AMDGPU::TMA;
1558       RegWidth = 2;
1559       return true;
1560     }
1561     return false;
1562   case IS_VGPR:
1563   case IS_SGPR:
1564   case IS_TTMP:
1565     if (Reg1 != Reg + RegWidth) {
1566       return false;
1567     }
1568     RegWidth++;
1569     return true;
1570   default:
1571     llvm_unreachable("unexpected register kind");
1572   }
1573 }
1574 
1575 bool AMDGPUAsmParser::ParseAMDGPURegister(RegisterKind &RegKind, unsigned &Reg,
1576                                           unsigned &RegNum, unsigned &RegWidth,
1577                                           unsigned *DwordRegIndex) {
1578   if (DwordRegIndex) { *DwordRegIndex = 0; }
1579   const MCRegisterInfo *TRI = getContext().getRegisterInfo();
1580   if (getLexer().is(AsmToken::Identifier)) {
1581     StringRef RegName = Parser.getTok().getString();
1582     if ((Reg = getSpecialRegForName(RegName))) {
1583       Parser.Lex();
1584       RegKind = IS_SPECIAL;
1585     } else {
1586       unsigned RegNumIndex = 0;
1587       if (RegName[0] == 'v') {
1588         RegNumIndex = 1;
1589         RegKind = IS_VGPR;
1590       } else if (RegName[0] == 's') {
1591         RegNumIndex = 1;
1592         RegKind = IS_SGPR;
1593       } else if (RegName.startswith("ttmp")) {
1594         RegNumIndex = strlen("ttmp");
1595         RegKind = IS_TTMP;
1596       } else {
1597         return false;
1598       }
1599       if (RegName.size() > RegNumIndex) {
1600         // Single 32-bit register: vXX.
1601         if (RegName.substr(RegNumIndex).getAsInteger(10, RegNum))
1602           return false;
1603         Parser.Lex();
1604         RegWidth = 1;
1605       } else {
1606         // Range of registers: v[XX:YY]. ":YY" is optional.
1607         Parser.Lex();
1608         int64_t RegLo, RegHi;
1609         if (getLexer().isNot(AsmToken::LBrac))
1610           return false;
1611         Parser.Lex();
1612 
1613         if (getParser().parseAbsoluteExpression(RegLo))
1614           return false;
1615 
1616         const bool isRBrace = getLexer().is(AsmToken::RBrac);
1617         if (!isRBrace && getLexer().isNot(AsmToken::Colon))
1618           return false;
1619         Parser.Lex();
1620 
1621         if (isRBrace) {
1622           RegHi = RegLo;
1623         } else {
1624           if (getParser().parseAbsoluteExpression(RegHi))
1625             return false;
1626 
1627           if (getLexer().isNot(AsmToken::RBrac))
1628             return false;
1629           Parser.Lex();
1630         }
1631         RegNum = (unsigned) RegLo;
1632         RegWidth = (RegHi - RegLo) + 1;
1633       }
1634     }
1635   } else if (getLexer().is(AsmToken::LBrac)) {
1636     // List of consecutive registers: [s0,s1,s2,s3]
1637     Parser.Lex();
1638     if (!ParseAMDGPURegister(RegKind, Reg, RegNum, RegWidth, nullptr))
1639       return false;
1640     if (RegWidth != 1)
1641       return false;
1642     RegisterKind RegKind1;
1643     unsigned Reg1, RegNum1, RegWidth1;
1644     do {
1645       if (getLexer().is(AsmToken::Comma)) {
1646         Parser.Lex();
1647       } else if (getLexer().is(AsmToken::RBrac)) {
1648         Parser.Lex();
1649         break;
1650       } else if (ParseAMDGPURegister(RegKind1, Reg1, RegNum1, RegWidth1, nullptr)) {
1651         if (RegWidth1 != 1) {
1652           return false;
1653         }
1654         if (RegKind1 != RegKind) {
1655           return false;
1656         }
1657         if (!AddNextRegisterToList(Reg, RegWidth, RegKind1, Reg1, RegNum1)) {
1658           return false;
1659         }
1660       } else {
1661         return false;
1662       }
1663     } while (true);
1664   } else {
1665     return false;
1666   }
1667   switch (RegKind) {
1668   case IS_SPECIAL:
1669     RegNum = 0;
1670     RegWidth = 1;
1671     break;
1672   case IS_VGPR:
1673   case IS_SGPR:
1674   case IS_TTMP:
1675   {
1676     unsigned Size = 1;
1677     if (RegKind == IS_SGPR || RegKind == IS_TTMP) {
1678       // SGPR and TTMP registers must be aligned. Max required alignment is 4 dwords.
1679       Size = std::min(RegWidth, 4u);
1680     }
1681     if (RegNum % Size != 0)
1682       return false;
1683     if (DwordRegIndex) { *DwordRegIndex = RegNum; }
1684     RegNum = RegNum / Size;
1685     int RCID = getRegClass(RegKind, RegWidth);
1686     if (RCID == -1)
1687       return false;
1688     const MCRegisterClass RC = TRI->getRegClass(RCID);
1689     if (RegNum >= RC.getNumRegs())
1690       return false;
1691     Reg = RC.getRegister(RegNum);
1692     break;
1693   }
1694 
1695   default:
1696     llvm_unreachable("unexpected register kind");
1697   }
1698 
1699   if (!subtargetHasRegister(*TRI, Reg))
1700     return false;
1701   return true;
1702 }
1703 
1704 std::unique_ptr<AMDGPUOperand> AMDGPUAsmParser::parseRegister() {
1705   const auto &Tok = Parser.getTok();
1706   SMLoc StartLoc = Tok.getLoc();
1707   SMLoc EndLoc = Tok.getEndLoc();
1708   RegisterKind RegKind;
1709   unsigned Reg, RegNum, RegWidth, DwordRegIndex;
1710 
1711   if (!ParseAMDGPURegister(RegKind, Reg, RegNum, RegWidth, &DwordRegIndex)) {
1712     return nullptr;
1713   }
1714   KernelScope.usesRegister(RegKind, DwordRegIndex, RegWidth);
1715   return AMDGPUOperand::CreateReg(this, Reg, StartLoc, EndLoc, false);
1716 }
1717 
1718 bool
1719 AMDGPUAsmParser::parseAbsoluteExpr(int64_t &Val, bool AbsMod) {
1720   if (AbsMod && getLexer().peekTok().is(AsmToken::Pipe) &&
1721       (getLexer().getKind() == AsmToken::Integer ||
1722        getLexer().getKind() == AsmToken::Real)) {
1723     // This is a workaround for handling operands like these:
1724     //     |1.0|
1725     //     |-1|
1726     // This syntax is not compatible with syntax of standard
1727     // MC expressions (due to the trailing '|').
1728 
1729     SMLoc EndLoc;
1730     const MCExpr *Expr;
1731 
1732     if (getParser().parsePrimaryExpr(Expr, EndLoc)) {
1733       return true;
1734     }
1735 
1736     return !Expr->evaluateAsAbsolute(Val);
1737   }
1738 
1739   return getParser().parseAbsoluteExpression(Val);
1740 }
1741 
1742 OperandMatchResultTy
1743 AMDGPUAsmParser::parseImm(OperandVector &Operands, bool AbsMod) {
1744   // TODO: add syntactic sugar for 1/(2*PI)
1745   bool Minus = false;
1746   if (getLexer().getKind() == AsmToken::Minus) {
1747     Minus = true;
1748     Parser.Lex();
1749   }
1750 
1751   SMLoc S = Parser.getTok().getLoc();
1752   switch(getLexer().getKind()) {
1753   case AsmToken::Integer: {
1754     int64_t IntVal;
1755     if (parseAbsoluteExpr(IntVal, AbsMod))
1756       return MatchOperand_ParseFail;
1757     if (Minus)
1758       IntVal *= -1;
1759     Operands.push_back(AMDGPUOperand::CreateImm(this, IntVal, S));
1760     return MatchOperand_Success;
1761   }
1762   case AsmToken::Real: {
1763     int64_t IntVal;
1764     if (parseAbsoluteExpr(IntVal, AbsMod))
1765       return MatchOperand_ParseFail;
1766 
1767     APFloat F(BitsToDouble(IntVal));
1768     if (Minus)
1769       F.changeSign();
1770     Operands.push_back(
1771         AMDGPUOperand::CreateImm(this, F.bitcastToAPInt().getZExtValue(), S,
1772                                  AMDGPUOperand::ImmTyNone, true));
1773     return MatchOperand_Success;
1774   }
1775   default:
1776     return Minus ? MatchOperand_ParseFail : MatchOperand_NoMatch;
1777   }
1778 }
1779 
1780 OperandMatchResultTy
1781 AMDGPUAsmParser::parseReg(OperandVector &Operands) {
1782   if (auto R = parseRegister()) {
1783     assert(R->isReg());
1784     R->Reg.IsForcedVOP3 = isForcedVOP3();
1785     Operands.push_back(std::move(R));
1786     return MatchOperand_Success;
1787   }
1788   return MatchOperand_NoMatch;
1789 }
1790 
1791 OperandMatchResultTy
1792 AMDGPUAsmParser::parseRegOrImm(OperandVector &Operands, bool AbsMod) {
1793   auto res = parseImm(Operands, AbsMod);
1794   if (res != MatchOperand_NoMatch) {
1795     return res;
1796   }
1797 
1798   return parseReg(Operands);
1799 }
1800 
1801 OperandMatchResultTy
1802 AMDGPUAsmParser::parseRegOrImmWithFPInputMods(OperandVector &Operands,
1803                                               bool AllowImm) {
1804   bool Negate = false, Negate2 = false, Abs = false, Abs2 = false;
1805 
1806   if (getLexer().getKind()== AsmToken::Minus) {
1807     const AsmToken NextToken = getLexer().peekTok();
1808 
1809     // Disable ambiguous constructs like '--1' etc. Should use neg(-1) instead.
1810     if (NextToken.is(AsmToken::Minus)) {
1811       Error(Parser.getTok().getLoc(), "invalid syntax, expected 'neg' modifier");
1812       return MatchOperand_ParseFail;
1813     }
1814 
1815     // '-' followed by an integer literal N should be interpreted as integer
1816     // negation rather than a floating-point NEG modifier applied to N.
1817     // Beside being contr-intuitive, such use of floating-point NEG modifier
1818     // results in different meaning of integer literals used with VOP1/2/C
1819     // and VOP3, for example:
1820     //    v_exp_f32_e32 v5, -1 // VOP1: src0 = 0xFFFFFFFF
1821     //    v_exp_f32_e64 v5, -1 // VOP3: src0 = 0x80000001
1822     // Negative fp literals should be handled likewise for unifomtity
1823     if (!NextToken.is(AsmToken::Integer) && !NextToken.is(AsmToken::Real)) {
1824       Parser.Lex();
1825       Negate = true;
1826     }
1827   }
1828 
1829   if (getLexer().getKind() == AsmToken::Identifier &&
1830       Parser.getTok().getString() == "neg") {
1831     if (Negate) {
1832       Error(Parser.getTok().getLoc(), "expected register or immediate");
1833       return MatchOperand_ParseFail;
1834     }
1835     Parser.Lex();
1836     Negate2 = true;
1837     if (getLexer().isNot(AsmToken::LParen)) {
1838       Error(Parser.getTok().getLoc(), "expected left paren after neg");
1839       return MatchOperand_ParseFail;
1840     }
1841     Parser.Lex();
1842   }
1843 
1844   if (getLexer().getKind() == AsmToken::Identifier &&
1845       Parser.getTok().getString() == "abs") {
1846     Parser.Lex();
1847     Abs2 = true;
1848     if (getLexer().isNot(AsmToken::LParen)) {
1849       Error(Parser.getTok().getLoc(), "expected left paren after abs");
1850       return MatchOperand_ParseFail;
1851     }
1852     Parser.Lex();
1853   }
1854 
1855   if (getLexer().getKind() == AsmToken::Pipe) {
1856     if (Abs2) {
1857       Error(Parser.getTok().getLoc(), "expected register or immediate");
1858       return MatchOperand_ParseFail;
1859     }
1860     Parser.Lex();
1861     Abs = true;
1862   }
1863 
1864   OperandMatchResultTy Res;
1865   if (AllowImm) {
1866     Res = parseRegOrImm(Operands, Abs);
1867   } else {
1868     Res = parseReg(Operands);
1869   }
1870   if (Res != MatchOperand_Success) {
1871     return Res;
1872   }
1873 
1874   AMDGPUOperand::Modifiers Mods;
1875   if (Abs) {
1876     if (getLexer().getKind() != AsmToken::Pipe) {
1877       Error(Parser.getTok().getLoc(), "expected vertical bar");
1878       return MatchOperand_ParseFail;
1879     }
1880     Parser.Lex();
1881     Mods.Abs = true;
1882   }
1883   if (Abs2) {
1884     if (getLexer().isNot(AsmToken::RParen)) {
1885       Error(Parser.getTok().getLoc(), "expected closing parentheses");
1886       return MatchOperand_ParseFail;
1887     }
1888     Parser.Lex();
1889     Mods.Abs = true;
1890   }
1891 
1892   if (Negate) {
1893     Mods.Neg = true;
1894   } else if (Negate2) {
1895     if (getLexer().isNot(AsmToken::RParen)) {
1896       Error(Parser.getTok().getLoc(), "expected closing parentheses");
1897       return MatchOperand_ParseFail;
1898     }
1899     Parser.Lex();
1900     Mods.Neg = true;
1901   }
1902 
1903   if (Mods.hasFPModifiers()) {
1904     AMDGPUOperand &Op = static_cast<AMDGPUOperand &>(*Operands.back());
1905     Op.setModifiers(Mods);
1906   }
1907   return MatchOperand_Success;
1908 }
1909 
1910 OperandMatchResultTy
1911 AMDGPUAsmParser::parseRegOrImmWithIntInputMods(OperandVector &Operands,
1912                                                bool AllowImm) {
1913   bool Sext = false;
1914 
1915   if (getLexer().getKind() == AsmToken::Identifier &&
1916       Parser.getTok().getString() == "sext") {
1917     Parser.Lex();
1918     Sext = true;
1919     if (getLexer().isNot(AsmToken::LParen)) {
1920       Error(Parser.getTok().getLoc(), "expected left paren after sext");
1921       return MatchOperand_ParseFail;
1922     }
1923     Parser.Lex();
1924   }
1925 
1926   OperandMatchResultTy Res;
1927   if (AllowImm) {
1928     Res = parseRegOrImm(Operands);
1929   } else {
1930     Res = parseReg(Operands);
1931   }
1932   if (Res != MatchOperand_Success) {
1933     return Res;
1934   }
1935 
1936   AMDGPUOperand::Modifiers Mods;
1937   if (Sext) {
1938     if (getLexer().isNot(AsmToken::RParen)) {
1939       Error(Parser.getTok().getLoc(), "expected closing parentheses");
1940       return MatchOperand_ParseFail;
1941     }
1942     Parser.Lex();
1943     Mods.Sext = true;
1944   }
1945 
1946   if (Mods.hasIntModifiers()) {
1947     AMDGPUOperand &Op = static_cast<AMDGPUOperand &>(*Operands.back());
1948     Op.setModifiers(Mods);
1949   }
1950 
1951   return MatchOperand_Success;
1952 }
1953 
1954 OperandMatchResultTy
1955 AMDGPUAsmParser::parseRegWithFPInputMods(OperandVector &Operands) {
1956   return parseRegOrImmWithFPInputMods(Operands, false);
1957 }
1958 
1959 OperandMatchResultTy
1960 AMDGPUAsmParser::parseRegWithIntInputMods(OperandVector &Operands) {
1961   return parseRegOrImmWithIntInputMods(Operands, false);
1962 }
1963 
1964 OperandMatchResultTy AMDGPUAsmParser::parseVReg32OrOff(OperandVector &Operands) {
1965   std::unique_ptr<AMDGPUOperand> Reg = parseRegister();
1966   if (Reg) {
1967     Operands.push_back(std::move(Reg));
1968     return MatchOperand_Success;
1969   }
1970 
1971   const AsmToken &Tok = Parser.getTok();
1972   if (Tok.getString() == "off") {
1973     Operands.push_back(AMDGPUOperand::CreateImm(this, 0, Tok.getLoc(),
1974                                                 AMDGPUOperand::ImmTyOff, false));
1975     Parser.Lex();
1976     return MatchOperand_Success;
1977   }
1978 
1979   return MatchOperand_NoMatch;
1980 }
1981 
1982 unsigned AMDGPUAsmParser::checkTargetMatchPredicate(MCInst &Inst) {
1983   uint64_t TSFlags = MII.get(Inst.getOpcode()).TSFlags;
1984 
1985   if ((getForcedEncodingSize() == 32 && (TSFlags & SIInstrFlags::VOP3)) ||
1986       (getForcedEncodingSize() == 64 && !(TSFlags & SIInstrFlags::VOP3)) ||
1987       (isForcedDPP() && !(TSFlags & SIInstrFlags::DPP)) ||
1988       (isForcedSDWA() && !(TSFlags & SIInstrFlags::SDWA)) )
1989     return Match_InvalidOperand;
1990 
1991   if ((TSFlags & SIInstrFlags::VOP3) &&
1992       (TSFlags & SIInstrFlags::VOPAsmPrefer32Bit) &&
1993       getForcedEncodingSize() != 64)
1994     return Match_PreferE32;
1995 
1996   if (Inst.getOpcode() == AMDGPU::V_MAC_F32_sdwa_vi ||
1997       Inst.getOpcode() == AMDGPU::V_MAC_F16_sdwa_vi) {
1998     // v_mac_f32/16 allow only dst_sel == DWORD;
1999     auto OpNum =
2000         AMDGPU::getNamedOperandIdx(Inst.getOpcode(), AMDGPU::OpName::dst_sel);
2001     const auto &Op = Inst.getOperand(OpNum);
2002     if (!Op.isImm() || Op.getImm() != AMDGPU::SDWA::SdwaSel::DWORD) {
2003       return Match_InvalidOperand;
2004     }
2005   }
2006 
2007   if ((TSFlags & SIInstrFlags::FLAT) && !hasFlatOffsets()) {
2008     // FIXME: Produces error without correct column reported.
2009     auto OpNum =
2010         AMDGPU::getNamedOperandIdx(Inst.getOpcode(), AMDGPU::OpName::offset);
2011     const auto &Op = Inst.getOperand(OpNum);
2012     if (Op.getImm() != 0)
2013       return Match_InvalidOperand;
2014   }
2015 
2016   return Match_Success;
2017 }
2018 
2019 // What asm variants we should check
2020 ArrayRef<unsigned> AMDGPUAsmParser::getMatchedVariants() const {
2021   if (getForcedEncodingSize() == 32) {
2022     static const unsigned Variants[] = {AMDGPUAsmVariants::DEFAULT};
2023     return makeArrayRef(Variants);
2024   }
2025 
2026   if (isForcedVOP3()) {
2027     static const unsigned Variants[] = {AMDGPUAsmVariants::VOP3};
2028     return makeArrayRef(Variants);
2029   }
2030 
2031   if (isForcedSDWA()) {
2032     static const unsigned Variants[] = {AMDGPUAsmVariants::SDWA,
2033                                         AMDGPUAsmVariants::SDWA9};
2034     return makeArrayRef(Variants);
2035   }
2036 
2037   if (isForcedDPP()) {
2038     static const unsigned Variants[] = {AMDGPUAsmVariants::DPP};
2039     return makeArrayRef(Variants);
2040   }
2041 
2042   static const unsigned Variants[] = {
2043     AMDGPUAsmVariants::DEFAULT, AMDGPUAsmVariants::VOP3,
2044     AMDGPUAsmVariants::SDWA, AMDGPUAsmVariants::SDWA9, AMDGPUAsmVariants::DPP
2045   };
2046 
2047   return makeArrayRef(Variants);
2048 }
2049 
2050 unsigned AMDGPUAsmParser::findImplicitSGPRReadInVOP(const MCInst &Inst) const {
2051   const MCInstrDesc &Desc = MII.get(Inst.getOpcode());
2052   const unsigned Num = Desc.getNumImplicitUses();
2053   for (unsigned i = 0; i < Num; ++i) {
2054     unsigned Reg = Desc.ImplicitUses[i];
2055     switch (Reg) {
2056     case AMDGPU::FLAT_SCR:
2057     case AMDGPU::VCC:
2058     case AMDGPU::M0:
2059       return Reg;
2060     default:
2061       break;
2062     }
2063   }
2064   return AMDGPU::NoRegister;
2065 }
2066 
2067 // NB: This code is correct only when used to check constant
2068 // bus limitations because GFX7 support no f16 inline constants.
2069 // Note that there are no cases when a GFX7 opcode violates
2070 // constant bus limitations due to the use of an f16 constant.
2071 bool AMDGPUAsmParser::isInlineConstant(const MCInst &Inst,
2072                                        unsigned OpIdx) const {
2073   const MCInstrDesc &Desc = MII.get(Inst.getOpcode());
2074 
2075   if (!AMDGPU::isSISrcOperand(Desc, OpIdx)) {
2076     return false;
2077   }
2078 
2079   const MCOperand &MO = Inst.getOperand(OpIdx);
2080 
2081   int64_t Val = MO.getImm();
2082   auto OpSize = AMDGPU::getOperandSize(Desc, OpIdx);
2083 
2084   switch (OpSize) { // expected operand size
2085   case 8:
2086     return AMDGPU::isInlinableLiteral64(Val, hasInv2PiInlineImm());
2087   case 4:
2088     return AMDGPU::isInlinableLiteral32(Val, hasInv2PiInlineImm());
2089   case 2: {
2090     const unsigned OperandType = Desc.OpInfo[OpIdx].OperandType;
2091     if (OperandType == AMDGPU::OPERAND_REG_INLINE_C_V2INT16 ||
2092         OperandType == AMDGPU::OPERAND_REG_INLINE_C_V2FP16) {
2093       return AMDGPU::isInlinableLiteralV216(Val, hasInv2PiInlineImm());
2094     } else {
2095       return AMDGPU::isInlinableLiteral16(Val, hasInv2PiInlineImm());
2096     }
2097   }
2098   default:
2099     llvm_unreachable("invalid operand size");
2100   }
2101 }
2102 
2103 bool AMDGPUAsmParser::usesConstantBus(const MCInst &Inst, unsigned OpIdx) {
2104   const MCOperand &MO = Inst.getOperand(OpIdx);
2105   if (MO.isImm()) {
2106     return !isInlineConstant(Inst, OpIdx);
2107   }
2108   return !MO.isReg() ||
2109          isSGPR(mc2PseudoReg(MO.getReg()), getContext().getRegisterInfo());
2110 }
2111 
2112 bool AMDGPUAsmParser::validateConstantBusLimitations(const MCInst &Inst) {
2113   const unsigned Opcode = Inst.getOpcode();
2114   const MCInstrDesc &Desc = MII.get(Opcode);
2115   unsigned ConstantBusUseCount = 0;
2116 
2117   if (Desc.TSFlags &
2118       (SIInstrFlags::VOPC |
2119        SIInstrFlags::VOP1 | SIInstrFlags::VOP2 |
2120        SIInstrFlags::VOP3 | SIInstrFlags::VOP3P |
2121        SIInstrFlags::SDWA)) {
2122     // Check special imm operands (used by madmk, etc)
2123     if (AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::imm) != -1) {
2124       ++ConstantBusUseCount;
2125     }
2126 
2127     unsigned SGPRUsed = findImplicitSGPRReadInVOP(Inst);
2128     if (SGPRUsed != AMDGPU::NoRegister) {
2129       ++ConstantBusUseCount;
2130     }
2131 
2132     const int Src0Idx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::src0);
2133     const int Src1Idx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::src1);
2134     const int Src2Idx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::src2);
2135 
2136     const int OpIndices[] = { Src0Idx, Src1Idx, Src2Idx };
2137 
2138     for (int OpIdx : OpIndices) {
2139       if (OpIdx == -1) break;
2140 
2141       const MCOperand &MO = Inst.getOperand(OpIdx);
2142       if (usesConstantBus(Inst, OpIdx)) {
2143         if (MO.isReg()) {
2144           const unsigned Reg = mc2PseudoReg(MO.getReg());
2145           // Pairs of registers with a partial intersections like these
2146           //   s0, s[0:1]
2147           //   flat_scratch_lo, flat_scratch
2148           //   flat_scratch_lo, flat_scratch_hi
2149           // are theoretically valid but they are disabled anyway.
2150           // Note that this code mimics SIInstrInfo::verifyInstruction
2151           if (Reg != SGPRUsed) {
2152             ++ConstantBusUseCount;
2153           }
2154           SGPRUsed = Reg;
2155         } else { // Expression or a literal
2156           ++ConstantBusUseCount;
2157         }
2158       }
2159     }
2160   }
2161 
2162   return ConstantBusUseCount <= 1;
2163 }
2164 
2165 bool AMDGPUAsmParser::validateEarlyClobberLimitations(const MCInst &Inst) {
2166   const unsigned Opcode = Inst.getOpcode();
2167   const MCInstrDesc &Desc = MII.get(Opcode);
2168 
2169   const int DstIdx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::vdst);
2170   if (DstIdx == -1 ||
2171       Desc.getOperandConstraint(DstIdx, MCOI::EARLY_CLOBBER) == -1) {
2172     return true;
2173   }
2174 
2175   const MCRegisterInfo *TRI = getContext().getRegisterInfo();
2176 
2177   const int Src0Idx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::src0);
2178   const int Src1Idx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::src1);
2179   const int Src2Idx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::src2);
2180 
2181   assert(DstIdx != -1);
2182   const MCOperand &Dst = Inst.getOperand(DstIdx);
2183   assert(Dst.isReg());
2184   const unsigned DstReg = mc2PseudoReg(Dst.getReg());
2185 
2186   const int SrcIndices[] = { Src0Idx, Src1Idx, Src2Idx };
2187 
2188   for (int SrcIdx : SrcIndices) {
2189     if (SrcIdx == -1) break;
2190     const MCOperand &Src = Inst.getOperand(SrcIdx);
2191     if (Src.isReg()) {
2192       const unsigned SrcReg = mc2PseudoReg(Src.getReg());
2193       if (isRegIntersect(DstReg, SrcReg, TRI)) {
2194         return false;
2195       }
2196     }
2197   }
2198 
2199   return true;
2200 }
2201 
2202 bool AMDGPUAsmParser::validateInstruction(const MCInst &Inst,
2203                                           const SMLoc &IDLoc) {
2204   if (!validateConstantBusLimitations(Inst)) {
2205     Error(IDLoc,
2206       "invalid operand (violates constant bus restrictions)");
2207     return false;
2208   }
2209   if (!validateEarlyClobberLimitations(Inst)) {
2210     Error(IDLoc,
2211       "destination must be different than all sources");
2212     return false;
2213   }
2214 
2215   return true;
2216 }
2217 
2218 bool AMDGPUAsmParser::MatchAndEmitInstruction(SMLoc IDLoc, unsigned &Opcode,
2219                                               OperandVector &Operands,
2220                                               MCStreamer &Out,
2221                                               uint64_t &ErrorInfo,
2222                                               bool MatchingInlineAsm) {
2223   MCInst Inst;
2224   unsigned Result = Match_Success;
2225   for (auto Variant : getMatchedVariants()) {
2226     uint64_t EI;
2227     auto R = MatchInstructionImpl(Operands, Inst, EI, MatchingInlineAsm,
2228                                   Variant);
2229     // We order match statuses from least to most specific. We use most specific
2230     // status as resulting
2231     // Match_MnemonicFail < Match_InvalidOperand < Match_MissingFeature < Match_PreferE32
2232     if ((R == Match_Success) ||
2233         (R == Match_PreferE32) ||
2234         (R == Match_MissingFeature && Result != Match_PreferE32) ||
2235         (R == Match_InvalidOperand && Result != Match_MissingFeature
2236                                    && Result != Match_PreferE32) ||
2237         (R == Match_MnemonicFail   && Result != Match_InvalidOperand
2238                                    && Result != Match_MissingFeature
2239                                    && Result != Match_PreferE32)) {
2240       Result = R;
2241       ErrorInfo = EI;
2242     }
2243     if (R == Match_Success)
2244       break;
2245   }
2246 
2247   switch (Result) {
2248   default: break;
2249   case Match_Success:
2250     if (!validateInstruction(Inst, IDLoc)) {
2251       return true;
2252     }
2253     Inst.setLoc(IDLoc);
2254     Out.EmitInstruction(Inst, getSTI());
2255     return false;
2256 
2257   case Match_MissingFeature:
2258     return Error(IDLoc, "instruction not supported on this GPU");
2259 
2260   case Match_MnemonicFail:
2261     return Error(IDLoc, "unrecognized instruction mnemonic");
2262 
2263   case Match_InvalidOperand: {
2264     SMLoc ErrorLoc = IDLoc;
2265     if (ErrorInfo != ~0ULL) {
2266       if (ErrorInfo >= Operands.size()) {
2267         return Error(IDLoc, "too few operands for instruction");
2268       }
2269       ErrorLoc = ((AMDGPUOperand &)*Operands[ErrorInfo]).getStartLoc();
2270       if (ErrorLoc == SMLoc())
2271         ErrorLoc = IDLoc;
2272     }
2273     return Error(ErrorLoc, "invalid operand for instruction");
2274   }
2275 
2276   case Match_PreferE32:
2277     return Error(IDLoc, "internal error: instruction without _e64 suffix "
2278                         "should be encoded as e32");
2279   }
2280   llvm_unreachable("Implement any new match types added!");
2281 }
2282 
2283 bool AMDGPUAsmParser::ParseAsAbsoluteExpression(uint32_t &Ret) {
2284   int64_t Tmp = -1;
2285   if (getLexer().isNot(AsmToken::Integer) && getLexer().isNot(AsmToken::Identifier)) {
2286     return true;
2287   }
2288   if (getParser().parseAbsoluteExpression(Tmp)) {
2289     return true;
2290   }
2291   Ret = static_cast<uint32_t>(Tmp);
2292   return false;
2293 }
2294 
2295 bool AMDGPUAsmParser::ParseDirectiveMajorMinor(uint32_t &Major,
2296                                                uint32_t &Minor) {
2297   if (ParseAsAbsoluteExpression(Major))
2298     return TokError("invalid major version");
2299 
2300   if (getLexer().isNot(AsmToken::Comma))
2301     return TokError("minor version number required, comma expected");
2302   Lex();
2303 
2304   if (ParseAsAbsoluteExpression(Minor))
2305     return TokError("invalid minor version");
2306 
2307   return false;
2308 }
2309 
2310 bool AMDGPUAsmParser::ParseDirectiveHSACodeObjectVersion() {
2311   uint32_t Major;
2312   uint32_t Minor;
2313 
2314   if (ParseDirectiveMajorMinor(Major, Minor))
2315     return true;
2316 
2317   getTargetStreamer().EmitDirectiveHSACodeObjectVersion(Major, Minor);
2318   return false;
2319 }
2320 
2321 bool AMDGPUAsmParser::ParseDirectiveHSACodeObjectISA() {
2322   uint32_t Major;
2323   uint32_t Minor;
2324   uint32_t Stepping;
2325   StringRef VendorName;
2326   StringRef ArchName;
2327 
2328   // If this directive has no arguments, then use the ISA version for the
2329   // targeted GPU.
2330   if (getLexer().is(AsmToken::EndOfStatement)) {
2331     AMDGPU::IsaInfo::IsaVersion ISA =
2332         AMDGPU::IsaInfo::getIsaVersion(getFeatureBits());
2333     getTargetStreamer().EmitDirectiveHSACodeObjectISA(ISA.Major, ISA.Minor,
2334                                                       ISA.Stepping,
2335                                                       "AMD", "AMDGPU");
2336     return false;
2337   }
2338 
2339   if (ParseDirectiveMajorMinor(Major, Minor))
2340     return true;
2341 
2342   if (getLexer().isNot(AsmToken::Comma))
2343     return TokError("stepping version number required, comma expected");
2344   Lex();
2345 
2346   if (ParseAsAbsoluteExpression(Stepping))
2347     return TokError("invalid stepping version");
2348 
2349   if (getLexer().isNot(AsmToken::Comma))
2350     return TokError("vendor name required, comma expected");
2351   Lex();
2352 
2353   if (getLexer().isNot(AsmToken::String))
2354     return TokError("invalid vendor name");
2355 
2356   VendorName = getLexer().getTok().getStringContents();
2357   Lex();
2358 
2359   if (getLexer().isNot(AsmToken::Comma))
2360     return TokError("arch name required, comma expected");
2361   Lex();
2362 
2363   if (getLexer().isNot(AsmToken::String))
2364     return TokError("invalid arch name");
2365 
2366   ArchName = getLexer().getTok().getStringContents();
2367   Lex();
2368 
2369   getTargetStreamer().EmitDirectiveHSACodeObjectISA(Major, Minor, Stepping,
2370                                                     VendorName, ArchName);
2371   return false;
2372 }
2373 
2374 bool AMDGPUAsmParser::ParseDirectiveCodeObjectMetadata() {
2375   std::string YamlString;
2376   raw_string_ostream YamlStream(YamlString);
2377 
2378   getLexer().setSkipSpace(false);
2379 
2380   bool FoundEnd = false;
2381   while (!getLexer().is(AsmToken::Eof)) {
2382     while (getLexer().is(AsmToken::Space)) {
2383       YamlStream << getLexer().getTok().getString();
2384       Lex();
2385     }
2386 
2387     if (getLexer().is(AsmToken::Identifier)) {
2388       StringRef ID = getLexer().getTok().getIdentifier();
2389       if (ID == AMDGPU::CodeObject::MetadataAssemblerDirectiveEnd) {
2390         Lex();
2391         FoundEnd = true;
2392         break;
2393       }
2394     }
2395 
2396     YamlStream << Parser.parseStringToEndOfStatement()
2397                << getContext().getAsmInfo()->getSeparatorString();
2398 
2399     Parser.eatToEndOfStatement();
2400   }
2401 
2402   getLexer().setSkipSpace(true);
2403 
2404   if (getLexer().is(AsmToken::Eof) && !FoundEnd) {
2405     return TokError(
2406         "expected directive .end_amdgpu_code_object_metadata not found");
2407   }
2408 
2409   YamlStream.flush();
2410 
2411   if (!getTargetStreamer().EmitCodeObjectMetadata(YamlString))
2412     return Error(getParser().getTok().getLoc(), "invalid code object metadata");
2413 
2414   return false;
2415 }
2416 
2417 bool AMDGPUAsmParser::ParseAMDKernelCodeTValue(StringRef ID,
2418                                                amd_kernel_code_t &Header) {
2419   SmallString<40> ErrStr;
2420   raw_svector_ostream Err(ErrStr);
2421   if (!parseAmdKernelCodeField(ID, getParser(), Header, Err)) {
2422     return TokError(Err.str());
2423   }
2424   Lex();
2425   return false;
2426 }
2427 
2428 bool AMDGPUAsmParser::ParseDirectiveAMDKernelCodeT() {
2429   amd_kernel_code_t Header;
2430   AMDGPU::initDefaultAMDKernelCodeT(Header, getFeatureBits());
2431 
2432   while (true) {
2433     // Lex EndOfStatement.  This is in a while loop, because lexing a comment
2434     // will set the current token to EndOfStatement.
2435     while(getLexer().is(AsmToken::EndOfStatement))
2436       Lex();
2437 
2438     if (getLexer().isNot(AsmToken::Identifier))
2439       return TokError("expected value identifier or .end_amd_kernel_code_t");
2440 
2441     StringRef ID = getLexer().getTok().getIdentifier();
2442     Lex();
2443 
2444     if (ID == ".end_amd_kernel_code_t")
2445       break;
2446 
2447     if (ParseAMDKernelCodeTValue(ID, Header))
2448       return true;
2449   }
2450 
2451   getTargetStreamer().EmitAMDKernelCodeT(Header);
2452 
2453   return false;
2454 }
2455 
2456 bool AMDGPUAsmParser::ParseDirectiveAMDGPUHsaKernel() {
2457   if (getLexer().isNot(AsmToken::Identifier))
2458     return TokError("expected symbol name");
2459 
2460   StringRef KernelName = Parser.getTok().getString();
2461 
2462   getTargetStreamer().EmitAMDGPUSymbolType(KernelName,
2463                                            ELF::STT_AMDGPU_HSA_KERNEL);
2464   Lex();
2465   KernelScope.initialize(getContext());
2466   return false;
2467 }
2468 
2469 bool AMDGPUAsmParser::ParseDirective(AsmToken DirectiveID) {
2470   StringRef IDVal = DirectiveID.getString();
2471 
2472   if (IDVal == ".hsa_code_object_version")
2473     return ParseDirectiveHSACodeObjectVersion();
2474 
2475   if (IDVal == ".hsa_code_object_isa")
2476     return ParseDirectiveHSACodeObjectISA();
2477 
2478   if (IDVal == AMDGPU::CodeObject::MetadataAssemblerDirectiveBegin)
2479     return ParseDirectiveCodeObjectMetadata();
2480 
2481   if (IDVal == ".amd_kernel_code_t")
2482     return ParseDirectiveAMDKernelCodeT();
2483 
2484   if (IDVal == ".amdgpu_hsa_kernel")
2485     return ParseDirectiveAMDGPUHsaKernel();
2486 
2487   return true;
2488 }
2489 
2490 bool AMDGPUAsmParser::subtargetHasRegister(const MCRegisterInfo &MRI,
2491                                            unsigned RegNo) const {
2492   if (isCI())
2493     return true;
2494 
2495   if (isSI()) {
2496     // No flat_scr
2497     switch (RegNo) {
2498     case AMDGPU::FLAT_SCR:
2499     case AMDGPU::FLAT_SCR_LO:
2500     case AMDGPU::FLAT_SCR_HI:
2501       return false;
2502     default:
2503       return true;
2504     }
2505   }
2506 
2507   // VI only has 102 SGPRs, so make sure we aren't trying to use the 2 more that
2508   // SI/CI have.
2509   for (MCRegAliasIterator R(AMDGPU::SGPR102_SGPR103, &MRI, true);
2510        R.isValid(); ++R) {
2511     if (*R == RegNo)
2512       return false;
2513   }
2514 
2515   return true;
2516 }
2517 
2518 OperandMatchResultTy
2519 AMDGPUAsmParser::parseOperand(OperandVector &Operands, StringRef Mnemonic) {
2520   // Try to parse with a custom parser
2521   OperandMatchResultTy ResTy = MatchOperandParserImpl(Operands, Mnemonic);
2522 
2523   // If we successfully parsed the operand or if there as an error parsing,
2524   // we are done.
2525   //
2526   // If we are parsing after we reach EndOfStatement then this means we
2527   // are appending default values to the Operands list.  This is only done
2528   // by custom parser, so we shouldn't continue on to the generic parsing.
2529   if (ResTy == MatchOperand_Success || ResTy == MatchOperand_ParseFail ||
2530       getLexer().is(AsmToken::EndOfStatement))
2531     return ResTy;
2532 
2533   ResTy = parseRegOrImm(Operands);
2534 
2535   if (ResTy == MatchOperand_Success)
2536     return ResTy;
2537 
2538   const auto &Tok = Parser.getTok();
2539   SMLoc S = Tok.getLoc();
2540 
2541   const MCExpr *Expr = nullptr;
2542   if (!Parser.parseExpression(Expr)) {
2543     Operands.push_back(AMDGPUOperand::CreateExpr(this, Expr, S));
2544     return MatchOperand_Success;
2545   }
2546 
2547   // Possibly this is an instruction flag like 'gds'.
2548   if (Tok.getKind() == AsmToken::Identifier) {
2549     Operands.push_back(AMDGPUOperand::CreateToken(this, Tok.getString(), S));
2550     Parser.Lex();
2551     return MatchOperand_Success;
2552   }
2553 
2554   return MatchOperand_NoMatch;
2555 }
2556 
2557 StringRef AMDGPUAsmParser::parseMnemonicSuffix(StringRef Name) {
2558   // Clear any forced encodings from the previous instruction.
2559   setForcedEncodingSize(0);
2560   setForcedDPP(false);
2561   setForcedSDWA(false);
2562 
2563   if (Name.endswith("_e64")) {
2564     setForcedEncodingSize(64);
2565     return Name.substr(0, Name.size() - 4);
2566   } else if (Name.endswith("_e32")) {
2567     setForcedEncodingSize(32);
2568     return Name.substr(0, Name.size() - 4);
2569   } else if (Name.endswith("_dpp")) {
2570     setForcedDPP(true);
2571     return Name.substr(0, Name.size() - 4);
2572   } else if (Name.endswith("_sdwa")) {
2573     setForcedSDWA(true);
2574     return Name.substr(0, Name.size() - 5);
2575   }
2576   return Name;
2577 }
2578 
2579 bool AMDGPUAsmParser::ParseInstruction(ParseInstructionInfo &Info,
2580                                        StringRef Name,
2581                                        SMLoc NameLoc, OperandVector &Operands) {
2582   // Add the instruction mnemonic
2583   Name = parseMnemonicSuffix(Name);
2584   Operands.push_back(AMDGPUOperand::CreateToken(this, Name, NameLoc));
2585 
2586   while (!getLexer().is(AsmToken::EndOfStatement)) {
2587     OperandMatchResultTy Res = parseOperand(Operands, Name);
2588 
2589     // Eat the comma or space if there is one.
2590     if (getLexer().is(AsmToken::Comma))
2591       Parser.Lex();
2592 
2593     switch (Res) {
2594       case MatchOperand_Success: break;
2595       case MatchOperand_ParseFail:
2596         Error(getLexer().getLoc(), "failed parsing operand.");
2597         while (!getLexer().is(AsmToken::EndOfStatement)) {
2598           Parser.Lex();
2599         }
2600         return true;
2601       case MatchOperand_NoMatch:
2602         Error(getLexer().getLoc(), "not a valid operand.");
2603         while (!getLexer().is(AsmToken::EndOfStatement)) {
2604           Parser.Lex();
2605         }
2606         return true;
2607     }
2608   }
2609 
2610   return false;
2611 }
2612 
2613 //===----------------------------------------------------------------------===//
2614 // Utility functions
2615 //===----------------------------------------------------------------------===//
2616 
2617 OperandMatchResultTy
2618 AMDGPUAsmParser::parseIntWithPrefix(const char *Prefix, int64_t &Int) {
2619   switch(getLexer().getKind()) {
2620     default: return MatchOperand_NoMatch;
2621     case AsmToken::Identifier: {
2622       StringRef Name = Parser.getTok().getString();
2623       if (!Name.equals(Prefix)) {
2624         return MatchOperand_NoMatch;
2625       }
2626 
2627       Parser.Lex();
2628       if (getLexer().isNot(AsmToken::Colon))
2629         return MatchOperand_ParseFail;
2630 
2631       Parser.Lex();
2632 
2633       bool IsMinus = false;
2634       if (getLexer().getKind() == AsmToken::Minus) {
2635         Parser.Lex();
2636         IsMinus = true;
2637       }
2638 
2639       if (getLexer().isNot(AsmToken::Integer))
2640         return MatchOperand_ParseFail;
2641 
2642       if (getParser().parseAbsoluteExpression(Int))
2643         return MatchOperand_ParseFail;
2644 
2645       if (IsMinus)
2646         Int = -Int;
2647       break;
2648     }
2649   }
2650   return MatchOperand_Success;
2651 }
2652 
2653 OperandMatchResultTy
2654 AMDGPUAsmParser::parseIntWithPrefix(const char *Prefix, OperandVector &Operands,
2655                                     AMDGPUOperand::ImmTy ImmTy,
2656                                     bool (*ConvertResult)(int64_t&)) {
2657   SMLoc S = Parser.getTok().getLoc();
2658   int64_t Value = 0;
2659 
2660   OperandMatchResultTy Res = parseIntWithPrefix(Prefix, Value);
2661   if (Res != MatchOperand_Success)
2662     return Res;
2663 
2664   if (ConvertResult && !ConvertResult(Value)) {
2665     return MatchOperand_ParseFail;
2666   }
2667 
2668   Operands.push_back(AMDGPUOperand::CreateImm(this, Value, S, ImmTy));
2669   return MatchOperand_Success;
2670 }
2671 
2672 OperandMatchResultTy AMDGPUAsmParser::parseOperandArrayWithPrefix(
2673   const char *Prefix,
2674   OperandVector &Operands,
2675   AMDGPUOperand::ImmTy ImmTy,
2676   bool (*ConvertResult)(int64_t&)) {
2677   StringRef Name = Parser.getTok().getString();
2678   if (!Name.equals(Prefix))
2679     return MatchOperand_NoMatch;
2680 
2681   Parser.Lex();
2682   if (getLexer().isNot(AsmToken::Colon))
2683     return MatchOperand_ParseFail;
2684 
2685   Parser.Lex();
2686   if (getLexer().isNot(AsmToken::LBrac))
2687     return MatchOperand_ParseFail;
2688   Parser.Lex();
2689 
2690   unsigned Val = 0;
2691   SMLoc S = Parser.getTok().getLoc();
2692 
2693   // FIXME: How to verify the number of elements matches the number of src
2694   // operands?
2695   for (int I = 0; I < 4; ++I) {
2696     if (I != 0) {
2697       if (getLexer().is(AsmToken::RBrac))
2698         break;
2699 
2700       if (getLexer().isNot(AsmToken::Comma))
2701         return MatchOperand_ParseFail;
2702       Parser.Lex();
2703     }
2704 
2705     if (getLexer().isNot(AsmToken::Integer))
2706       return MatchOperand_ParseFail;
2707 
2708     int64_t Op;
2709     if (getParser().parseAbsoluteExpression(Op))
2710       return MatchOperand_ParseFail;
2711 
2712     if (Op != 0 && Op != 1)
2713       return MatchOperand_ParseFail;
2714     Val |= (Op << I);
2715   }
2716 
2717   Parser.Lex();
2718   Operands.push_back(AMDGPUOperand::CreateImm(this, Val, S, ImmTy));
2719   return MatchOperand_Success;
2720 }
2721 
2722 OperandMatchResultTy
2723 AMDGPUAsmParser::parseNamedBit(const char *Name, OperandVector &Operands,
2724                                AMDGPUOperand::ImmTy ImmTy) {
2725   int64_t Bit = 0;
2726   SMLoc S = Parser.getTok().getLoc();
2727 
2728   // We are at the end of the statement, and this is a default argument, so
2729   // use a default value.
2730   if (getLexer().isNot(AsmToken::EndOfStatement)) {
2731     switch(getLexer().getKind()) {
2732       case AsmToken::Identifier: {
2733         StringRef Tok = Parser.getTok().getString();
2734         if (Tok == Name) {
2735           Bit = 1;
2736           Parser.Lex();
2737         } else if (Tok.startswith("no") && Tok.endswith(Name)) {
2738           Bit = 0;
2739           Parser.Lex();
2740         } else {
2741           return MatchOperand_NoMatch;
2742         }
2743         break;
2744       }
2745       default:
2746         return MatchOperand_NoMatch;
2747     }
2748   }
2749 
2750   Operands.push_back(AMDGPUOperand::CreateImm(this, Bit, S, ImmTy));
2751   return MatchOperand_Success;
2752 }
2753 
2754 static void addOptionalImmOperand(
2755   MCInst& Inst, const OperandVector& Operands,
2756   AMDGPUAsmParser::OptionalImmIndexMap& OptionalIdx,
2757   AMDGPUOperand::ImmTy ImmT,
2758   int64_t Default = 0) {
2759   auto i = OptionalIdx.find(ImmT);
2760   if (i != OptionalIdx.end()) {
2761     unsigned Idx = i->second;
2762     ((AMDGPUOperand &)*Operands[Idx]).addImmOperands(Inst, 1);
2763   } else {
2764     Inst.addOperand(MCOperand::createImm(Default));
2765   }
2766 }
2767 
2768 OperandMatchResultTy
2769 AMDGPUAsmParser::parseStringWithPrefix(StringRef Prefix, StringRef &Value) {
2770   if (getLexer().isNot(AsmToken::Identifier)) {
2771     return MatchOperand_NoMatch;
2772   }
2773   StringRef Tok = Parser.getTok().getString();
2774   if (Tok != Prefix) {
2775     return MatchOperand_NoMatch;
2776   }
2777 
2778   Parser.Lex();
2779   if (getLexer().isNot(AsmToken::Colon)) {
2780     return MatchOperand_ParseFail;
2781   }
2782 
2783   Parser.Lex();
2784   if (getLexer().isNot(AsmToken::Identifier)) {
2785     return MatchOperand_ParseFail;
2786   }
2787 
2788   Value = Parser.getTok().getString();
2789   return MatchOperand_Success;
2790 }
2791 
2792 //===----------------------------------------------------------------------===//
2793 // ds
2794 //===----------------------------------------------------------------------===//
2795 
2796 void AMDGPUAsmParser::cvtDSOffset01(MCInst &Inst,
2797                                     const OperandVector &Operands) {
2798   OptionalImmIndexMap OptionalIdx;
2799 
2800   for (unsigned i = 1, e = Operands.size(); i != e; ++i) {
2801     AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[i]);
2802 
2803     // Add the register arguments
2804     if (Op.isReg()) {
2805       Op.addRegOperands(Inst, 1);
2806       continue;
2807     }
2808 
2809     // Handle optional arguments
2810     OptionalIdx[Op.getImmTy()] = i;
2811   }
2812 
2813   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyOffset0);
2814   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyOffset1);
2815   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyGDS);
2816 
2817   Inst.addOperand(MCOperand::createReg(AMDGPU::M0)); // m0
2818 }
2819 
2820 void AMDGPUAsmParser::cvtDSImpl(MCInst &Inst, const OperandVector &Operands,
2821                                 bool IsGdsHardcoded) {
2822   OptionalImmIndexMap OptionalIdx;
2823 
2824   for (unsigned i = 1, e = Operands.size(); i != e; ++i) {
2825     AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[i]);
2826 
2827     // Add the register arguments
2828     if (Op.isReg()) {
2829       Op.addRegOperands(Inst, 1);
2830       continue;
2831     }
2832 
2833     if (Op.isToken() && Op.getToken() == "gds") {
2834       IsGdsHardcoded = true;
2835       continue;
2836     }
2837 
2838     // Handle optional arguments
2839     OptionalIdx[Op.getImmTy()] = i;
2840   }
2841 
2842   AMDGPUOperand::ImmTy OffsetType =
2843     (Inst.getOpcode() == AMDGPU::DS_SWIZZLE_B32_si ||
2844      Inst.getOpcode() == AMDGPU::DS_SWIZZLE_B32_vi) ? AMDGPUOperand::ImmTySwizzle :
2845                                                       AMDGPUOperand::ImmTyOffset;
2846 
2847   addOptionalImmOperand(Inst, Operands, OptionalIdx, OffsetType);
2848 
2849   if (!IsGdsHardcoded) {
2850     addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyGDS);
2851   }
2852   Inst.addOperand(MCOperand::createReg(AMDGPU::M0)); // m0
2853 }
2854 
2855 void AMDGPUAsmParser::cvtExp(MCInst &Inst, const OperandVector &Operands) {
2856   OptionalImmIndexMap OptionalIdx;
2857 
2858   unsigned OperandIdx[4];
2859   unsigned EnMask = 0;
2860   int SrcIdx = 0;
2861 
2862   for (unsigned i = 1, e = Operands.size(); i != e; ++i) {
2863     AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[i]);
2864 
2865     // Add the register arguments
2866     if (Op.isReg()) {
2867       assert(SrcIdx < 4);
2868       OperandIdx[SrcIdx] = Inst.size();
2869       Op.addRegOperands(Inst, 1);
2870       ++SrcIdx;
2871       continue;
2872     }
2873 
2874     if (Op.isOff()) {
2875       assert(SrcIdx < 4);
2876       OperandIdx[SrcIdx] = Inst.size();
2877       Inst.addOperand(MCOperand::createReg(AMDGPU::NoRegister));
2878       ++SrcIdx;
2879       continue;
2880     }
2881 
2882     if (Op.isImm() && Op.getImmTy() == AMDGPUOperand::ImmTyExpTgt) {
2883       Op.addImmOperands(Inst, 1);
2884       continue;
2885     }
2886 
2887     if (Op.isToken() && Op.getToken() == "done")
2888       continue;
2889 
2890     // Handle optional arguments
2891     OptionalIdx[Op.getImmTy()] = i;
2892   }
2893 
2894   assert(SrcIdx == 4);
2895 
2896   bool Compr = false;
2897   if (OptionalIdx.find(AMDGPUOperand::ImmTyExpCompr) != OptionalIdx.end()) {
2898     Compr = true;
2899     Inst.getOperand(OperandIdx[1]) = Inst.getOperand(OperandIdx[2]);
2900     Inst.getOperand(OperandIdx[2]).setReg(AMDGPU::NoRegister);
2901     Inst.getOperand(OperandIdx[3]).setReg(AMDGPU::NoRegister);
2902   }
2903 
2904   for (auto i = 0; i < SrcIdx; ++i) {
2905     if (Inst.getOperand(OperandIdx[i]).getReg() != AMDGPU::NoRegister) {
2906       EnMask |= Compr? (0x3 << i * 2) : (0x1 << i);
2907     }
2908   }
2909 
2910   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyExpVM);
2911   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyExpCompr);
2912 
2913   Inst.addOperand(MCOperand::createImm(EnMask));
2914 }
2915 
2916 //===----------------------------------------------------------------------===//
2917 // s_waitcnt
2918 //===----------------------------------------------------------------------===//
2919 
2920 static bool
2921 encodeCnt(
2922   const AMDGPU::IsaInfo::IsaVersion ISA,
2923   int64_t &IntVal,
2924   int64_t CntVal,
2925   bool Saturate,
2926   unsigned (*encode)(const IsaInfo::IsaVersion &Version, unsigned, unsigned),
2927   unsigned (*decode)(const IsaInfo::IsaVersion &Version, unsigned))
2928 {
2929   bool Failed = false;
2930 
2931   IntVal = encode(ISA, IntVal, CntVal);
2932   if (CntVal != decode(ISA, IntVal)) {
2933     if (Saturate) {
2934       IntVal = encode(ISA, IntVal, -1);
2935     } else {
2936       Failed = true;
2937     }
2938   }
2939   return Failed;
2940 }
2941 
2942 bool AMDGPUAsmParser::parseCnt(int64_t &IntVal) {
2943   StringRef CntName = Parser.getTok().getString();
2944   int64_t CntVal;
2945 
2946   Parser.Lex();
2947   if (getLexer().isNot(AsmToken::LParen))
2948     return true;
2949 
2950   Parser.Lex();
2951   if (getLexer().isNot(AsmToken::Integer))
2952     return true;
2953 
2954   SMLoc ValLoc = Parser.getTok().getLoc();
2955   if (getParser().parseAbsoluteExpression(CntVal))
2956     return true;
2957 
2958   AMDGPU::IsaInfo::IsaVersion ISA =
2959       AMDGPU::IsaInfo::getIsaVersion(getFeatureBits());
2960 
2961   bool Failed = true;
2962   bool Sat = CntName.endswith("_sat");
2963 
2964   if (CntName == "vmcnt" || CntName == "vmcnt_sat") {
2965     Failed = encodeCnt(ISA, IntVal, CntVal, Sat, encodeVmcnt, decodeVmcnt);
2966   } else if (CntName == "expcnt" || CntName == "expcnt_sat") {
2967     Failed = encodeCnt(ISA, IntVal, CntVal, Sat, encodeExpcnt, decodeExpcnt);
2968   } else if (CntName == "lgkmcnt" || CntName == "lgkmcnt_sat") {
2969     Failed = encodeCnt(ISA, IntVal, CntVal, Sat, encodeLgkmcnt, decodeLgkmcnt);
2970   }
2971 
2972   if (Failed) {
2973     Error(ValLoc, "too large value for " + CntName);
2974     return true;
2975   }
2976 
2977   if (getLexer().isNot(AsmToken::RParen)) {
2978     return true;
2979   }
2980 
2981   Parser.Lex();
2982   if (getLexer().is(AsmToken::Amp) || getLexer().is(AsmToken::Comma)) {
2983     const AsmToken NextToken = getLexer().peekTok();
2984     if (NextToken.is(AsmToken::Identifier)) {
2985       Parser.Lex();
2986     }
2987   }
2988 
2989   return false;
2990 }
2991 
2992 OperandMatchResultTy
2993 AMDGPUAsmParser::parseSWaitCntOps(OperandVector &Operands) {
2994   AMDGPU::IsaInfo::IsaVersion ISA =
2995       AMDGPU::IsaInfo::getIsaVersion(getFeatureBits());
2996   int64_t Waitcnt = getWaitcntBitMask(ISA);
2997   SMLoc S = Parser.getTok().getLoc();
2998 
2999   switch(getLexer().getKind()) {
3000     default: return MatchOperand_ParseFail;
3001     case AsmToken::Integer:
3002       // The operand can be an integer value.
3003       if (getParser().parseAbsoluteExpression(Waitcnt))
3004         return MatchOperand_ParseFail;
3005       break;
3006 
3007     case AsmToken::Identifier:
3008       do {
3009         if (parseCnt(Waitcnt))
3010           return MatchOperand_ParseFail;
3011       } while(getLexer().isNot(AsmToken::EndOfStatement));
3012       break;
3013   }
3014   Operands.push_back(AMDGPUOperand::CreateImm(this, Waitcnt, S));
3015   return MatchOperand_Success;
3016 }
3017 
3018 bool AMDGPUAsmParser::parseHwregConstruct(OperandInfoTy &HwReg, int64_t &Offset,
3019                                           int64_t &Width) {
3020   using namespace llvm::AMDGPU::Hwreg;
3021 
3022   if (Parser.getTok().getString() != "hwreg")
3023     return true;
3024   Parser.Lex();
3025 
3026   if (getLexer().isNot(AsmToken::LParen))
3027     return true;
3028   Parser.Lex();
3029 
3030   if (getLexer().is(AsmToken::Identifier)) {
3031     HwReg.IsSymbolic = true;
3032     HwReg.Id = ID_UNKNOWN_;
3033     const StringRef tok = Parser.getTok().getString();
3034     for (int i = ID_SYMBOLIC_FIRST_; i < ID_SYMBOLIC_LAST_; ++i) {
3035       if (tok == IdSymbolic[i]) {
3036         HwReg.Id = i;
3037         break;
3038       }
3039     }
3040     Parser.Lex();
3041   } else {
3042     HwReg.IsSymbolic = false;
3043     if (getLexer().isNot(AsmToken::Integer))
3044       return true;
3045     if (getParser().parseAbsoluteExpression(HwReg.Id))
3046       return true;
3047   }
3048 
3049   if (getLexer().is(AsmToken::RParen)) {
3050     Parser.Lex();
3051     return false;
3052   }
3053 
3054   // optional params
3055   if (getLexer().isNot(AsmToken::Comma))
3056     return true;
3057   Parser.Lex();
3058 
3059   if (getLexer().isNot(AsmToken::Integer))
3060     return true;
3061   if (getParser().parseAbsoluteExpression(Offset))
3062     return true;
3063 
3064   if (getLexer().isNot(AsmToken::Comma))
3065     return true;
3066   Parser.Lex();
3067 
3068   if (getLexer().isNot(AsmToken::Integer))
3069     return true;
3070   if (getParser().parseAbsoluteExpression(Width))
3071     return true;
3072 
3073   if (getLexer().isNot(AsmToken::RParen))
3074     return true;
3075   Parser.Lex();
3076 
3077   return false;
3078 }
3079 
3080 OperandMatchResultTy AMDGPUAsmParser::parseHwreg(OperandVector &Operands) {
3081   using namespace llvm::AMDGPU::Hwreg;
3082 
3083   int64_t Imm16Val = 0;
3084   SMLoc S = Parser.getTok().getLoc();
3085 
3086   switch(getLexer().getKind()) {
3087     default: return MatchOperand_NoMatch;
3088     case AsmToken::Integer:
3089       // The operand can be an integer value.
3090       if (getParser().parseAbsoluteExpression(Imm16Val))
3091         return MatchOperand_NoMatch;
3092       if (Imm16Val < 0 || !isUInt<16>(Imm16Val)) {
3093         Error(S, "invalid immediate: only 16-bit values are legal");
3094         // Do not return error code, but create an imm operand anyway and proceed
3095         // to the next operand, if any. That avoids unneccessary error messages.
3096       }
3097       break;
3098 
3099     case AsmToken::Identifier: {
3100         OperandInfoTy HwReg(ID_UNKNOWN_);
3101         int64_t Offset = OFFSET_DEFAULT_;
3102         int64_t Width = WIDTH_M1_DEFAULT_ + 1;
3103         if (parseHwregConstruct(HwReg, Offset, Width))
3104           return MatchOperand_ParseFail;
3105         if (HwReg.Id < 0 || !isUInt<ID_WIDTH_>(HwReg.Id)) {
3106           if (HwReg.IsSymbolic)
3107             Error(S, "invalid symbolic name of hardware register");
3108           else
3109             Error(S, "invalid code of hardware register: only 6-bit values are legal");
3110         }
3111         if (Offset < 0 || !isUInt<OFFSET_WIDTH_>(Offset))
3112           Error(S, "invalid bit offset: only 5-bit values are legal");
3113         if ((Width-1) < 0 || !isUInt<WIDTH_M1_WIDTH_>(Width-1))
3114           Error(S, "invalid bitfield width: only values from 1 to 32 are legal");
3115         Imm16Val = (HwReg.Id << ID_SHIFT_) | (Offset << OFFSET_SHIFT_) | ((Width-1) << WIDTH_M1_SHIFT_);
3116       }
3117       break;
3118   }
3119   Operands.push_back(AMDGPUOperand::CreateImm(this, Imm16Val, S, AMDGPUOperand::ImmTyHwreg));
3120   return MatchOperand_Success;
3121 }
3122 
3123 bool AMDGPUOperand::isSWaitCnt() const {
3124   return isImm();
3125 }
3126 
3127 bool AMDGPUOperand::isHwreg() const {
3128   return isImmTy(ImmTyHwreg);
3129 }
3130 
3131 bool AMDGPUAsmParser::parseSendMsgConstruct(OperandInfoTy &Msg, OperandInfoTy &Operation, int64_t &StreamId) {
3132   using namespace llvm::AMDGPU::SendMsg;
3133 
3134   if (Parser.getTok().getString() != "sendmsg")
3135     return true;
3136   Parser.Lex();
3137 
3138   if (getLexer().isNot(AsmToken::LParen))
3139     return true;
3140   Parser.Lex();
3141 
3142   if (getLexer().is(AsmToken::Identifier)) {
3143     Msg.IsSymbolic = true;
3144     Msg.Id = ID_UNKNOWN_;
3145     const std::string tok = Parser.getTok().getString();
3146     for (int i = ID_GAPS_FIRST_; i < ID_GAPS_LAST_; ++i) {
3147       switch(i) {
3148         default: continue; // Omit gaps.
3149         case ID_INTERRUPT: case ID_GS: case ID_GS_DONE:  case ID_SYSMSG: break;
3150       }
3151       if (tok == IdSymbolic[i]) {
3152         Msg.Id = i;
3153         break;
3154       }
3155     }
3156     Parser.Lex();
3157   } else {
3158     Msg.IsSymbolic = false;
3159     if (getLexer().isNot(AsmToken::Integer))
3160       return true;
3161     if (getParser().parseAbsoluteExpression(Msg.Id))
3162       return true;
3163     if (getLexer().is(AsmToken::Integer))
3164       if (getParser().parseAbsoluteExpression(Msg.Id))
3165         Msg.Id = ID_UNKNOWN_;
3166   }
3167   if (Msg.Id == ID_UNKNOWN_) // Don't know how to parse the rest.
3168     return false;
3169 
3170   if (!(Msg.Id == ID_GS || Msg.Id == ID_GS_DONE || Msg.Id == ID_SYSMSG)) {
3171     if (getLexer().isNot(AsmToken::RParen))
3172       return true;
3173     Parser.Lex();
3174     return false;
3175   }
3176 
3177   if (getLexer().isNot(AsmToken::Comma))
3178     return true;
3179   Parser.Lex();
3180 
3181   assert(Msg.Id == ID_GS || Msg.Id == ID_GS_DONE || Msg.Id == ID_SYSMSG);
3182   Operation.Id = ID_UNKNOWN_;
3183   if (getLexer().is(AsmToken::Identifier)) {
3184     Operation.IsSymbolic = true;
3185     const char* const *S = (Msg.Id == ID_SYSMSG) ? OpSysSymbolic : OpGsSymbolic;
3186     const int F = (Msg.Id == ID_SYSMSG) ? OP_SYS_FIRST_ : OP_GS_FIRST_;
3187     const int L = (Msg.Id == ID_SYSMSG) ? OP_SYS_LAST_ : OP_GS_LAST_;
3188     const StringRef Tok = Parser.getTok().getString();
3189     for (int i = F; i < L; ++i) {
3190       if (Tok == S[i]) {
3191         Operation.Id = i;
3192         break;
3193       }
3194     }
3195     Parser.Lex();
3196   } else {
3197     Operation.IsSymbolic = false;
3198     if (getLexer().isNot(AsmToken::Integer))
3199       return true;
3200     if (getParser().parseAbsoluteExpression(Operation.Id))
3201       return true;
3202   }
3203 
3204   if ((Msg.Id == ID_GS || Msg.Id == ID_GS_DONE) && Operation.Id != OP_GS_NOP) {
3205     // Stream id is optional.
3206     if (getLexer().is(AsmToken::RParen)) {
3207       Parser.Lex();
3208       return false;
3209     }
3210 
3211     if (getLexer().isNot(AsmToken::Comma))
3212       return true;
3213     Parser.Lex();
3214 
3215     if (getLexer().isNot(AsmToken::Integer))
3216       return true;
3217     if (getParser().parseAbsoluteExpression(StreamId))
3218       return true;
3219   }
3220 
3221   if (getLexer().isNot(AsmToken::RParen))
3222     return true;
3223   Parser.Lex();
3224   return false;
3225 }
3226 
3227 OperandMatchResultTy AMDGPUAsmParser::parseInterpSlot(OperandVector &Operands) {
3228   if (getLexer().getKind() != AsmToken::Identifier)
3229     return MatchOperand_NoMatch;
3230 
3231   StringRef Str = Parser.getTok().getString();
3232   int Slot = StringSwitch<int>(Str)
3233     .Case("p10", 0)
3234     .Case("p20", 1)
3235     .Case("p0", 2)
3236     .Default(-1);
3237 
3238   SMLoc S = Parser.getTok().getLoc();
3239   if (Slot == -1)
3240     return MatchOperand_ParseFail;
3241 
3242   Parser.Lex();
3243   Operands.push_back(AMDGPUOperand::CreateImm(this, Slot, S,
3244                                               AMDGPUOperand::ImmTyInterpSlot));
3245   return MatchOperand_Success;
3246 }
3247 
3248 OperandMatchResultTy AMDGPUAsmParser::parseInterpAttr(OperandVector &Operands) {
3249   if (getLexer().getKind() != AsmToken::Identifier)
3250     return MatchOperand_NoMatch;
3251 
3252   StringRef Str = Parser.getTok().getString();
3253   if (!Str.startswith("attr"))
3254     return MatchOperand_NoMatch;
3255 
3256   StringRef Chan = Str.take_back(2);
3257   int AttrChan = StringSwitch<int>(Chan)
3258     .Case(".x", 0)
3259     .Case(".y", 1)
3260     .Case(".z", 2)
3261     .Case(".w", 3)
3262     .Default(-1);
3263   if (AttrChan == -1)
3264     return MatchOperand_ParseFail;
3265 
3266   Str = Str.drop_back(2).drop_front(4);
3267 
3268   uint8_t Attr;
3269   if (Str.getAsInteger(10, Attr))
3270     return MatchOperand_ParseFail;
3271 
3272   SMLoc S = Parser.getTok().getLoc();
3273   Parser.Lex();
3274   if (Attr > 63) {
3275     Error(S, "out of bounds attr");
3276     return MatchOperand_Success;
3277   }
3278 
3279   SMLoc SChan = SMLoc::getFromPointer(Chan.data());
3280 
3281   Operands.push_back(AMDGPUOperand::CreateImm(this, Attr, S,
3282                                               AMDGPUOperand::ImmTyInterpAttr));
3283   Operands.push_back(AMDGPUOperand::CreateImm(this, AttrChan, SChan,
3284                                               AMDGPUOperand::ImmTyAttrChan));
3285   return MatchOperand_Success;
3286 }
3287 
3288 void AMDGPUAsmParser::errorExpTgt() {
3289   Error(Parser.getTok().getLoc(), "invalid exp target");
3290 }
3291 
3292 OperandMatchResultTy AMDGPUAsmParser::parseExpTgtImpl(StringRef Str,
3293                                                       uint8_t &Val) {
3294   if (Str == "null") {
3295     Val = 9;
3296     return MatchOperand_Success;
3297   }
3298 
3299   if (Str.startswith("mrt")) {
3300     Str = Str.drop_front(3);
3301     if (Str == "z") { // == mrtz
3302       Val = 8;
3303       return MatchOperand_Success;
3304     }
3305 
3306     if (Str.getAsInteger(10, Val))
3307       return MatchOperand_ParseFail;
3308 
3309     if (Val > 7)
3310       errorExpTgt();
3311 
3312     return MatchOperand_Success;
3313   }
3314 
3315   if (Str.startswith("pos")) {
3316     Str = Str.drop_front(3);
3317     if (Str.getAsInteger(10, Val))
3318       return MatchOperand_ParseFail;
3319 
3320     if (Val > 3)
3321       errorExpTgt();
3322 
3323     Val += 12;
3324     return MatchOperand_Success;
3325   }
3326 
3327   if (Str.startswith("param")) {
3328     Str = Str.drop_front(5);
3329     if (Str.getAsInteger(10, Val))
3330       return MatchOperand_ParseFail;
3331 
3332     if (Val >= 32)
3333       errorExpTgt();
3334 
3335     Val += 32;
3336     return MatchOperand_Success;
3337   }
3338 
3339   if (Str.startswith("invalid_target_")) {
3340     Str = Str.drop_front(15);
3341     if (Str.getAsInteger(10, Val))
3342       return MatchOperand_ParseFail;
3343 
3344     errorExpTgt();
3345     return MatchOperand_Success;
3346   }
3347 
3348   return MatchOperand_NoMatch;
3349 }
3350 
3351 OperandMatchResultTy AMDGPUAsmParser::parseExpTgt(OperandVector &Operands) {
3352   uint8_t Val;
3353   StringRef Str = Parser.getTok().getString();
3354 
3355   auto Res = parseExpTgtImpl(Str, Val);
3356   if (Res != MatchOperand_Success)
3357     return Res;
3358 
3359   SMLoc S = Parser.getTok().getLoc();
3360   Parser.Lex();
3361 
3362   Operands.push_back(AMDGPUOperand::CreateImm(this, Val, S,
3363                                               AMDGPUOperand::ImmTyExpTgt));
3364   return MatchOperand_Success;
3365 }
3366 
3367 OperandMatchResultTy
3368 AMDGPUAsmParser::parseSendMsgOp(OperandVector &Operands) {
3369   using namespace llvm::AMDGPU::SendMsg;
3370 
3371   int64_t Imm16Val = 0;
3372   SMLoc S = Parser.getTok().getLoc();
3373 
3374   switch(getLexer().getKind()) {
3375   default:
3376     return MatchOperand_NoMatch;
3377   case AsmToken::Integer:
3378     // The operand can be an integer value.
3379     if (getParser().parseAbsoluteExpression(Imm16Val))
3380       return MatchOperand_NoMatch;
3381     if (Imm16Val < 0 || !isUInt<16>(Imm16Val)) {
3382       Error(S, "invalid immediate: only 16-bit values are legal");
3383       // Do not return error code, but create an imm operand anyway and proceed
3384       // to the next operand, if any. That avoids unneccessary error messages.
3385     }
3386     break;
3387   case AsmToken::Identifier: {
3388       OperandInfoTy Msg(ID_UNKNOWN_);
3389       OperandInfoTy Operation(OP_UNKNOWN_);
3390       int64_t StreamId = STREAM_ID_DEFAULT_;
3391       if (parseSendMsgConstruct(Msg, Operation, StreamId))
3392         return MatchOperand_ParseFail;
3393       do {
3394         // Validate and encode message ID.
3395         if (! ((ID_INTERRUPT <= Msg.Id && Msg.Id <= ID_GS_DONE)
3396                 || Msg.Id == ID_SYSMSG)) {
3397           if (Msg.IsSymbolic)
3398             Error(S, "invalid/unsupported symbolic name of message");
3399           else
3400             Error(S, "invalid/unsupported code of message");
3401           break;
3402         }
3403         Imm16Val = (Msg.Id << ID_SHIFT_);
3404         // Validate and encode operation ID.
3405         if (Msg.Id == ID_GS || Msg.Id == ID_GS_DONE) {
3406           if (! (OP_GS_FIRST_ <= Operation.Id && Operation.Id < OP_GS_LAST_)) {
3407             if (Operation.IsSymbolic)
3408               Error(S, "invalid symbolic name of GS_OP");
3409             else
3410               Error(S, "invalid code of GS_OP: only 2-bit values are legal");
3411             break;
3412           }
3413           if (Operation.Id == OP_GS_NOP
3414               && Msg.Id != ID_GS_DONE) {
3415             Error(S, "invalid GS_OP: NOP is for GS_DONE only");
3416             break;
3417           }
3418           Imm16Val |= (Operation.Id << OP_SHIFT_);
3419         }
3420         if (Msg.Id == ID_SYSMSG) {
3421           if (! (OP_SYS_FIRST_ <= Operation.Id && Operation.Id < OP_SYS_LAST_)) {
3422             if (Operation.IsSymbolic)
3423               Error(S, "invalid/unsupported symbolic name of SYSMSG_OP");
3424             else
3425               Error(S, "invalid/unsupported code of SYSMSG_OP");
3426             break;
3427           }
3428           Imm16Val |= (Operation.Id << OP_SHIFT_);
3429         }
3430         // Validate and encode stream ID.
3431         if ((Msg.Id == ID_GS || Msg.Id == ID_GS_DONE) && Operation.Id != OP_GS_NOP) {
3432           if (! (STREAM_ID_FIRST_ <= StreamId && StreamId < STREAM_ID_LAST_)) {
3433             Error(S, "invalid stream id: only 2-bit values are legal");
3434             break;
3435           }
3436           Imm16Val |= (StreamId << STREAM_ID_SHIFT_);
3437         }
3438       } while (false);
3439     }
3440     break;
3441   }
3442   Operands.push_back(AMDGPUOperand::CreateImm(this, Imm16Val, S, AMDGPUOperand::ImmTySendMsg));
3443   return MatchOperand_Success;
3444 }
3445 
3446 bool AMDGPUOperand::isSendMsg() const {
3447   return isImmTy(ImmTySendMsg);
3448 }
3449 
3450 //===----------------------------------------------------------------------===//
3451 // parser helpers
3452 //===----------------------------------------------------------------------===//
3453 
3454 bool
3455 AMDGPUAsmParser::trySkipId(const StringRef Id) {
3456   if (getLexer().getKind() == AsmToken::Identifier &&
3457       Parser.getTok().getString() == Id) {
3458     Parser.Lex();
3459     return true;
3460   }
3461   return false;
3462 }
3463 
3464 bool
3465 AMDGPUAsmParser::trySkipToken(const AsmToken::TokenKind Kind) {
3466   if (getLexer().getKind() == Kind) {
3467     Parser.Lex();
3468     return true;
3469   }
3470   return false;
3471 }
3472 
3473 bool
3474 AMDGPUAsmParser::skipToken(const AsmToken::TokenKind Kind,
3475                            const StringRef ErrMsg) {
3476   if (!trySkipToken(Kind)) {
3477     Error(Parser.getTok().getLoc(), ErrMsg);
3478     return false;
3479   }
3480   return true;
3481 }
3482 
3483 bool
3484 AMDGPUAsmParser::parseExpr(int64_t &Imm) {
3485   return !getParser().parseAbsoluteExpression(Imm);
3486 }
3487 
3488 bool
3489 AMDGPUAsmParser::parseString(StringRef &Val, const StringRef ErrMsg) {
3490   SMLoc S = Parser.getTok().getLoc();
3491   if (getLexer().getKind() == AsmToken::String) {
3492     Val = Parser.getTok().getStringContents();
3493     Parser.Lex();
3494     return true;
3495   } else {
3496     Error(S, ErrMsg);
3497     return false;
3498   }
3499 }
3500 
3501 //===----------------------------------------------------------------------===//
3502 // swizzle
3503 //===----------------------------------------------------------------------===//
3504 
3505 LLVM_READNONE
3506 static unsigned
3507 encodeBitmaskPerm(const unsigned AndMask,
3508                   const unsigned OrMask,
3509                   const unsigned XorMask) {
3510   using namespace llvm::AMDGPU::Swizzle;
3511 
3512   return BITMASK_PERM_ENC |
3513          (AndMask << BITMASK_AND_SHIFT) |
3514          (OrMask  << BITMASK_OR_SHIFT)  |
3515          (XorMask << BITMASK_XOR_SHIFT);
3516 }
3517 
3518 bool
3519 AMDGPUAsmParser::parseSwizzleOperands(const unsigned OpNum, int64_t* Op,
3520                                       const unsigned MinVal,
3521                                       const unsigned MaxVal,
3522                                       const StringRef ErrMsg) {
3523   for (unsigned i = 0; i < OpNum; ++i) {
3524     if (!skipToken(AsmToken::Comma, "expected a comma")){
3525       return false;
3526     }
3527     SMLoc ExprLoc = Parser.getTok().getLoc();
3528     if (!parseExpr(Op[i])) {
3529       return false;
3530     }
3531     if (Op[i] < MinVal || Op[i] > MaxVal) {
3532       Error(ExprLoc, ErrMsg);
3533       return false;
3534     }
3535   }
3536 
3537   return true;
3538 }
3539 
3540 bool
3541 AMDGPUAsmParser::parseSwizzleQuadPerm(int64_t &Imm) {
3542   using namespace llvm::AMDGPU::Swizzle;
3543 
3544   int64_t Lane[LANE_NUM];
3545   if (parseSwizzleOperands(LANE_NUM, Lane, 0, LANE_MAX,
3546                            "expected a 2-bit lane id")) {
3547     Imm = QUAD_PERM_ENC;
3548     for (auto i = 0; i < LANE_NUM; ++i) {
3549       Imm |= Lane[i] << (LANE_SHIFT * i);
3550     }
3551     return true;
3552   }
3553   return false;
3554 }
3555 
3556 bool
3557 AMDGPUAsmParser::parseSwizzleBroadcast(int64_t &Imm) {
3558   using namespace llvm::AMDGPU::Swizzle;
3559 
3560   SMLoc S = Parser.getTok().getLoc();
3561   int64_t GroupSize;
3562   int64_t LaneIdx;
3563 
3564   if (!parseSwizzleOperands(1, &GroupSize,
3565                             2, 32,
3566                             "group size must be in the interval [2,32]")) {
3567     return false;
3568   }
3569   if (!isPowerOf2_64(GroupSize)) {
3570     Error(S, "group size must be a power of two");
3571     return false;
3572   }
3573   if (parseSwizzleOperands(1, &LaneIdx,
3574                            0, GroupSize - 1,
3575                            "lane id must be in the interval [0,group size - 1]")) {
3576     Imm = encodeBitmaskPerm(BITMASK_MAX - GroupSize + 1, LaneIdx, 0);
3577     return true;
3578   }
3579   return false;
3580 }
3581 
3582 bool
3583 AMDGPUAsmParser::parseSwizzleReverse(int64_t &Imm) {
3584   using namespace llvm::AMDGPU::Swizzle;
3585 
3586   SMLoc S = Parser.getTok().getLoc();
3587   int64_t GroupSize;
3588 
3589   if (!parseSwizzleOperands(1, &GroupSize,
3590       2, 32, "group size must be in the interval [2,32]")) {
3591     return false;
3592   }
3593   if (!isPowerOf2_64(GroupSize)) {
3594     Error(S, "group size must be a power of two");
3595     return false;
3596   }
3597 
3598   Imm = encodeBitmaskPerm(BITMASK_MAX, 0, GroupSize - 1);
3599   return true;
3600 }
3601 
3602 bool
3603 AMDGPUAsmParser::parseSwizzleSwap(int64_t &Imm) {
3604   using namespace llvm::AMDGPU::Swizzle;
3605 
3606   SMLoc S = Parser.getTok().getLoc();
3607   int64_t GroupSize;
3608 
3609   if (!parseSwizzleOperands(1, &GroupSize,
3610       1, 16, "group size must be in the interval [1,16]")) {
3611     return false;
3612   }
3613   if (!isPowerOf2_64(GroupSize)) {
3614     Error(S, "group size must be a power of two");
3615     return false;
3616   }
3617 
3618   Imm = encodeBitmaskPerm(BITMASK_MAX, 0, GroupSize);
3619   return true;
3620 }
3621 
3622 bool
3623 AMDGPUAsmParser::parseSwizzleBitmaskPerm(int64_t &Imm) {
3624   using namespace llvm::AMDGPU::Swizzle;
3625 
3626   if (!skipToken(AsmToken::Comma, "expected a comma")) {
3627     return false;
3628   }
3629 
3630   StringRef Ctl;
3631   SMLoc StrLoc = Parser.getTok().getLoc();
3632   if (!parseString(Ctl)) {
3633     return false;
3634   }
3635   if (Ctl.size() != BITMASK_WIDTH) {
3636     Error(StrLoc, "expected a 5-character mask");
3637     return false;
3638   }
3639 
3640   unsigned AndMask = 0;
3641   unsigned OrMask = 0;
3642   unsigned XorMask = 0;
3643 
3644   for (size_t i = 0; i < Ctl.size(); ++i) {
3645     unsigned Mask = 1 << (BITMASK_WIDTH - 1 - i);
3646     switch(Ctl[i]) {
3647     default:
3648       Error(StrLoc, "invalid mask");
3649       return false;
3650     case '0':
3651       break;
3652     case '1':
3653       OrMask |= Mask;
3654       break;
3655     case 'p':
3656       AndMask |= Mask;
3657       break;
3658     case 'i':
3659       AndMask |= Mask;
3660       XorMask |= Mask;
3661       break;
3662     }
3663   }
3664 
3665   Imm = encodeBitmaskPerm(AndMask, OrMask, XorMask);
3666   return true;
3667 }
3668 
3669 bool
3670 AMDGPUAsmParser::parseSwizzleOffset(int64_t &Imm) {
3671 
3672   SMLoc OffsetLoc = Parser.getTok().getLoc();
3673 
3674   if (!parseExpr(Imm)) {
3675     return false;
3676   }
3677   if (!isUInt<16>(Imm)) {
3678     Error(OffsetLoc, "expected a 16-bit offset");
3679     return false;
3680   }
3681   return true;
3682 }
3683 
3684 bool
3685 AMDGPUAsmParser::parseSwizzleMacro(int64_t &Imm) {
3686   using namespace llvm::AMDGPU::Swizzle;
3687 
3688   if (skipToken(AsmToken::LParen, "expected a left parentheses")) {
3689 
3690     SMLoc ModeLoc = Parser.getTok().getLoc();
3691     bool Ok = false;
3692 
3693     if (trySkipId(IdSymbolic[ID_QUAD_PERM])) {
3694       Ok = parseSwizzleQuadPerm(Imm);
3695     } else if (trySkipId(IdSymbolic[ID_BITMASK_PERM])) {
3696       Ok = parseSwizzleBitmaskPerm(Imm);
3697     } else if (trySkipId(IdSymbolic[ID_BROADCAST])) {
3698       Ok = parseSwizzleBroadcast(Imm);
3699     } else if (trySkipId(IdSymbolic[ID_SWAP])) {
3700       Ok = parseSwizzleSwap(Imm);
3701     } else if (trySkipId(IdSymbolic[ID_REVERSE])) {
3702       Ok = parseSwizzleReverse(Imm);
3703     } else {
3704       Error(ModeLoc, "expected a swizzle mode");
3705     }
3706 
3707     return Ok && skipToken(AsmToken::RParen, "expected a closing parentheses");
3708   }
3709 
3710   return false;
3711 }
3712 
3713 OperandMatchResultTy
3714 AMDGPUAsmParser::parseSwizzleOp(OperandVector &Operands) {
3715   SMLoc S = Parser.getTok().getLoc();
3716   int64_t Imm = 0;
3717 
3718   if (trySkipId("offset")) {
3719 
3720     bool Ok = false;
3721     if (skipToken(AsmToken::Colon, "expected a colon")) {
3722       if (trySkipId("swizzle")) {
3723         Ok = parseSwizzleMacro(Imm);
3724       } else {
3725         Ok = parseSwizzleOffset(Imm);
3726       }
3727     }
3728 
3729     Operands.push_back(AMDGPUOperand::CreateImm(this, Imm, S, AMDGPUOperand::ImmTySwizzle));
3730 
3731     return Ok? MatchOperand_Success : MatchOperand_ParseFail;
3732   } else {
3733     return MatchOperand_NoMatch;
3734   }
3735 }
3736 
3737 bool
3738 AMDGPUOperand::isSwizzle() const {
3739   return isImmTy(ImmTySwizzle);
3740 }
3741 
3742 //===----------------------------------------------------------------------===//
3743 // sopp branch targets
3744 //===----------------------------------------------------------------------===//
3745 
3746 OperandMatchResultTy
3747 AMDGPUAsmParser::parseSOppBrTarget(OperandVector &Operands) {
3748   SMLoc S = Parser.getTok().getLoc();
3749 
3750   switch (getLexer().getKind()) {
3751     default: return MatchOperand_ParseFail;
3752     case AsmToken::Integer: {
3753       int64_t Imm;
3754       if (getParser().parseAbsoluteExpression(Imm))
3755         return MatchOperand_ParseFail;
3756       Operands.push_back(AMDGPUOperand::CreateImm(this, Imm, S));
3757       return MatchOperand_Success;
3758     }
3759 
3760     case AsmToken::Identifier:
3761       Operands.push_back(AMDGPUOperand::CreateExpr(this,
3762           MCSymbolRefExpr::create(getContext().getOrCreateSymbol(
3763                                   Parser.getTok().getString()), getContext()), S));
3764       Parser.Lex();
3765       return MatchOperand_Success;
3766   }
3767 }
3768 
3769 //===----------------------------------------------------------------------===//
3770 // mubuf
3771 //===----------------------------------------------------------------------===//
3772 
3773 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultGLC() const {
3774   return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTyGLC);
3775 }
3776 
3777 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultSLC() const {
3778   return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTySLC);
3779 }
3780 
3781 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultTFE() const {
3782   return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTyTFE);
3783 }
3784 
3785 void AMDGPUAsmParser::cvtMubufImpl(MCInst &Inst,
3786                                const OperandVector &Operands,
3787                                bool IsAtomic, bool IsAtomicReturn) {
3788   OptionalImmIndexMap OptionalIdx;
3789   assert(IsAtomicReturn ? IsAtomic : true);
3790 
3791   for (unsigned i = 1, e = Operands.size(); i != e; ++i) {
3792     AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[i]);
3793 
3794     // Add the register arguments
3795     if (Op.isReg()) {
3796       Op.addRegOperands(Inst, 1);
3797       continue;
3798     }
3799 
3800     // Handle the case where soffset is an immediate
3801     if (Op.isImm() && Op.getImmTy() == AMDGPUOperand::ImmTyNone) {
3802       Op.addImmOperands(Inst, 1);
3803       continue;
3804     }
3805 
3806     // Handle tokens like 'offen' which are sometimes hard-coded into the
3807     // asm string.  There are no MCInst operands for these.
3808     if (Op.isToken()) {
3809       continue;
3810     }
3811     assert(Op.isImm());
3812 
3813     // Handle optional arguments
3814     OptionalIdx[Op.getImmTy()] = i;
3815   }
3816 
3817   // Copy $vdata_in operand and insert as $vdata for MUBUF_Atomic RTN insns.
3818   if (IsAtomicReturn) {
3819     MCInst::iterator I = Inst.begin(); // $vdata_in is always at the beginning.
3820     Inst.insert(I, *I);
3821   }
3822 
3823   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyOffset);
3824   if (!IsAtomic) { // glc is hard-coded.
3825     addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyGLC);
3826   }
3827   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySLC);
3828   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyTFE);
3829 }
3830 
3831 void AMDGPUAsmParser::cvtMtbuf(MCInst &Inst, const OperandVector &Operands) {
3832   OptionalImmIndexMap OptionalIdx;
3833 
3834   for (unsigned i = 1, e = Operands.size(); i != e; ++i) {
3835     AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[i]);
3836 
3837     // Add the register arguments
3838     if (Op.isReg()) {
3839       Op.addRegOperands(Inst, 1);
3840       continue;
3841     }
3842 
3843     // Handle the case where soffset is an immediate
3844     if (Op.isImm() && Op.getImmTy() == AMDGPUOperand::ImmTyNone) {
3845       Op.addImmOperands(Inst, 1);
3846       continue;
3847     }
3848 
3849     // Handle tokens like 'offen' which are sometimes hard-coded into the
3850     // asm string.  There are no MCInst operands for these.
3851     if (Op.isToken()) {
3852       continue;
3853     }
3854     assert(Op.isImm());
3855 
3856     // Handle optional arguments
3857     OptionalIdx[Op.getImmTy()] = i;
3858   }
3859 
3860   addOptionalImmOperand(Inst, Operands, OptionalIdx,
3861                         AMDGPUOperand::ImmTyOffset);
3862   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyDFMT);
3863   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyNFMT);
3864   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyGLC);
3865   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySLC);
3866   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyTFE);
3867 }
3868 
3869 //===----------------------------------------------------------------------===//
3870 // mimg
3871 //===----------------------------------------------------------------------===//
3872 
3873 void AMDGPUAsmParser::cvtMIMG(MCInst &Inst, const OperandVector &Operands,
3874                               bool IsAtomic) {
3875   unsigned I = 1;
3876   const MCInstrDesc &Desc = MII.get(Inst.getOpcode());
3877   for (unsigned J = 0; J < Desc.getNumDefs(); ++J) {
3878     ((AMDGPUOperand &)*Operands[I++]).addRegOperands(Inst, 1);
3879   }
3880 
3881   if (IsAtomic) {
3882     // Add src, same as dst
3883     ((AMDGPUOperand &)*Operands[I]).addRegOperands(Inst, 1);
3884   }
3885 
3886   OptionalImmIndexMap OptionalIdx;
3887 
3888   for (unsigned E = Operands.size(); I != E; ++I) {
3889     AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[I]);
3890 
3891     // Add the register arguments
3892     if (Op.isRegOrImm()) {
3893       Op.addRegOrImmOperands(Inst, 1);
3894       continue;
3895     } else if (Op.isImmModifier()) {
3896       OptionalIdx[Op.getImmTy()] = I;
3897     } else {
3898       llvm_unreachable("unexpected operand type");
3899     }
3900   }
3901 
3902   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyDMask);
3903   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyUNorm);
3904   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyGLC);
3905   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyDA);
3906   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyR128);
3907   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyTFE);
3908   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyLWE);
3909   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySLC);
3910 }
3911 
3912 void AMDGPUAsmParser::cvtMIMGAtomic(MCInst &Inst, const OperandVector &Operands) {
3913   cvtMIMG(Inst, Operands, true);
3914 }
3915 
3916 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultDMask() const {
3917   return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTyDMask);
3918 }
3919 
3920 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultUNorm() const {
3921   return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTyUNorm);
3922 }
3923 
3924 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultDA() const {
3925   return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTyDA);
3926 }
3927 
3928 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultR128() const {
3929   return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTyR128);
3930 }
3931 
3932 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultLWE() const {
3933   return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTyLWE);
3934 }
3935 
3936 //===----------------------------------------------------------------------===//
3937 // smrd
3938 //===----------------------------------------------------------------------===//
3939 
3940 bool AMDGPUOperand::isSMRDOffset8() const {
3941   return isImm() && isUInt<8>(getImm());
3942 }
3943 
3944 bool AMDGPUOperand::isSMRDOffset20() const {
3945   return isImm() && isUInt<20>(getImm());
3946 }
3947 
3948 bool AMDGPUOperand::isSMRDLiteralOffset() const {
3949   // 32-bit literals are only supported on CI and we only want to use them
3950   // when the offset is > 8-bits.
3951   return isImm() && !isUInt<8>(getImm()) && isUInt<32>(getImm());
3952 }
3953 
3954 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultSMRDOffset8() const {
3955   return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTyOffset);
3956 }
3957 
3958 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultSMRDOffset20() const {
3959   return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTyOffset);
3960 }
3961 
3962 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultSMRDLiteralOffset() const {
3963   return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTyOffset);
3964 }
3965 
3966 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultOffsetU12() const {
3967   return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTyOffset);
3968 }
3969 
3970 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultOffsetS13() const {
3971   return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTyOffset);
3972 }
3973 
3974 //===----------------------------------------------------------------------===//
3975 // vop3
3976 //===----------------------------------------------------------------------===//
3977 
3978 static bool ConvertOmodMul(int64_t &Mul) {
3979   if (Mul != 1 && Mul != 2 && Mul != 4)
3980     return false;
3981 
3982   Mul >>= 1;
3983   return true;
3984 }
3985 
3986 static bool ConvertOmodDiv(int64_t &Div) {
3987   if (Div == 1) {
3988     Div = 0;
3989     return true;
3990   }
3991 
3992   if (Div == 2) {
3993     Div = 3;
3994     return true;
3995   }
3996 
3997   return false;
3998 }
3999 
4000 static bool ConvertBoundCtrl(int64_t &BoundCtrl) {
4001   if (BoundCtrl == 0) {
4002     BoundCtrl = 1;
4003     return true;
4004   }
4005 
4006   if (BoundCtrl == -1) {
4007     BoundCtrl = 0;
4008     return true;
4009   }
4010 
4011   return false;
4012 }
4013 
4014 // Note: the order in this table matches the order of operands in AsmString.
4015 static const OptionalOperand AMDGPUOptionalOperandTable[] = {
4016   {"offen",   AMDGPUOperand::ImmTyOffen, true, nullptr},
4017   {"idxen",   AMDGPUOperand::ImmTyIdxen, true, nullptr},
4018   {"addr64",  AMDGPUOperand::ImmTyAddr64, true, nullptr},
4019   {"offset0", AMDGPUOperand::ImmTyOffset0, false, nullptr},
4020   {"offset1", AMDGPUOperand::ImmTyOffset1, false, nullptr},
4021   {"gds",     AMDGPUOperand::ImmTyGDS, true, nullptr},
4022   {"offset",  AMDGPUOperand::ImmTyOffset, false, nullptr},
4023   {"dfmt",    AMDGPUOperand::ImmTyDFMT, false, nullptr},
4024   {"nfmt",    AMDGPUOperand::ImmTyNFMT, false, nullptr},
4025   {"glc",     AMDGPUOperand::ImmTyGLC, true, nullptr},
4026   {"slc",     AMDGPUOperand::ImmTySLC, true, nullptr},
4027   {"tfe",     AMDGPUOperand::ImmTyTFE, true, nullptr},
4028   {"high",    AMDGPUOperand::ImmTyHigh, true, nullptr},
4029   {"clamp",   AMDGPUOperand::ImmTyClampSI, true, nullptr},
4030   {"omod",    AMDGPUOperand::ImmTyOModSI, false, ConvertOmodMul},
4031   {"unorm",   AMDGPUOperand::ImmTyUNorm, true, nullptr},
4032   {"da",      AMDGPUOperand::ImmTyDA,    true, nullptr},
4033   {"r128",    AMDGPUOperand::ImmTyR128,  true, nullptr},
4034   {"lwe",     AMDGPUOperand::ImmTyLWE,   true, nullptr},
4035   {"dmask",   AMDGPUOperand::ImmTyDMask, false, nullptr},
4036   {"row_mask",   AMDGPUOperand::ImmTyDppRowMask, false, nullptr},
4037   {"bank_mask",  AMDGPUOperand::ImmTyDppBankMask, false, nullptr},
4038   {"bound_ctrl", AMDGPUOperand::ImmTyDppBoundCtrl, false, ConvertBoundCtrl},
4039   {"dst_sel",    AMDGPUOperand::ImmTySdwaDstSel, false, nullptr},
4040   {"src0_sel",   AMDGPUOperand::ImmTySdwaSrc0Sel, false, nullptr},
4041   {"src1_sel",   AMDGPUOperand::ImmTySdwaSrc1Sel, false, nullptr},
4042   {"dst_unused", AMDGPUOperand::ImmTySdwaDstUnused, false, nullptr},
4043   {"compr", AMDGPUOperand::ImmTyExpCompr, true, nullptr },
4044   {"vm", AMDGPUOperand::ImmTyExpVM, true, nullptr},
4045   {"op_sel", AMDGPUOperand::ImmTyOpSel, false, nullptr},
4046   {"op_sel_hi", AMDGPUOperand::ImmTyOpSelHi, false, nullptr},
4047   {"neg_lo", AMDGPUOperand::ImmTyNegLo, false, nullptr},
4048   {"neg_hi", AMDGPUOperand::ImmTyNegHi, false, nullptr}
4049 };
4050 
4051 OperandMatchResultTy AMDGPUAsmParser::parseOptionalOperand(OperandVector &Operands) {
4052   OperandMatchResultTy res;
4053   for (const OptionalOperand &Op : AMDGPUOptionalOperandTable) {
4054     // try to parse any optional operand here
4055     if (Op.IsBit) {
4056       res = parseNamedBit(Op.Name, Operands, Op.Type);
4057     } else if (Op.Type == AMDGPUOperand::ImmTyOModSI) {
4058       res = parseOModOperand(Operands);
4059     } else if (Op.Type == AMDGPUOperand::ImmTySdwaDstSel ||
4060                Op.Type == AMDGPUOperand::ImmTySdwaSrc0Sel ||
4061                Op.Type == AMDGPUOperand::ImmTySdwaSrc1Sel) {
4062       res = parseSDWASel(Operands, Op.Name, Op.Type);
4063     } else if (Op.Type == AMDGPUOperand::ImmTySdwaDstUnused) {
4064       res = parseSDWADstUnused(Operands);
4065     } else if (Op.Type == AMDGPUOperand::ImmTyOpSel ||
4066                Op.Type == AMDGPUOperand::ImmTyOpSelHi ||
4067                Op.Type == AMDGPUOperand::ImmTyNegLo ||
4068                Op.Type == AMDGPUOperand::ImmTyNegHi) {
4069       res = parseOperandArrayWithPrefix(Op.Name, Operands, Op.Type,
4070                                         Op.ConvertResult);
4071     } else {
4072       res = parseIntWithPrefix(Op.Name, Operands, Op.Type, Op.ConvertResult);
4073     }
4074     if (res != MatchOperand_NoMatch) {
4075       return res;
4076     }
4077   }
4078   return MatchOperand_NoMatch;
4079 }
4080 
4081 OperandMatchResultTy AMDGPUAsmParser::parseOModOperand(OperandVector &Operands) {
4082   StringRef Name = Parser.getTok().getString();
4083   if (Name == "mul") {
4084     return parseIntWithPrefix("mul", Operands,
4085                               AMDGPUOperand::ImmTyOModSI, ConvertOmodMul);
4086   }
4087 
4088   if (Name == "div") {
4089     return parseIntWithPrefix("div", Operands,
4090                               AMDGPUOperand::ImmTyOModSI, ConvertOmodDiv);
4091   }
4092 
4093   return MatchOperand_NoMatch;
4094 }
4095 
4096 void AMDGPUAsmParser::cvtVOP3OpSel(MCInst &Inst, const OperandVector &Operands) {
4097   cvtVOP3P(Inst, Operands);
4098 
4099   int Opc = Inst.getOpcode();
4100 
4101   int SrcNum;
4102   const int Ops[] = { AMDGPU::OpName::src0,
4103                       AMDGPU::OpName::src1,
4104                       AMDGPU::OpName::src2 };
4105   for (SrcNum = 0;
4106        SrcNum < 3 && AMDGPU::getNamedOperandIdx(Opc, Ops[SrcNum]) != -1;
4107        ++SrcNum);
4108   assert(SrcNum > 0);
4109 
4110   int OpSelIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::op_sel);
4111   unsigned OpSel = Inst.getOperand(OpSelIdx).getImm();
4112 
4113   if ((OpSel & (1 << SrcNum)) != 0) {
4114     int ModIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src0_modifiers);
4115     uint32_t ModVal = Inst.getOperand(ModIdx).getImm();
4116     Inst.getOperand(ModIdx).setImm(ModVal | SISrcMods::DST_OP_SEL);
4117   }
4118 }
4119 
4120 static bool isRegOrImmWithInputMods(const MCInstrDesc &Desc, unsigned OpNum) {
4121       // 1. This operand is input modifiers
4122   return Desc.OpInfo[OpNum].OperandType == AMDGPU::OPERAND_INPUT_MODS
4123       // 2. This is not last operand
4124       && Desc.NumOperands > (OpNum + 1)
4125       // 3. Next operand is register class
4126       && Desc.OpInfo[OpNum + 1].RegClass != -1
4127       // 4. Next register is not tied to any other operand
4128       && Desc.getOperandConstraint(OpNum + 1, MCOI::OperandConstraint::TIED_TO) == -1;
4129 }
4130 
4131 void AMDGPUAsmParser::cvtVOP3Interp(MCInst &Inst, const OperandVector &Operands)
4132 {
4133   OptionalImmIndexMap OptionalIdx;
4134   unsigned Opc = Inst.getOpcode();
4135 
4136   unsigned I = 1;
4137   const MCInstrDesc &Desc = MII.get(Inst.getOpcode());
4138   for (unsigned J = 0; J < Desc.getNumDefs(); ++J) {
4139     ((AMDGPUOperand &)*Operands[I++]).addRegOperands(Inst, 1);
4140   }
4141 
4142   for (unsigned E = Operands.size(); I != E; ++I) {
4143     AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[I]);
4144     if (isRegOrImmWithInputMods(Desc, Inst.getNumOperands())) {
4145       Op.addRegOrImmWithFPInputModsOperands(Inst, 2);
4146     } else if (Op.isInterpSlot() ||
4147                Op.isInterpAttr() ||
4148                Op.isAttrChan()) {
4149       Inst.addOperand(MCOperand::createImm(Op.Imm.Val));
4150     } else if (Op.isImmModifier()) {
4151       OptionalIdx[Op.getImmTy()] = I;
4152     } else {
4153       llvm_unreachable("unhandled operand type");
4154     }
4155   }
4156 
4157   if (AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::high) != -1) {
4158     addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyHigh);
4159   }
4160 
4161   if (AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::clamp) != -1) {
4162     addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyClampSI);
4163   }
4164 
4165   if (AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::omod) != -1) {
4166     addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyOModSI);
4167   }
4168 }
4169 
4170 void AMDGPUAsmParser::cvtVOP3(MCInst &Inst, const OperandVector &Operands,
4171                               OptionalImmIndexMap &OptionalIdx) {
4172   unsigned Opc = Inst.getOpcode();
4173 
4174   unsigned I = 1;
4175   const MCInstrDesc &Desc = MII.get(Inst.getOpcode());
4176   for (unsigned J = 0; J < Desc.getNumDefs(); ++J) {
4177     ((AMDGPUOperand &)*Operands[I++]).addRegOperands(Inst, 1);
4178   }
4179 
4180   if (AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src0_modifiers) != -1) {
4181     // This instruction has src modifiers
4182     for (unsigned E = Operands.size(); I != E; ++I) {
4183       AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[I]);
4184       if (isRegOrImmWithInputMods(Desc, Inst.getNumOperands())) {
4185         Op.addRegOrImmWithFPInputModsOperands(Inst, 2);
4186       } else if (Op.isImmModifier()) {
4187         OptionalIdx[Op.getImmTy()] = I;
4188       } else if (Op.isRegOrImm()) {
4189         Op.addRegOrImmOperands(Inst, 1);
4190       } else {
4191         llvm_unreachable("unhandled operand type");
4192       }
4193     }
4194   } else {
4195     // No src modifiers
4196     for (unsigned E = Operands.size(); I != E; ++I) {
4197       AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[I]);
4198       if (Op.isMod()) {
4199         OptionalIdx[Op.getImmTy()] = I;
4200       } else {
4201         Op.addRegOrImmOperands(Inst, 1);
4202       }
4203     }
4204   }
4205 
4206   if (AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::clamp) != -1) {
4207     addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyClampSI);
4208   }
4209 
4210   if (AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::omod) != -1) {
4211     addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyOModSI);
4212   }
4213 
4214   // special case v_mac_{f16, f32}:
4215   // it has src2 register operand that is tied to dst operand
4216   // we don't allow modifiers for this operand in assembler so src2_modifiers
4217   // should be 0
4218   if (Opc == AMDGPU::V_MAC_F32_e64_si || Opc == AMDGPU::V_MAC_F32_e64_vi ||
4219       Opc == AMDGPU::V_MAC_F16_e64_vi) {
4220     auto it = Inst.begin();
4221     std::advance(it, AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src2_modifiers));
4222     it = Inst.insert(it, MCOperand::createImm(0)); // no modifiers for src2
4223     ++it;
4224     Inst.insert(it, Inst.getOperand(0)); // src2 = dst
4225   }
4226 }
4227 
4228 void AMDGPUAsmParser::cvtVOP3(MCInst &Inst, const OperandVector &Operands) {
4229   OptionalImmIndexMap OptionalIdx;
4230   cvtVOP3(Inst, Operands, OptionalIdx);
4231 }
4232 
4233 void AMDGPUAsmParser::cvtVOP3P(MCInst &Inst, const OperandVector &Operands) {
4234   OptionalImmIndexMap OptIdx;
4235 
4236   cvtVOP3(Inst, Operands, OptIdx);
4237 
4238   // FIXME: This is messy. Parse the modifiers as if it was a normal VOP3
4239   // instruction, and then figure out where to actually put the modifiers
4240   int Opc = Inst.getOpcode();
4241 
4242   addOptionalImmOperand(Inst, Operands, OptIdx, AMDGPUOperand::ImmTyOpSel);
4243 
4244   int OpSelHiIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::op_sel_hi);
4245   if (OpSelHiIdx != -1) {
4246     addOptionalImmOperand(Inst, Operands, OptIdx, AMDGPUOperand::ImmTyOpSelHi, -1);
4247   }
4248 
4249   int NegLoIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::neg_lo);
4250   if (NegLoIdx != -1) {
4251     addOptionalImmOperand(Inst, Operands, OptIdx, AMDGPUOperand::ImmTyNegLo);
4252     addOptionalImmOperand(Inst, Operands, OptIdx, AMDGPUOperand::ImmTyNegHi);
4253   }
4254 
4255   const int Ops[] = { AMDGPU::OpName::src0,
4256                       AMDGPU::OpName::src1,
4257                       AMDGPU::OpName::src2 };
4258   const int ModOps[] = { AMDGPU::OpName::src0_modifiers,
4259                          AMDGPU::OpName::src1_modifiers,
4260                          AMDGPU::OpName::src2_modifiers };
4261 
4262   int OpSelIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::op_sel);
4263 
4264   unsigned OpSel = Inst.getOperand(OpSelIdx).getImm();
4265   unsigned OpSelHi = 0;
4266   unsigned NegLo = 0;
4267   unsigned NegHi = 0;
4268 
4269   if (OpSelHiIdx != -1) {
4270     OpSelHi = Inst.getOperand(OpSelHiIdx).getImm();
4271   }
4272 
4273   if (NegLoIdx != -1) {
4274     int NegHiIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::neg_hi);
4275     NegLo = Inst.getOperand(NegLoIdx).getImm();
4276     NegHi = Inst.getOperand(NegHiIdx).getImm();
4277   }
4278 
4279   for (int J = 0; J < 3; ++J) {
4280     int OpIdx = AMDGPU::getNamedOperandIdx(Opc, Ops[J]);
4281     if (OpIdx == -1)
4282       break;
4283 
4284     uint32_t ModVal = 0;
4285 
4286     if ((OpSel & (1 << J)) != 0)
4287       ModVal |= SISrcMods::OP_SEL_0;
4288 
4289     if ((OpSelHi & (1 << J)) != 0)
4290       ModVal |= SISrcMods::OP_SEL_1;
4291 
4292     if ((NegLo & (1 << J)) != 0)
4293       ModVal |= SISrcMods::NEG;
4294 
4295     if ((NegHi & (1 << J)) != 0)
4296       ModVal |= SISrcMods::NEG_HI;
4297 
4298     int ModIdx = AMDGPU::getNamedOperandIdx(Opc, ModOps[J]);
4299 
4300     Inst.getOperand(ModIdx).setImm(Inst.getOperand(ModIdx).getImm() | ModVal);
4301   }
4302 }
4303 
4304 //===----------------------------------------------------------------------===//
4305 // dpp
4306 //===----------------------------------------------------------------------===//
4307 
4308 bool AMDGPUOperand::isDPPCtrl() const {
4309   bool result = isImm() && getImmTy() == ImmTyDppCtrl && isUInt<9>(getImm());
4310   if (result) {
4311     int64_t Imm = getImm();
4312     return ((Imm >= 0x000) && (Imm <= 0x0ff)) ||
4313            ((Imm >= 0x101) && (Imm <= 0x10f)) ||
4314            ((Imm >= 0x111) && (Imm <= 0x11f)) ||
4315            ((Imm >= 0x121) && (Imm <= 0x12f)) ||
4316            (Imm == 0x130) ||
4317            (Imm == 0x134) ||
4318            (Imm == 0x138) ||
4319            (Imm == 0x13c) ||
4320            (Imm == 0x140) ||
4321            (Imm == 0x141) ||
4322            (Imm == 0x142) ||
4323            (Imm == 0x143);
4324   }
4325   return false;
4326 }
4327 
4328 bool AMDGPUOperand::isGPRIdxMode() const {
4329   return isImm() && isUInt<4>(getImm());
4330 }
4331 
4332 bool AMDGPUOperand::isS16Imm() const {
4333   return isImm() && (isInt<16>(getImm()) || isUInt<16>(getImm()));
4334 }
4335 
4336 bool AMDGPUOperand::isU16Imm() const {
4337   return isImm() && isUInt<16>(getImm());
4338 }
4339 
4340 OperandMatchResultTy
4341 AMDGPUAsmParser::parseDPPCtrl(OperandVector &Operands) {
4342   SMLoc S = Parser.getTok().getLoc();
4343   StringRef Prefix;
4344   int64_t Int;
4345 
4346   if (getLexer().getKind() == AsmToken::Identifier) {
4347     Prefix = Parser.getTok().getString();
4348   } else {
4349     return MatchOperand_NoMatch;
4350   }
4351 
4352   if (Prefix == "row_mirror") {
4353     Int = 0x140;
4354     Parser.Lex();
4355   } else if (Prefix == "row_half_mirror") {
4356     Int = 0x141;
4357     Parser.Lex();
4358   } else {
4359     // Check to prevent parseDPPCtrlOps from eating invalid tokens
4360     if (Prefix != "quad_perm"
4361         && Prefix != "row_shl"
4362         && Prefix != "row_shr"
4363         && Prefix != "row_ror"
4364         && Prefix != "wave_shl"
4365         && Prefix != "wave_rol"
4366         && Prefix != "wave_shr"
4367         && Prefix != "wave_ror"
4368         && Prefix != "row_bcast") {
4369       return MatchOperand_NoMatch;
4370     }
4371 
4372     Parser.Lex();
4373     if (getLexer().isNot(AsmToken::Colon))
4374       return MatchOperand_ParseFail;
4375 
4376     if (Prefix == "quad_perm") {
4377       // quad_perm:[%d,%d,%d,%d]
4378       Parser.Lex();
4379       if (getLexer().isNot(AsmToken::LBrac))
4380         return MatchOperand_ParseFail;
4381       Parser.Lex();
4382 
4383       if (getParser().parseAbsoluteExpression(Int) || !(0 <= Int && Int <=3))
4384         return MatchOperand_ParseFail;
4385 
4386       for (int i = 0; i < 3; ++i) {
4387         if (getLexer().isNot(AsmToken::Comma))
4388           return MatchOperand_ParseFail;
4389         Parser.Lex();
4390 
4391         int64_t Temp;
4392         if (getParser().parseAbsoluteExpression(Temp) || !(0 <= Temp && Temp <=3))
4393           return MatchOperand_ParseFail;
4394         const int shift = i*2 + 2;
4395         Int += (Temp << shift);
4396       }
4397 
4398       if (getLexer().isNot(AsmToken::RBrac))
4399         return MatchOperand_ParseFail;
4400       Parser.Lex();
4401     } else {
4402       // sel:%d
4403       Parser.Lex();
4404       if (getParser().parseAbsoluteExpression(Int))
4405         return MatchOperand_ParseFail;
4406 
4407       if (Prefix == "row_shl" && 1 <= Int && Int <= 15) {
4408         Int |= 0x100;
4409       } else if (Prefix == "row_shr" && 1 <= Int && Int <= 15) {
4410         Int |= 0x110;
4411       } else if (Prefix == "row_ror" && 1 <= Int && Int <= 15) {
4412         Int |= 0x120;
4413       } else if (Prefix == "wave_shl" && 1 == Int) {
4414         Int = 0x130;
4415       } else if (Prefix == "wave_rol" && 1 == Int) {
4416         Int = 0x134;
4417       } else if (Prefix == "wave_shr" && 1 == Int) {
4418         Int = 0x138;
4419       } else if (Prefix == "wave_ror" && 1 == Int) {
4420         Int = 0x13C;
4421       } else if (Prefix == "row_bcast") {
4422         if (Int == 15) {
4423           Int = 0x142;
4424         } else if (Int == 31) {
4425           Int = 0x143;
4426         } else {
4427           return MatchOperand_ParseFail;
4428         }
4429       } else {
4430         return MatchOperand_ParseFail;
4431       }
4432     }
4433   }
4434 
4435   Operands.push_back(AMDGPUOperand::CreateImm(this, Int, S, AMDGPUOperand::ImmTyDppCtrl));
4436   return MatchOperand_Success;
4437 }
4438 
4439 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultRowMask() const {
4440   return AMDGPUOperand::CreateImm(this, 0xf, SMLoc(), AMDGPUOperand::ImmTyDppRowMask);
4441 }
4442 
4443 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultBankMask() const {
4444   return AMDGPUOperand::CreateImm(this, 0xf, SMLoc(), AMDGPUOperand::ImmTyDppBankMask);
4445 }
4446 
4447 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultBoundCtrl() const {
4448   return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTyDppBoundCtrl);
4449 }
4450 
4451 void AMDGPUAsmParser::cvtDPP(MCInst &Inst, const OperandVector &Operands) {
4452   OptionalImmIndexMap OptionalIdx;
4453 
4454   unsigned I = 1;
4455   const MCInstrDesc &Desc = MII.get(Inst.getOpcode());
4456   for (unsigned J = 0; J < Desc.getNumDefs(); ++J) {
4457     ((AMDGPUOperand &)*Operands[I++]).addRegOperands(Inst, 1);
4458   }
4459 
4460   // All DPP instructions with at least one source operand have a fake "old"
4461   // source at the beginning that's tied to the dst operand. Handle it here.
4462   if (Desc.getNumOperands() >= 2)
4463     Inst.addOperand(Inst.getOperand(0));
4464 
4465   for (unsigned E = Operands.size(); I != E; ++I) {
4466     AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[I]);
4467     // Add the register arguments
4468     if (Op.isReg() && Op.Reg.RegNo == AMDGPU::VCC) {
4469       // VOP2b (v_add_u32, v_sub_u32 ...) dpp use "vcc" token.
4470       // Skip it.
4471       continue;
4472     } if (isRegOrImmWithInputMods(Desc, Inst.getNumOperands())) {
4473       Op.addRegWithFPInputModsOperands(Inst, 2);
4474     } else if (Op.isDPPCtrl()) {
4475       Op.addImmOperands(Inst, 1);
4476     } else if (Op.isImm()) {
4477       // Handle optional arguments
4478       OptionalIdx[Op.getImmTy()] = I;
4479     } else {
4480       llvm_unreachable("Invalid operand type");
4481     }
4482   }
4483 
4484   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyDppRowMask, 0xf);
4485   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyDppBankMask, 0xf);
4486   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyDppBoundCtrl);
4487 }
4488 
4489 //===----------------------------------------------------------------------===//
4490 // sdwa
4491 //===----------------------------------------------------------------------===//
4492 
4493 OperandMatchResultTy
4494 AMDGPUAsmParser::parseSDWASel(OperandVector &Operands, StringRef Prefix,
4495                               AMDGPUOperand::ImmTy Type) {
4496   using namespace llvm::AMDGPU::SDWA;
4497 
4498   SMLoc S = Parser.getTok().getLoc();
4499   StringRef Value;
4500   OperandMatchResultTy res;
4501 
4502   res = parseStringWithPrefix(Prefix, Value);
4503   if (res != MatchOperand_Success) {
4504     return res;
4505   }
4506 
4507   int64_t Int;
4508   Int = StringSwitch<int64_t>(Value)
4509         .Case("BYTE_0", SdwaSel::BYTE_0)
4510         .Case("BYTE_1", SdwaSel::BYTE_1)
4511         .Case("BYTE_2", SdwaSel::BYTE_2)
4512         .Case("BYTE_3", SdwaSel::BYTE_3)
4513         .Case("WORD_0", SdwaSel::WORD_0)
4514         .Case("WORD_1", SdwaSel::WORD_1)
4515         .Case("DWORD", SdwaSel::DWORD)
4516         .Default(0xffffffff);
4517   Parser.Lex(); // eat last token
4518 
4519   if (Int == 0xffffffff) {
4520     return MatchOperand_ParseFail;
4521   }
4522 
4523   Operands.push_back(AMDGPUOperand::CreateImm(this, Int, S, Type));
4524   return MatchOperand_Success;
4525 }
4526 
4527 OperandMatchResultTy
4528 AMDGPUAsmParser::parseSDWADstUnused(OperandVector &Operands) {
4529   using namespace llvm::AMDGPU::SDWA;
4530 
4531   SMLoc S = Parser.getTok().getLoc();
4532   StringRef Value;
4533   OperandMatchResultTy res;
4534 
4535   res = parseStringWithPrefix("dst_unused", Value);
4536   if (res != MatchOperand_Success) {
4537     return res;
4538   }
4539 
4540   int64_t Int;
4541   Int = StringSwitch<int64_t>(Value)
4542         .Case("UNUSED_PAD", DstUnused::UNUSED_PAD)
4543         .Case("UNUSED_SEXT", DstUnused::UNUSED_SEXT)
4544         .Case("UNUSED_PRESERVE", DstUnused::UNUSED_PRESERVE)
4545         .Default(0xffffffff);
4546   Parser.Lex(); // eat last token
4547 
4548   if (Int == 0xffffffff) {
4549     return MatchOperand_ParseFail;
4550   }
4551 
4552   Operands.push_back(AMDGPUOperand::CreateImm(this, Int, S, AMDGPUOperand::ImmTySdwaDstUnused));
4553   return MatchOperand_Success;
4554 }
4555 
4556 void AMDGPUAsmParser::cvtSdwaVOP1(MCInst &Inst, const OperandVector &Operands) {
4557   cvtSDWA(Inst, Operands, SIInstrFlags::VOP1);
4558 }
4559 
4560 void AMDGPUAsmParser::cvtSdwaVOP2(MCInst &Inst, const OperandVector &Operands) {
4561   cvtSDWA(Inst, Operands, SIInstrFlags::VOP2);
4562 }
4563 
4564 void AMDGPUAsmParser::cvtSdwaVOP2b(MCInst &Inst, const OperandVector &Operands) {
4565   cvtSDWA(Inst, Operands, SIInstrFlags::VOP2, true);
4566 }
4567 
4568 void AMDGPUAsmParser::cvtSdwaVOPC(MCInst &Inst, const OperandVector &Operands) {
4569   cvtSDWA(Inst, Operands, SIInstrFlags::VOPC, isVI());
4570 }
4571 
4572 void AMDGPUAsmParser::cvtSDWA(MCInst &Inst, const OperandVector &Operands,
4573                               uint64_t BasicInstType, bool skipVcc) {
4574   using namespace llvm::AMDGPU::SDWA;
4575 
4576   OptionalImmIndexMap OptionalIdx;
4577   bool skippedVcc = false;
4578 
4579   unsigned I = 1;
4580   const MCInstrDesc &Desc = MII.get(Inst.getOpcode());
4581   for (unsigned J = 0; J < Desc.getNumDefs(); ++J) {
4582     ((AMDGPUOperand &)*Operands[I++]).addRegOperands(Inst, 1);
4583   }
4584 
4585   for (unsigned E = Operands.size(); I != E; ++I) {
4586     AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[I]);
4587     if (skipVcc && !skippedVcc && Op.isReg() && Op.Reg.RegNo == AMDGPU::VCC) {
4588       // VOP2b (v_add_u32, v_sub_u32 ...) sdwa use "vcc" token as dst.
4589       // Skip it if it's 2nd (e.g. v_add_i32_sdwa v1, vcc, v2, v3)
4590       // or 4th (v_addc_u32_sdwa v1, vcc, v2, v3, vcc) operand.
4591       // Skip VCC only if we didn't skip it on previous iteration.
4592       if (BasicInstType == SIInstrFlags::VOP2 &&
4593           (Inst.getNumOperands() == 1 || Inst.getNumOperands() == 5)) {
4594         skippedVcc = true;
4595         continue;
4596       } else if (BasicInstType == SIInstrFlags::VOPC &&
4597                  Inst.getNumOperands() == 0) {
4598         skippedVcc = true;
4599         continue;
4600       }
4601     }
4602     if (isRegOrImmWithInputMods(Desc, Inst.getNumOperands())) {
4603       Op.addRegWithInputModsOperands(Inst, 2);
4604     } else if (Op.isImm()) {
4605       // Handle optional arguments
4606       OptionalIdx[Op.getImmTy()] = I;
4607     } else {
4608       llvm_unreachable("Invalid operand type");
4609     }
4610     skippedVcc = false;
4611   }
4612 
4613   if (Inst.getOpcode() != AMDGPU::V_NOP_sdwa_gfx9 &&
4614       Inst.getOpcode() != AMDGPU::V_NOP_sdwa_vi) {
4615     // v_nop_sdwa_sdwa_vi/gfx9 has no optional sdwa arguments
4616     switch (BasicInstType) {
4617     case SIInstrFlags::VOP1:
4618       addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyClampSI, 0);
4619       if (AMDGPU::getNamedOperandIdx(Inst.getOpcode(), AMDGPU::OpName::omod) != -1) {
4620         addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyOModSI, 0);
4621       }
4622       addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySdwaDstSel, SdwaSel::DWORD);
4623       addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySdwaDstUnused, DstUnused::UNUSED_PRESERVE);
4624       addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySdwaSrc0Sel, SdwaSel::DWORD);
4625       break;
4626 
4627     case SIInstrFlags::VOP2:
4628       addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyClampSI, 0);
4629       if (AMDGPU::getNamedOperandIdx(Inst.getOpcode(), AMDGPU::OpName::omod) != -1) {
4630         addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyOModSI, 0);
4631       }
4632       addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySdwaDstSel, SdwaSel::DWORD);
4633       addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySdwaDstUnused, DstUnused::UNUSED_PRESERVE);
4634       addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySdwaSrc0Sel, SdwaSel::DWORD);
4635       addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySdwaSrc1Sel, SdwaSel::DWORD);
4636       break;
4637 
4638     case SIInstrFlags::VOPC:
4639       addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyClampSI, 0);
4640       addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySdwaSrc0Sel, SdwaSel::DWORD);
4641       addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySdwaSrc1Sel, SdwaSel::DWORD);
4642       break;
4643 
4644     default:
4645       llvm_unreachable("Invalid instruction type. Only VOP1, VOP2 and VOPC allowed");
4646     }
4647   }
4648 
4649   // special case v_mac_{f16, f32}:
4650   // it has src2 register operand that is tied to dst operand
4651   if (Inst.getOpcode() == AMDGPU::V_MAC_F32_sdwa_vi ||
4652       Inst.getOpcode() == AMDGPU::V_MAC_F16_sdwa_vi)  {
4653     auto it = Inst.begin();
4654     std::advance(
4655       it, AMDGPU::getNamedOperandIdx(Inst.getOpcode(), AMDGPU::OpName::src2));
4656     Inst.insert(it, Inst.getOperand(0)); // src2 = dst
4657   }
4658 }
4659 
4660 /// Force static initialization.
4661 extern "C" void LLVMInitializeAMDGPUAsmParser() {
4662   RegisterMCAsmParser<AMDGPUAsmParser> A(getTheAMDGPUTarget());
4663   RegisterMCAsmParser<AMDGPUAsmParser> B(getTheGCNTarget());
4664 }
4665 
4666 #define GET_REGISTER_MATCHER
4667 #define GET_MATCHER_IMPLEMENTATION
4668 #include "AMDGPUGenAsmMatcher.inc"
4669 
4670 // This fuction should be defined after auto-generated include so that we have
4671 // MatchClassKind enum defined
4672 unsigned AMDGPUAsmParser::validateTargetOperandClass(MCParsedAsmOperand &Op,
4673                                                      unsigned Kind) {
4674   // Tokens like "glc" would be parsed as immediate operands in ParseOperand().
4675   // But MatchInstructionImpl() expects to meet token and fails to validate
4676   // operand. This method checks if we are given immediate operand but expect to
4677   // get corresponding token.
4678   AMDGPUOperand &Operand = (AMDGPUOperand&)Op;
4679   switch (Kind) {
4680   case MCK_addr64:
4681     return Operand.isAddr64() ? Match_Success : Match_InvalidOperand;
4682   case MCK_gds:
4683     return Operand.isGDS() ? Match_Success : Match_InvalidOperand;
4684   case MCK_glc:
4685     return Operand.isGLC() ? Match_Success : Match_InvalidOperand;
4686   case MCK_idxen:
4687     return Operand.isIdxen() ? Match_Success : Match_InvalidOperand;
4688   case MCK_offen:
4689     return Operand.isOffen() ? Match_Success : Match_InvalidOperand;
4690   case MCK_SSrcB32:
4691     // When operands have expression values, they will return true for isToken,
4692     // because it is not possible to distinguish between a token and an
4693     // expression at parse time. MatchInstructionImpl() will always try to
4694     // match an operand as a token, when isToken returns true, and when the
4695     // name of the expression is not a valid token, the match will fail,
4696     // so we need to handle it here.
4697     return Operand.isSSrcB32() ? Match_Success : Match_InvalidOperand;
4698   case MCK_SSrcF32:
4699     return Operand.isSSrcF32() ? Match_Success : Match_InvalidOperand;
4700   case MCK_SoppBrTarget:
4701     return Operand.isSoppBrTarget() ? Match_Success : Match_InvalidOperand;
4702   case MCK_VReg32OrOff:
4703     return Operand.isVReg32OrOff() ? Match_Success : Match_InvalidOperand;
4704   case MCK_InterpSlot:
4705     return Operand.isInterpSlot() ? Match_Success : Match_InvalidOperand;
4706   case MCK_Attr:
4707     return Operand.isInterpAttr() ? Match_Success : Match_InvalidOperand;
4708   case MCK_AttrChan:
4709     return Operand.isAttrChan() ? Match_Success : Match_InvalidOperand;
4710   default:
4711     return Match_InvalidOperand;
4712   }
4713 }
4714