1 //===- AMDGPUAsmParser.cpp - Parse SI asm to MCInst instructions ----------===//
2 //
3 //                     The LLVM Compiler Infrastructure
4 //
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
7 //
8 //===----------------------------------------------------------------------===//
9 
10 #include "AMDGPU.h"
11 #include "AMDKernelCodeT.h"
12 #include "MCTargetDesc/AMDGPUMCTargetDesc.h"
13 #include "MCTargetDesc/AMDGPUTargetStreamer.h"
14 #include "SIDefines.h"
15 #include "Utils/AMDGPUAsmUtils.h"
16 #include "Utils/AMDGPUBaseInfo.h"
17 #include "Utils/AMDKernelCodeTUtils.h"
18 #include "llvm/ADT/APFloat.h"
19 #include "llvm/ADT/APInt.h"
20 #include "llvm/ADT/ArrayRef.h"
21 #include "llvm/ADT/STLExtras.h"
22 #include "llvm/ADT/SmallBitVector.h"
23 #include "llvm/ADT/SmallString.h"
24 #include "llvm/ADT/StringRef.h"
25 #include "llvm/ADT/StringSwitch.h"
26 #include "llvm/ADT/Twine.h"
27 #include "llvm/BinaryFormat/ELF.h"
28 #include "llvm/CodeGen/MachineValueType.h"
29 #include "llvm/MC/MCAsmInfo.h"
30 #include "llvm/MC/MCContext.h"
31 #include "llvm/MC/MCExpr.h"
32 #include "llvm/MC/MCInst.h"
33 #include "llvm/MC/MCInstrDesc.h"
34 #include "llvm/MC/MCInstrInfo.h"
35 #include "llvm/MC/MCParser/MCAsmLexer.h"
36 #include "llvm/MC/MCParser/MCAsmParser.h"
37 #include "llvm/MC/MCParser/MCAsmParserExtension.h"
38 #include "llvm/MC/MCParser/MCParsedAsmOperand.h"
39 #include "llvm/MC/MCParser/MCTargetAsmParser.h"
40 #include "llvm/MC/MCRegisterInfo.h"
41 #include "llvm/MC/MCStreamer.h"
42 #include "llvm/MC/MCSubtargetInfo.h"
43 #include "llvm/MC/MCSymbol.h"
44 #include "llvm/Support/AMDGPUMetadata.h"
45 #include "llvm/Support/Casting.h"
46 #include "llvm/Support/Compiler.h"
47 #include "llvm/Support/ErrorHandling.h"
48 #include "llvm/Support/MathExtras.h"
49 #include "llvm/Support/SMLoc.h"
50 #include "llvm/Support/TargetRegistry.h"
51 #include "llvm/Support/raw_ostream.h"
52 #include <algorithm>
53 #include <cassert>
54 #include <cstdint>
55 #include <cstring>
56 #include <iterator>
57 #include <map>
58 #include <memory>
59 #include <string>
60 
61 using namespace llvm;
62 using namespace llvm::AMDGPU;
63 
64 namespace {
65 
66 class AMDGPUAsmParser;
67 
68 enum RegisterKind { IS_UNKNOWN, IS_VGPR, IS_SGPR, IS_TTMP, IS_SPECIAL };
69 
70 //===----------------------------------------------------------------------===//
71 // Operand
72 //===----------------------------------------------------------------------===//
73 
74 class AMDGPUOperand : public MCParsedAsmOperand {
75   enum KindTy {
76     Token,
77     Immediate,
78     Register,
79     Expression
80   } Kind;
81 
82   SMLoc StartLoc, EndLoc;
83   const AMDGPUAsmParser *AsmParser;
84 
85 public:
86   AMDGPUOperand(KindTy Kind_, const AMDGPUAsmParser *AsmParser_)
87     : MCParsedAsmOperand(), Kind(Kind_), AsmParser(AsmParser_) {}
88 
89   using Ptr = std::unique_ptr<AMDGPUOperand>;
90 
91   struct Modifiers {
92     bool Abs = false;
93     bool Neg = false;
94     bool Sext = false;
95 
96     bool hasFPModifiers() const { return Abs || Neg; }
97     bool hasIntModifiers() const { return Sext; }
98     bool hasModifiers() const { return hasFPModifiers() || hasIntModifiers(); }
99 
100     int64_t getFPModifiersOperand() const {
101       int64_t Operand = 0;
102       Operand |= Abs ? SISrcMods::ABS : 0;
103       Operand |= Neg ? SISrcMods::NEG : 0;
104       return Operand;
105     }
106 
107     int64_t getIntModifiersOperand() const {
108       int64_t Operand = 0;
109       Operand |= Sext ? SISrcMods::SEXT : 0;
110       return Operand;
111     }
112 
113     int64_t getModifiersOperand() const {
114       assert(!(hasFPModifiers() && hasIntModifiers())
115            && "fp and int modifiers should not be used simultaneously");
116       if (hasFPModifiers()) {
117         return getFPModifiersOperand();
118       } else if (hasIntModifiers()) {
119         return getIntModifiersOperand();
120       } else {
121         return 0;
122       }
123     }
124 
125     friend raw_ostream &operator <<(raw_ostream &OS, AMDGPUOperand::Modifiers Mods);
126   };
127 
128   enum ImmTy {
129     ImmTyNone,
130     ImmTyGDS,
131     ImmTyOffen,
132     ImmTyIdxen,
133     ImmTyAddr64,
134     ImmTyOffset,
135     ImmTyOffset0,
136     ImmTyOffset1,
137     ImmTyGLC,
138     ImmTySLC,
139     ImmTyTFE,
140     ImmTyClampSI,
141     ImmTyOModSI,
142     ImmTyDppCtrl,
143     ImmTyDppRowMask,
144     ImmTyDppBankMask,
145     ImmTyDppBoundCtrl,
146     ImmTySdwaDstSel,
147     ImmTySdwaSrc0Sel,
148     ImmTySdwaSrc1Sel,
149     ImmTySdwaDstUnused,
150     ImmTyDMask,
151     ImmTyUNorm,
152     ImmTyDA,
153     ImmTyR128,
154     ImmTyLWE,
155     ImmTyExpTgt,
156     ImmTyExpCompr,
157     ImmTyExpVM,
158     ImmTyDFMT,
159     ImmTyNFMT,
160     ImmTyHwreg,
161     ImmTyOff,
162     ImmTySendMsg,
163     ImmTyInterpSlot,
164     ImmTyInterpAttr,
165     ImmTyAttrChan,
166     ImmTyOpSel,
167     ImmTyOpSelHi,
168     ImmTyNegLo,
169     ImmTyNegHi,
170     ImmTySwizzle,
171     ImmTyHigh
172   };
173 
174   struct TokOp {
175     const char *Data;
176     unsigned Length;
177   };
178 
179   struct ImmOp {
180     int64_t Val;
181     ImmTy Type;
182     bool IsFPImm;
183     Modifiers Mods;
184   };
185 
186   struct RegOp {
187     unsigned RegNo;
188     bool IsForcedVOP3;
189     Modifiers Mods;
190   };
191 
192   union {
193     TokOp Tok;
194     ImmOp Imm;
195     RegOp Reg;
196     const MCExpr *Expr;
197   };
198 
199   bool isToken() const override {
200     if (Kind == Token)
201       return true;
202 
203     if (Kind != Expression || !Expr)
204       return false;
205 
206     // When parsing operands, we can't always tell if something was meant to be
207     // a token, like 'gds', or an expression that references a global variable.
208     // In this case, we assume the string is an expression, and if we need to
209     // interpret is a token, then we treat the symbol name as the token.
210     return isa<MCSymbolRefExpr>(Expr);
211   }
212 
213   bool isImm() const override {
214     return Kind == Immediate;
215   }
216 
217   bool isInlinableImm(MVT type) const;
218   bool isLiteralImm(MVT type) const;
219 
220   bool isRegKind() const {
221     return Kind == Register;
222   }
223 
224   bool isReg() const override {
225     return isRegKind() && !hasModifiers();
226   }
227 
228   bool isRegOrImmWithInputMods(MVT type) const {
229     return isRegKind() || isInlinableImm(type);
230   }
231 
232   bool isRegOrImmWithInt16InputMods() const {
233     return isRegOrImmWithInputMods(MVT::i16);
234   }
235 
236   bool isRegOrImmWithInt32InputMods() const {
237     return isRegOrImmWithInputMods(MVT::i32);
238   }
239 
240   bool isRegOrImmWithInt64InputMods() const {
241     return isRegOrImmWithInputMods(MVT::i64);
242   }
243 
244   bool isRegOrImmWithFP16InputMods() const {
245     return isRegOrImmWithInputMods(MVT::f16);
246   }
247 
248   bool isRegOrImmWithFP32InputMods() const {
249     return isRegOrImmWithInputMods(MVT::f32);
250   }
251 
252   bool isRegOrImmWithFP64InputMods() const {
253     return isRegOrImmWithInputMods(MVT::f64);
254   }
255 
256   bool isVReg() const {
257     return isRegClass(AMDGPU::VGPR_32RegClassID) ||
258            isRegClass(AMDGPU::VReg_64RegClassID) ||
259            isRegClass(AMDGPU::VReg_96RegClassID) ||
260            isRegClass(AMDGPU::VReg_128RegClassID) ||
261            isRegClass(AMDGPU::VReg_256RegClassID) ||
262            isRegClass(AMDGPU::VReg_512RegClassID);
263   }
264 
265   bool isVReg32OrOff() const {
266     return isOff() || isRegClass(AMDGPU::VGPR_32RegClassID);
267   }
268 
269   bool isSDWARegKind() const;
270 
271   bool isImmTy(ImmTy ImmT) const {
272     return isImm() && Imm.Type == ImmT;
273   }
274 
275   bool isImmModifier() const {
276     return isImm() && Imm.Type != ImmTyNone;
277   }
278 
279   bool isClampSI() const { return isImmTy(ImmTyClampSI); }
280   bool isOModSI() const { return isImmTy(ImmTyOModSI); }
281   bool isDMask() const { return isImmTy(ImmTyDMask); }
282   bool isUNorm() const { return isImmTy(ImmTyUNorm); }
283   bool isDA() const { return isImmTy(ImmTyDA); }
284   bool isR128() const { return isImmTy(ImmTyUNorm); }
285   bool isLWE() const { return isImmTy(ImmTyLWE); }
286   bool isOff() const { return isImmTy(ImmTyOff); }
287   bool isExpTgt() const { return isImmTy(ImmTyExpTgt); }
288   bool isExpVM() const { return isImmTy(ImmTyExpVM); }
289   bool isExpCompr() const { return isImmTy(ImmTyExpCompr); }
290   bool isOffen() const { return isImmTy(ImmTyOffen); }
291   bool isIdxen() const { return isImmTy(ImmTyIdxen); }
292   bool isAddr64() const { return isImmTy(ImmTyAddr64); }
293   bool isOffset() const { return isImmTy(ImmTyOffset) && isUInt<16>(getImm()); }
294   bool isOffset0() const { return isImmTy(ImmTyOffset0) && isUInt<16>(getImm()); }
295   bool isOffset1() const { return isImmTy(ImmTyOffset1) && isUInt<8>(getImm()); }
296 
297   bool isOffsetU12() const { return isImmTy(ImmTyOffset) && isUInt<12>(getImm()); }
298   bool isOffsetS13() const { return isImmTy(ImmTyOffset) && isInt<13>(getImm()); }
299   bool isGDS() const { return isImmTy(ImmTyGDS); }
300   bool isGLC() const { return isImmTy(ImmTyGLC); }
301   bool isSLC() const { return isImmTy(ImmTySLC); }
302   bool isTFE() const { return isImmTy(ImmTyTFE); }
303   bool isDFMT() const { return isImmTy(ImmTyDFMT) && isUInt<8>(getImm()); }
304   bool isNFMT() const { return isImmTy(ImmTyNFMT) && isUInt<8>(getImm()); }
305   bool isBankMask() const { return isImmTy(ImmTyDppBankMask); }
306   bool isRowMask() const { return isImmTy(ImmTyDppRowMask); }
307   bool isBoundCtrl() const { return isImmTy(ImmTyDppBoundCtrl); }
308   bool isSDWADstSel() const { return isImmTy(ImmTySdwaDstSel); }
309   bool isSDWASrc0Sel() const { return isImmTy(ImmTySdwaSrc0Sel); }
310   bool isSDWASrc1Sel() const { return isImmTy(ImmTySdwaSrc1Sel); }
311   bool isSDWADstUnused() const { return isImmTy(ImmTySdwaDstUnused); }
312   bool isInterpSlot() const { return isImmTy(ImmTyInterpSlot); }
313   bool isInterpAttr() const { return isImmTy(ImmTyInterpAttr); }
314   bool isAttrChan() const { return isImmTy(ImmTyAttrChan); }
315   bool isOpSel() const { return isImmTy(ImmTyOpSel); }
316   bool isOpSelHi() const { return isImmTy(ImmTyOpSelHi); }
317   bool isNegLo() const { return isImmTy(ImmTyNegLo); }
318   bool isNegHi() const { return isImmTy(ImmTyNegHi); }
319   bool isHigh() const { return isImmTy(ImmTyHigh); }
320 
321   bool isMod() const {
322     return isClampSI() || isOModSI();
323   }
324 
325   bool isRegOrImm() const {
326     return isReg() || isImm();
327   }
328 
329   bool isRegClass(unsigned RCID) const;
330 
331   bool isRegOrInlineNoMods(unsigned RCID, MVT type) const {
332     return (isRegClass(RCID) || isInlinableImm(type)) && !hasModifiers();
333   }
334 
335   bool isSCSrcB16() const {
336     return isRegOrInlineNoMods(AMDGPU::SReg_32RegClassID, MVT::i16);
337   }
338 
339   bool isSCSrcV2B16() const {
340     return isSCSrcB16();
341   }
342 
343   bool isSCSrcB32() const {
344     return isRegOrInlineNoMods(AMDGPU::SReg_32RegClassID, MVT::i32);
345   }
346 
347   bool isSCSrcB64() const {
348     return isRegOrInlineNoMods(AMDGPU::SReg_64RegClassID, MVT::i64);
349   }
350 
351   bool isSCSrcF16() const {
352     return isRegOrInlineNoMods(AMDGPU::SReg_32RegClassID, MVT::f16);
353   }
354 
355   bool isSCSrcV2F16() const {
356     return isSCSrcF16();
357   }
358 
359   bool isSCSrcF32() const {
360     return isRegOrInlineNoMods(AMDGPU::SReg_32RegClassID, MVT::f32);
361   }
362 
363   bool isSCSrcF64() const {
364     return isRegOrInlineNoMods(AMDGPU::SReg_64RegClassID, MVT::f64);
365   }
366 
367   bool isSSrcB32() const {
368     return isSCSrcB32() || isLiteralImm(MVT::i32) || isExpr();
369   }
370 
371   bool isSSrcB16() const {
372     return isSCSrcB16() || isLiteralImm(MVT::i16);
373   }
374 
375   bool isSSrcV2B16() const {
376     llvm_unreachable("cannot happen");
377     return isSSrcB16();
378   }
379 
380   bool isSSrcB64() const {
381     // TODO: Find out how SALU supports extension of 32-bit literals to 64 bits.
382     // See isVSrc64().
383     return isSCSrcB64() || isLiteralImm(MVT::i64);
384   }
385 
386   bool isSSrcF32() const {
387     return isSCSrcB32() || isLiteralImm(MVT::f32) || isExpr();
388   }
389 
390   bool isSSrcF64() const {
391     return isSCSrcB64() || isLiteralImm(MVT::f64);
392   }
393 
394   bool isSSrcF16() const {
395     return isSCSrcB16() || isLiteralImm(MVT::f16);
396   }
397 
398   bool isSSrcV2F16() const {
399     llvm_unreachable("cannot happen");
400     return isSSrcF16();
401   }
402 
403   bool isVCSrcB32() const {
404     return isRegOrInlineNoMods(AMDGPU::VS_32RegClassID, MVT::i32);
405   }
406 
407   bool isVCSrcB64() const {
408     return isRegOrInlineNoMods(AMDGPU::VS_64RegClassID, MVT::i64);
409   }
410 
411   bool isVCSrcB16() const {
412     return isRegOrInlineNoMods(AMDGPU::VS_32RegClassID, MVT::i16);
413   }
414 
415   bool isVCSrcV2B16() const {
416     return isVCSrcB16();
417   }
418 
419   bool isVCSrcF32() const {
420     return isRegOrInlineNoMods(AMDGPU::VS_32RegClassID, MVT::f32);
421   }
422 
423   bool isVCSrcF64() const {
424     return isRegOrInlineNoMods(AMDGPU::VS_64RegClassID, MVT::f64);
425   }
426 
427   bool isVCSrcF16() const {
428     return isRegOrInlineNoMods(AMDGPU::VS_32RegClassID, MVT::f16);
429   }
430 
431   bool isVCSrcV2F16() const {
432     return isVCSrcF16();
433   }
434 
435   bool isVSrcB32() const {
436     return isVCSrcF32() || isLiteralImm(MVT::i32);
437   }
438 
439   bool isVSrcB64() const {
440     return isVCSrcF64() || isLiteralImm(MVT::i64);
441   }
442 
443   bool isVSrcB16() const {
444     return isVCSrcF16() || isLiteralImm(MVT::i16);
445   }
446 
447   bool isVSrcV2B16() const {
448     llvm_unreachable("cannot happen");
449     return isVSrcB16();
450   }
451 
452   bool isVSrcF32() const {
453     return isVCSrcF32() || isLiteralImm(MVT::f32);
454   }
455 
456   bool isVSrcF64() const {
457     return isVCSrcF64() || isLiteralImm(MVT::f64);
458   }
459 
460   bool isVSrcF16() const {
461     return isVCSrcF16() || isLiteralImm(MVT::f16);
462   }
463 
464   bool isVSrcV2F16() const {
465     llvm_unreachable("cannot happen");
466     return isVSrcF16();
467   }
468 
469   bool isKImmFP32() const {
470     return isLiteralImm(MVT::f32);
471   }
472 
473   bool isKImmFP16() const {
474     return isLiteralImm(MVT::f16);
475   }
476 
477   bool isMem() const override {
478     return false;
479   }
480 
481   bool isExpr() const {
482     return Kind == Expression;
483   }
484 
485   bool isSoppBrTarget() const {
486     return isExpr() || isImm();
487   }
488 
489   bool isSWaitCnt() const;
490   bool isHwreg() const;
491   bool isSendMsg() const;
492   bool isSwizzle() const;
493   bool isSMRDOffset8() const;
494   bool isSMRDOffset20() const;
495   bool isSMRDLiteralOffset() const;
496   bool isDPPCtrl() const;
497   bool isGPRIdxMode() const;
498   bool isS16Imm() const;
499   bool isU16Imm() const;
500 
501   StringRef getExpressionAsToken() const {
502     assert(isExpr());
503     const MCSymbolRefExpr *S = cast<MCSymbolRefExpr>(Expr);
504     return S->getSymbol().getName();
505   }
506 
507   StringRef getToken() const {
508     assert(isToken());
509 
510     if (Kind == Expression)
511       return getExpressionAsToken();
512 
513     return StringRef(Tok.Data, Tok.Length);
514   }
515 
516   int64_t getImm() const {
517     assert(isImm());
518     return Imm.Val;
519   }
520 
521   ImmTy getImmTy() const {
522     assert(isImm());
523     return Imm.Type;
524   }
525 
526   unsigned getReg() const override {
527     return Reg.RegNo;
528   }
529 
530   SMLoc getStartLoc() const override {
531     return StartLoc;
532   }
533 
534   SMLoc getEndLoc() const override {
535     return EndLoc;
536   }
537 
538   Modifiers getModifiers() const {
539     assert(isRegKind() || isImmTy(ImmTyNone));
540     return isRegKind() ? Reg.Mods : Imm.Mods;
541   }
542 
543   void setModifiers(Modifiers Mods) {
544     assert(isRegKind() || isImmTy(ImmTyNone));
545     if (isRegKind())
546       Reg.Mods = Mods;
547     else
548       Imm.Mods = Mods;
549   }
550 
551   bool hasModifiers() const {
552     return getModifiers().hasModifiers();
553   }
554 
555   bool hasFPModifiers() const {
556     return getModifiers().hasFPModifiers();
557   }
558 
559   bool hasIntModifiers() const {
560     return getModifiers().hasIntModifiers();
561   }
562 
563   uint64_t applyInputFPModifiers(uint64_t Val, unsigned Size) const;
564 
565   void addImmOperands(MCInst &Inst, unsigned N, bool ApplyModifiers = true) const;
566 
567   void addLiteralImmOperand(MCInst &Inst, int64_t Val, bool ApplyModifiers) const;
568 
569   template <unsigned Bitwidth>
570   void addKImmFPOperands(MCInst &Inst, unsigned N) const;
571 
572   void addKImmFP16Operands(MCInst &Inst, unsigned N) const {
573     addKImmFPOperands<16>(Inst, N);
574   }
575 
576   void addKImmFP32Operands(MCInst &Inst, unsigned N) const {
577     addKImmFPOperands<32>(Inst, N);
578   }
579 
580   void addRegOperands(MCInst &Inst, unsigned N) const;
581 
582   void addRegOrImmOperands(MCInst &Inst, unsigned N) const {
583     if (isRegKind())
584       addRegOperands(Inst, N);
585     else if (isExpr())
586       Inst.addOperand(MCOperand::createExpr(Expr));
587     else
588       addImmOperands(Inst, N);
589   }
590 
591   void addRegOrImmWithInputModsOperands(MCInst &Inst, unsigned N) const {
592     Modifiers Mods = getModifiers();
593     Inst.addOperand(MCOperand::createImm(Mods.getModifiersOperand()));
594     if (isRegKind()) {
595       addRegOperands(Inst, N);
596     } else {
597       addImmOperands(Inst, N, false);
598     }
599   }
600 
601   void addRegOrImmWithFPInputModsOperands(MCInst &Inst, unsigned N) const {
602     assert(!hasIntModifiers());
603     addRegOrImmWithInputModsOperands(Inst, N);
604   }
605 
606   void addRegOrImmWithIntInputModsOperands(MCInst &Inst, unsigned N) const {
607     assert(!hasFPModifiers());
608     addRegOrImmWithInputModsOperands(Inst, N);
609   }
610 
611   void addRegWithInputModsOperands(MCInst &Inst, unsigned N) const {
612     Modifiers Mods = getModifiers();
613     Inst.addOperand(MCOperand::createImm(Mods.getModifiersOperand()));
614     assert(isRegKind());
615     addRegOperands(Inst, N);
616   }
617 
618   void addRegWithFPInputModsOperands(MCInst &Inst, unsigned N) const {
619     assert(!hasIntModifiers());
620     addRegWithInputModsOperands(Inst, N);
621   }
622 
623   void addRegWithIntInputModsOperands(MCInst &Inst, unsigned N) const {
624     assert(!hasFPModifiers());
625     addRegWithInputModsOperands(Inst, N);
626   }
627 
628   void addSoppBrTargetOperands(MCInst &Inst, unsigned N) const {
629     if (isImm())
630       addImmOperands(Inst, N);
631     else {
632       assert(isExpr());
633       Inst.addOperand(MCOperand::createExpr(Expr));
634     }
635   }
636 
637   static void printImmTy(raw_ostream& OS, ImmTy Type) {
638     switch (Type) {
639     case ImmTyNone: OS << "None"; break;
640     case ImmTyGDS: OS << "GDS"; break;
641     case ImmTyOffen: OS << "Offen"; break;
642     case ImmTyIdxen: OS << "Idxen"; break;
643     case ImmTyAddr64: OS << "Addr64"; break;
644     case ImmTyOffset: OS << "Offset"; break;
645     case ImmTyOffset0: OS << "Offset0"; break;
646     case ImmTyOffset1: OS << "Offset1"; break;
647     case ImmTyGLC: OS << "GLC"; break;
648     case ImmTySLC: OS << "SLC"; break;
649     case ImmTyTFE: OS << "TFE"; break;
650     case ImmTyDFMT: OS << "DFMT"; break;
651     case ImmTyNFMT: OS << "NFMT"; break;
652     case ImmTyClampSI: OS << "ClampSI"; break;
653     case ImmTyOModSI: OS << "OModSI"; break;
654     case ImmTyDppCtrl: OS << "DppCtrl"; break;
655     case ImmTyDppRowMask: OS << "DppRowMask"; break;
656     case ImmTyDppBankMask: OS << "DppBankMask"; break;
657     case ImmTyDppBoundCtrl: OS << "DppBoundCtrl"; break;
658     case ImmTySdwaDstSel: OS << "SdwaDstSel"; break;
659     case ImmTySdwaSrc0Sel: OS << "SdwaSrc0Sel"; break;
660     case ImmTySdwaSrc1Sel: OS << "SdwaSrc1Sel"; break;
661     case ImmTySdwaDstUnused: OS << "SdwaDstUnused"; break;
662     case ImmTyDMask: OS << "DMask"; break;
663     case ImmTyUNorm: OS << "UNorm"; break;
664     case ImmTyDA: OS << "DA"; break;
665     case ImmTyR128: OS << "R128"; break;
666     case ImmTyLWE: OS << "LWE"; break;
667     case ImmTyOff: OS << "Off"; break;
668     case ImmTyExpTgt: OS << "ExpTgt"; break;
669     case ImmTyExpCompr: OS << "ExpCompr"; break;
670     case ImmTyExpVM: OS << "ExpVM"; break;
671     case ImmTyHwreg: OS << "Hwreg"; break;
672     case ImmTySendMsg: OS << "SendMsg"; break;
673     case ImmTyInterpSlot: OS << "InterpSlot"; break;
674     case ImmTyInterpAttr: OS << "InterpAttr"; break;
675     case ImmTyAttrChan: OS << "AttrChan"; break;
676     case ImmTyOpSel: OS << "OpSel"; break;
677     case ImmTyOpSelHi: OS << "OpSelHi"; break;
678     case ImmTyNegLo: OS << "NegLo"; break;
679     case ImmTyNegHi: OS << "NegHi"; break;
680     case ImmTySwizzle: OS << "Swizzle"; break;
681     case ImmTyHigh: OS << "High"; break;
682     }
683   }
684 
685   void print(raw_ostream &OS) const override {
686     switch (Kind) {
687     case Register:
688       OS << "<register " << getReg() << " mods: " << Reg.Mods << '>';
689       break;
690     case Immediate:
691       OS << '<' << getImm();
692       if (getImmTy() != ImmTyNone) {
693         OS << " type: "; printImmTy(OS, getImmTy());
694       }
695       OS << " mods: " << Imm.Mods << '>';
696       break;
697     case Token:
698       OS << '\'' << getToken() << '\'';
699       break;
700     case Expression:
701       OS << "<expr " << *Expr << '>';
702       break;
703     }
704   }
705 
706   static AMDGPUOperand::Ptr CreateImm(const AMDGPUAsmParser *AsmParser,
707                                       int64_t Val, SMLoc Loc,
708                                       ImmTy Type = ImmTyNone,
709                                       bool IsFPImm = false) {
710     auto Op = llvm::make_unique<AMDGPUOperand>(Immediate, AsmParser);
711     Op->Imm.Val = Val;
712     Op->Imm.IsFPImm = IsFPImm;
713     Op->Imm.Type = Type;
714     Op->Imm.Mods = Modifiers();
715     Op->StartLoc = Loc;
716     Op->EndLoc = Loc;
717     return Op;
718   }
719 
720   static AMDGPUOperand::Ptr CreateToken(const AMDGPUAsmParser *AsmParser,
721                                         StringRef Str, SMLoc Loc,
722                                         bool HasExplicitEncodingSize = true) {
723     auto Res = llvm::make_unique<AMDGPUOperand>(Token, AsmParser);
724     Res->Tok.Data = Str.data();
725     Res->Tok.Length = Str.size();
726     Res->StartLoc = Loc;
727     Res->EndLoc = Loc;
728     return Res;
729   }
730 
731   static AMDGPUOperand::Ptr CreateReg(const AMDGPUAsmParser *AsmParser,
732                                       unsigned RegNo, SMLoc S,
733                                       SMLoc E,
734                                       bool ForceVOP3) {
735     auto Op = llvm::make_unique<AMDGPUOperand>(Register, AsmParser);
736     Op->Reg.RegNo = RegNo;
737     Op->Reg.Mods = Modifiers();
738     Op->Reg.IsForcedVOP3 = ForceVOP3;
739     Op->StartLoc = S;
740     Op->EndLoc = E;
741     return Op;
742   }
743 
744   static AMDGPUOperand::Ptr CreateExpr(const AMDGPUAsmParser *AsmParser,
745                                        const class MCExpr *Expr, SMLoc S) {
746     auto Op = llvm::make_unique<AMDGPUOperand>(Expression, AsmParser);
747     Op->Expr = Expr;
748     Op->StartLoc = S;
749     Op->EndLoc = S;
750     return Op;
751   }
752 };
753 
754 raw_ostream &operator <<(raw_ostream &OS, AMDGPUOperand::Modifiers Mods) {
755   OS << "abs:" << Mods.Abs << " neg: " << Mods.Neg << " sext:" << Mods.Sext;
756   return OS;
757 }
758 
759 //===----------------------------------------------------------------------===//
760 // AsmParser
761 //===----------------------------------------------------------------------===//
762 
763 // Holds info related to the current kernel, e.g. count of SGPRs used.
764 // Kernel scope begins at .amdgpu_hsa_kernel directive, ends at next
765 // .amdgpu_hsa_kernel or at EOF.
766 class KernelScopeInfo {
767   int SgprIndexUnusedMin = -1;
768   int VgprIndexUnusedMin = -1;
769   MCContext *Ctx = nullptr;
770 
771   void usesSgprAt(int i) {
772     if (i >= SgprIndexUnusedMin) {
773       SgprIndexUnusedMin = ++i;
774       if (Ctx) {
775         MCSymbol * const Sym = Ctx->getOrCreateSymbol(Twine(".kernel.sgpr_count"));
776         Sym->setVariableValue(MCConstantExpr::create(SgprIndexUnusedMin, *Ctx));
777       }
778     }
779   }
780 
781   void usesVgprAt(int i) {
782     if (i >= VgprIndexUnusedMin) {
783       VgprIndexUnusedMin = ++i;
784       if (Ctx) {
785         MCSymbol * const Sym = Ctx->getOrCreateSymbol(Twine(".kernel.vgpr_count"));
786         Sym->setVariableValue(MCConstantExpr::create(VgprIndexUnusedMin, *Ctx));
787       }
788     }
789   }
790 
791 public:
792   KernelScopeInfo() = default;
793 
794   void initialize(MCContext &Context) {
795     Ctx = &Context;
796     usesSgprAt(SgprIndexUnusedMin = -1);
797     usesVgprAt(VgprIndexUnusedMin = -1);
798   }
799 
800   void usesRegister(RegisterKind RegKind, unsigned DwordRegIndex, unsigned RegWidth) {
801     switch (RegKind) {
802       case IS_SGPR: usesSgprAt(DwordRegIndex + RegWidth - 1); break;
803       case IS_VGPR: usesVgprAt(DwordRegIndex + RegWidth - 1); break;
804       default: break;
805     }
806   }
807 };
808 
809 class AMDGPUAsmParser : public MCTargetAsmParser {
810   MCAsmParser &Parser;
811 
812   unsigned ForcedEncodingSize = 0;
813   bool ForcedDPP = false;
814   bool ForcedSDWA = false;
815   KernelScopeInfo KernelScope;
816 
817   /// @name Auto-generated Match Functions
818   /// {
819 
820 #define GET_ASSEMBLER_HEADER
821 #include "AMDGPUGenAsmMatcher.inc"
822 
823   /// }
824 
825 private:
826   bool ParseAsAbsoluteExpression(uint32_t &Ret);
827   bool ParseDirectiveMajorMinor(uint32_t &Major, uint32_t &Minor);
828   bool ParseDirectiveHSACodeObjectVersion();
829   bool ParseDirectiveHSACodeObjectISA();
830   bool ParseAMDKernelCodeTValue(StringRef ID, amd_kernel_code_t &Header);
831   bool ParseDirectiveAMDKernelCodeT();
832   bool subtargetHasRegister(const MCRegisterInfo &MRI, unsigned RegNo) const;
833   bool ParseDirectiveAMDGPUHsaKernel();
834 
835   bool ParseDirectiveHSAMetadata();
836   bool ParseDirectivePALMetadata();
837 
838   bool AddNextRegisterToList(unsigned& Reg, unsigned& RegWidth,
839                              RegisterKind RegKind, unsigned Reg1,
840                              unsigned RegNum);
841   bool ParseAMDGPURegister(RegisterKind& RegKind, unsigned& Reg,
842                            unsigned& RegNum, unsigned& RegWidth,
843                            unsigned *DwordRegIndex);
844   void cvtMubufImpl(MCInst &Inst, const OperandVector &Operands,
845                     bool IsAtomic, bool IsAtomicReturn);
846   void cvtDSImpl(MCInst &Inst, const OperandVector &Operands,
847                  bool IsGdsHardcoded);
848 
849 public:
850   enum AMDGPUMatchResultTy {
851     Match_PreferE32 = FIRST_TARGET_MATCH_RESULT_TY
852   };
853 
854   using OptionalImmIndexMap = std::map<AMDGPUOperand::ImmTy, unsigned>;
855 
856   AMDGPUAsmParser(const MCSubtargetInfo &STI, MCAsmParser &_Parser,
857                const MCInstrInfo &MII,
858                const MCTargetOptions &Options)
859       : MCTargetAsmParser(Options, STI, MII), Parser(_Parser) {
860     MCAsmParserExtension::Initialize(Parser);
861 
862     if (getFeatureBits().none()) {
863       // Set default features.
864       copySTI().ToggleFeature("SOUTHERN_ISLANDS");
865     }
866 
867     setAvailableFeatures(ComputeAvailableFeatures(getFeatureBits()));
868 
869     {
870       // TODO: make those pre-defined variables read-only.
871       // Currently there is none suitable machinery in the core llvm-mc for this.
872       // MCSymbol::isRedefinable is intended for another purpose, and
873       // AsmParser::parseDirectiveSet() cannot be specialized for specific target.
874       AMDGPU::IsaInfo::IsaVersion ISA =
875           AMDGPU::IsaInfo::getIsaVersion(getFeatureBits());
876       MCContext &Ctx = getContext();
877       MCSymbol *Sym =
878           Ctx.getOrCreateSymbol(Twine(".option.machine_version_major"));
879       Sym->setVariableValue(MCConstantExpr::create(ISA.Major, Ctx));
880       Sym = Ctx.getOrCreateSymbol(Twine(".option.machine_version_minor"));
881       Sym->setVariableValue(MCConstantExpr::create(ISA.Minor, Ctx));
882       Sym = Ctx.getOrCreateSymbol(Twine(".option.machine_version_stepping"));
883       Sym->setVariableValue(MCConstantExpr::create(ISA.Stepping, Ctx));
884     }
885     KernelScope.initialize(getContext());
886   }
887 
888   bool isSI() const {
889     return AMDGPU::isSI(getSTI());
890   }
891 
892   bool isCI() const {
893     return AMDGPU::isCI(getSTI());
894   }
895 
896   bool isVI() const {
897     return AMDGPU::isVI(getSTI());
898   }
899 
900   bool isGFX9() const {
901     return AMDGPU::isGFX9(getSTI());
902   }
903 
904   bool hasInv2PiInlineImm() const {
905     return getFeatureBits()[AMDGPU::FeatureInv2PiInlineImm];
906   }
907 
908   bool hasFlatOffsets() const {
909     return getFeatureBits()[AMDGPU::FeatureFlatInstOffsets];
910   }
911 
912   bool hasSGPR102_SGPR103() const {
913     return !isVI();
914   }
915 
916   bool hasIntClamp() const {
917     return getFeatureBits()[AMDGPU::FeatureIntClamp];
918   }
919 
920   AMDGPUTargetStreamer &getTargetStreamer() {
921     MCTargetStreamer &TS = *getParser().getStreamer().getTargetStreamer();
922     return static_cast<AMDGPUTargetStreamer &>(TS);
923   }
924 
925   const MCRegisterInfo *getMRI() const {
926     // We need this const_cast because for some reason getContext() is not const
927     // in MCAsmParser.
928     return const_cast<AMDGPUAsmParser*>(this)->getContext().getRegisterInfo();
929   }
930 
931   const MCInstrInfo *getMII() const {
932     return &MII;
933   }
934 
935   const FeatureBitset &getFeatureBits() const {
936     return getSTI().getFeatureBits();
937   }
938 
939   void setForcedEncodingSize(unsigned Size) { ForcedEncodingSize = Size; }
940   void setForcedDPP(bool ForceDPP_) { ForcedDPP = ForceDPP_; }
941   void setForcedSDWA(bool ForceSDWA_) { ForcedSDWA = ForceSDWA_; }
942 
943   unsigned getForcedEncodingSize() const { return ForcedEncodingSize; }
944   bool isForcedVOP3() const { return ForcedEncodingSize == 64; }
945   bool isForcedDPP() const { return ForcedDPP; }
946   bool isForcedSDWA() const { return ForcedSDWA; }
947   ArrayRef<unsigned> getMatchedVariants() const;
948 
949   std::unique_ptr<AMDGPUOperand> parseRegister();
950   bool ParseRegister(unsigned &RegNo, SMLoc &StartLoc, SMLoc &EndLoc) override;
951   unsigned checkTargetMatchPredicate(MCInst &Inst) override;
952   unsigned validateTargetOperandClass(MCParsedAsmOperand &Op,
953                                       unsigned Kind) override;
954   bool MatchAndEmitInstruction(SMLoc IDLoc, unsigned &Opcode,
955                                OperandVector &Operands, MCStreamer &Out,
956                                uint64_t &ErrorInfo,
957                                bool MatchingInlineAsm) override;
958   bool ParseDirective(AsmToken DirectiveID) override;
959   OperandMatchResultTy parseOperand(OperandVector &Operands, StringRef Mnemonic);
960   StringRef parseMnemonicSuffix(StringRef Name);
961   bool ParseInstruction(ParseInstructionInfo &Info, StringRef Name,
962                         SMLoc NameLoc, OperandVector &Operands) override;
963   //bool ProcessInstruction(MCInst &Inst);
964 
965   OperandMatchResultTy parseIntWithPrefix(const char *Prefix, int64_t &Int);
966 
967   OperandMatchResultTy
968   parseIntWithPrefix(const char *Prefix, OperandVector &Operands,
969                      AMDGPUOperand::ImmTy ImmTy = AMDGPUOperand::ImmTyNone,
970                      bool (*ConvertResult)(int64_t &) = nullptr);
971 
972   OperandMatchResultTy parseOperandArrayWithPrefix(
973     const char *Prefix,
974     OperandVector &Operands,
975     AMDGPUOperand::ImmTy ImmTy = AMDGPUOperand::ImmTyNone,
976     bool (*ConvertResult)(int64_t&) = nullptr);
977 
978   OperandMatchResultTy
979   parseNamedBit(const char *Name, OperandVector &Operands,
980                 AMDGPUOperand::ImmTy ImmTy = AMDGPUOperand::ImmTyNone);
981   OperandMatchResultTy parseStringWithPrefix(StringRef Prefix,
982                                              StringRef &Value);
983 
984   bool parseAbsoluteExpr(int64_t &Val, bool AbsMod = false);
985   OperandMatchResultTy parseImm(OperandVector &Operands, bool AbsMod = false);
986   OperandMatchResultTy parseReg(OperandVector &Operands);
987   OperandMatchResultTy parseRegOrImm(OperandVector &Operands, bool AbsMod = false);
988   OperandMatchResultTy parseRegOrImmWithFPInputMods(OperandVector &Operands, bool AllowImm = true);
989   OperandMatchResultTy parseRegOrImmWithIntInputMods(OperandVector &Operands, bool AllowImm = true);
990   OperandMatchResultTy parseRegWithFPInputMods(OperandVector &Operands);
991   OperandMatchResultTy parseRegWithIntInputMods(OperandVector &Operands);
992   OperandMatchResultTy parseVReg32OrOff(OperandVector &Operands);
993 
994   void cvtDSOffset01(MCInst &Inst, const OperandVector &Operands);
995   void cvtDS(MCInst &Inst, const OperandVector &Operands) { cvtDSImpl(Inst, Operands, false); }
996   void cvtDSGds(MCInst &Inst, const OperandVector &Operands) { cvtDSImpl(Inst, Operands, true); }
997   void cvtExp(MCInst &Inst, const OperandVector &Operands);
998 
999   bool parseCnt(int64_t &IntVal);
1000   OperandMatchResultTy parseSWaitCntOps(OperandVector &Operands);
1001   OperandMatchResultTy parseHwreg(OperandVector &Operands);
1002 
1003 private:
1004   struct OperandInfoTy {
1005     int64_t Id;
1006     bool IsSymbolic = false;
1007 
1008     OperandInfoTy(int64_t Id_) : Id(Id_) {}
1009   };
1010 
1011   bool parseSendMsgConstruct(OperandInfoTy &Msg, OperandInfoTy &Operation, int64_t &StreamId);
1012   bool parseHwregConstruct(OperandInfoTy &HwReg, int64_t &Offset, int64_t &Width);
1013 
1014   void errorExpTgt();
1015   OperandMatchResultTy parseExpTgtImpl(StringRef Str, uint8_t &Val);
1016 
1017   bool validateInstruction(const MCInst &Inst, const SMLoc &IDLoc);
1018   bool validateConstantBusLimitations(const MCInst &Inst);
1019   bool validateEarlyClobberLimitations(const MCInst &Inst);
1020   bool validateIntClampSupported(const MCInst &Inst);
1021   bool usesConstantBus(const MCInst &Inst, unsigned OpIdx);
1022   bool isInlineConstant(const MCInst &Inst, unsigned OpIdx) const;
1023   unsigned findImplicitSGPRReadInVOP(const MCInst &Inst) const;
1024 
1025   bool trySkipId(const StringRef Id);
1026   bool trySkipToken(const AsmToken::TokenKind Kind);
1027   bool skipToken(const AsmToken::TokenKind Kind, const StringRef ErrMsg);
1028   bool parseString(StringRef &Val, const StringRef ErrMsg = "expected a string");
1029   bool parseExpr(int64_t &Imm);
1030 
1031 public:
1032   OperandMatchResultTy parseOptionalOperand(OperandVector &Operands);
1033 
1034   OperandMatchResultTy parseExpTgt(OperandVector &Operands);
1035   OperandMatchResultTy parseSendMsgOp(OperandVector &Operands);
1036   OperandMatchResultTy parseInterpSlot(OperandVector &Operands);
1037   OperandMatchResultTy parseInterpAttr(OperandVector &Operands);
1038   OperandMatchResultTy parseSOppBrTarget(OperandVector &Operands);
1039 
1040   bool parseSwizzleOperands(const unsigned OpNum, int64_t* Op,
1041                             const unsigned MinVal,
1042                             const unsigned MaxVal,
1043                             const StringRef ErrMsg);
1044   OperandMatchResultTy parseSwizzleOp(OperandVector &Operands);
1045   bool parseSwizzleOffset(int64_t &Imm);
1046   bool parseSwizzleMacro(int64_t &Imm);
1047   bool parseSwizzleQuadPerm(int64_t &Imm);
1048   bool parseSwizzleBitmaskPerm(int64_t &Imm);
1049   bool parseSwizzleBroadcast(int64_t &Imm);
1050   bool parseSwizzleSwap(int64_t &Imm);
1051   bool parseSwizzleReverse(int64_t &Imm);
1052 
1053   void cvtMubuf(MCInst &Inst, const OperandVector &Operands) { cvtMubufImpl(Inst, Operands, false, false); }
1054   void cvtMubufAtomic(MCInst &Inst, const OperandVector &Operands) { cvtMubufImpl(Inst, Operands, true, false); }
1055   void cvtMubufAtomicReturn(MCInst &Inst, const OperandVector &Operands) { cvtMubufImpl(Inst, Operands, true, true); }
1056   void cvtMtbuf(MCInst &Inst, const OperandVector &Operands);
1057 
1058   AMDGPUOperand::Ptr defaultGLC() const;
1059   AMDGPUOperand::Ptr defaultSLC() const;
1060   AMDGPUOperand::Ptr defaultTFE() const;
1061 
1062   AMDGPUOperand::Ptr defaultDMask() const;
1063   AMDGPUOperand::Ptr defaultUNorm() const;
1064   AMDGPUOperand::Ptr defaultDA() const;
1065   AMDGPUOperand::Ptr defaultR128() const;
1066   AMDGPUOperand::Ptr defaultLWE() const;
1067   AMDGPUOperand::Ptr defaultSMRDOffset8() const;
1068   AMDGPUOperand::Ptr defaultSMRDOffset20() const;
1069   AMDGPUOperand::Ptr defaultSMRDLiteralOffset() const;
1070   AMDGPUOperand::Ptr defaultOffsetU12() const;
1071   AMDGPUOperand::Ptr defaultOffsetS13() const;
1072 
1073   OperandMatchResultTy parseOModOperand(OperandVector &Operands);
1074 
1075   void cvtVOP3(MCInst &Inst, const OperandVector &Operands,
1076                OptionalImmIndexMap &OptionalIdx);
1077   void cvtVOP3OpSel(MCInst &Inst, const OperandVector &Operands);
1078   void cvtVOP3(MCInst &Inst, const OperandVector &Operands);
1079   void cvtVOP3PImpl(MCInst &Inst, const OperandVector &Operands,
1080                     bool IsPacked);
1081   void cvtVOP3P(MCInst &Inst, const OperandVector &Operands);
1082   void cvtVOP3P_NotPacked(MCInst &Inst, const OperandVector &Operands);
1083 
1084   void cvtVOP3Interp(MCInst &Inst, const OperandVector &Operands);
1085 
1086   void cvtMIMG(MCInst &Inst, const OperandVector &Operands,
1087                bool IsAtomic = false);
1088   void cvtMIMGAtomic(MCInst &Inst, const OperandVector &Operands);
1089 
1090   OperandMatchResultTy parseDPPCtrl(OperandVector &Operands);
1091   AMDGPUOperand::Ptr defaultRowMask() const;
1092   AMDGPUOperand::Ptr defaultBankMask() const;
1093   AMDGPUOperand::Ptr defaultBoundCtrl() const;
1094   void cvtDPP(MCInst &Inst, const OperandVector &Operands);
1095 
1096   OperandMatchResultTy parseSDWASel(OperandVector &Operands, StringRef Prefix,
1097                                     AMDGPUOperand::ImmTy Type);
1098   OperandMatchResultTy parseSDWADstUnused(OperandVector &Operands);
1099   void cvtSdwaVOP1(MCInst &Inst, const OperandVector &Operands);
1100   void cvtSdwaVOP2(MCInst &Inst, const OperandVector &Operands);
1101   void cvtSdwaVOP2b(MCInst &Inst, const OperandVector &Operands);
1102   void cvtSdwaVOPC(MCInst &Inst, const OperandVector &Operands);
1103   void cvtSDWA(MCInst &Inst, const OperandVector &Operands,
1104                 uint64_t BasicInstType, bool skipVcc = false);
1105 };
1106 
1107 struct OptionalOperand {
1108   const char *Name;
1109   AMDGPUOperand::ImmTy Type;
1110   bool IsBit;
1111   bool (*ConvertResult)(int64_t&);
1112 };
1113 
1114 } // end anonymous namespace
1115 
1116 // May be called with integer type with equivalent bitwidth.
1117 static const fltSemantics *getFltSemantics(unsigned Size) {
1118   switch (Size) {
1119   case 4:
1120     return &APFloat::IEEEsingle();
1121   case 8:
1122     return &APFloat::IEEEdouble();
1123   case 2:
1124     return &APFloat::IEEEhalf();
1125   default:
1126     llvm_unreachable("unsupported fp type");
1127   }
1128 }
1129 
1130 static const fltSemantics *getFltSemantics(MVT VT) {
1131   return getFltSemantics(VT.getSizeInBits() / 8);
1132 }
1133 
1134 static const fltSemantics *getOpFltSemantics(uint8_t OperandType) {
1135   switch (OperandType) {
1136   case AMDGPU::OPERAND_REG_IMM_INT32:
1137   case AMDGPU::OPERAND_REG_IMM_FP32:
1138   case AMDGPU::OPERAND_REG_INLINE_C_INT32:
1139   case AMDGPU::OPERAND_REG_INLINE_C_FP32:
1140     return &APFloat::IEEEsingle();
1141   case AMDGPU::OPERAND_REG_IMM_INT64:
1142   case AMDGPU::OPERAND_REG_IMM_FP64:
1143   case AMDGPU::OPERAND_REG_INLINE_C_INT64:
1144   case AMDGPU::OPERAND_REG_INLINE_C_FP64:
1145     return &APFloat::IEEEdouble();
1146   case AMDGPU::OPERAND_REG_IMM_INT16:
1147   case AMDGPU::OPERAND_REG_IMM_FP16:
1148   case AMDGPU::OPERAND_REG_INLINE_C_INT16:
1149   case AMDGPU::OPERAND_REG_INLINE_C_FP16:
1150   case AMDGPU::OPERAND_REG_INLINE_C_V2INT16:
1151   case AMDGPU::OPERAND_REG_INLINE_C_V2FP16:
1152     return &APFloat::IEEEhalf();
1153   default:
1154     llvm_unreachable("unsupported fp type");
1155   }
1156 }
1157 
1158 //===----------------------------------------------------------------------===//
1159 // Operand
1160 //===----------------------------------------------------------------------===//
1161 
1162 static bool canLosslesslyConvertToFPType(APFloat &FPLiteral, MVT VT) {
1163   bool Lost;
1164 
1165   // Convert literal to single precision
1166   APFloat::opStatus Status = FPLiteral.convert(*getFltSemantics(VT),
1167                                                APFloat::rmNearestTiesToEven,
1168                                                &Lost);
1169   // We allow precision lost but not overflow or underflow
1170   if (Status != APFloat::opOK &&
1171       Lost &&
1172       ((Status & APFloat::opOverflow)  != 0 ||
1173        (Status & APFloat::opUnderflow) != 0)) {
1174     return false;
1175   }
1176 
1177   return true;
1178 }
1179 
1180 bool AMDGPUOperand::isInlinableImm(MVT type) const {
1181   if (!isImmTy(ImmTyNone)) {
1182     // Only plain immediates are inlinable (e.g. "clamp" attribute is not)
1183     return false;
1184   }
1185   // TODO: We should avoid using host float here. It would be better to
1186   // check the float bit values which is what a few other places do.
1187   // We've had bot failures before due to weird NaN support on mips hosts.
1188 
1189   APInt Literal(64, Imm.Val);
1190 
1191   if (Imm.IsFPImm) { // We got fp literal token
1192     if (type == MVT::f64 || type == MVT::i64) { // Expected 64-bit operand
1193       return AMDGPU::isInlinableLiteral64(Imm.Val,
1194                                           AsmParser->hasInv2PiInlineImm());
1195     }
1196 
1197     APFloat FPLiteral(APFloat::IEEEdouble(), APInt(64, Imm.Val));
1198     if (!canLosslesslyConvertToFPType(FPLiteral, type))
1199       return false;
1200 
1201     if (type.getScalarSizeInBits() == 16) {
1202       return AMDGPU::isInlinableLiteral16(
1203         static_cast<int16_t>(FPLiteral.bitcastToAPInt().getZExtValue()),
1204         AsmParser->hasInv2PiInlineImm());
1205     }
1206 
1207     // Check if single precision literal is inlinable
1208     return AMDGPU::isInlinableLiteral32(
1209       static_cast<int32_t>(FPLiteral.bitcastToAPInt().getZExtValue()),
1210       AsmParser->hasInv2PiInlineImm());
1211   }
1212 
1213   // We got int literal token.
1214   if (type == MVT::f64 || type == MVT::i64) { // Expected 64-bit operand
1215     return AMDGPU::isInlinableLiteral64(Imm.Val,
1216                                         AsmParser->hasInv2PiInlineImm());
1217   }
1218 
1219   if (type.getScalarSizeInBits() == 16) {
1220     return AMDGPU::isInlinableLiteral16(
1221       static_cast<int16_t>(Literal.getLoBits(16).getSExtValue()),
1222       AsmParser->hasInv2PiInlineImm());
1223   }
1224 
1225   return AMDGPU::isInlinableLiteral32(
1226     static_cast<int32_t>(Literal.getLoBits(32).getZExtValue()),
1227     AsmParser->hasInv2PiInlineImm());
1228 }
1229 
1230 bool AMDGPUOperand::isLiteralImm(MVT type) const {
1231   // Check that this immediate can be added as literal
1232   if (!isImmTy(ImmTyNone)) {
1233     return false;
1234   }
1235 
1236   if (!Imm.IsFPImm) {
1237     // We got int literal token.
1238 
1239     if (type == MVT::f64 && hasFPModifiers()) {
1240       // Cannot apply fp modifiers to int literals preserving the same semantics
1241       // for VOP1/2/C and VOP3 because of integer truncation. To avoid ambiguity,
1242       // disable these cases.
1243       return false;
1244     }
1245 
1246     unsigned Size = type.getSizeInBits();
1247     if (Size == 64)
1248       Size = 32;
1249 
1250     // FIXME: 64-bit operands can zero extend, sign extend, or pad zeroes for FP
1251     // types.
1252     return isUIntN(Size, Imm.Val) || isIntN(Size, Imm.Val);
1253   }
1254 
1255   // We got fp literal token
1256   if (type == MVT::f64) { // Expected 64-bit fp operand
1257     // We would set low 64-bits of literal to zeroes but we accept this literals
1258     return true;
1259   }
1260 
1261   if (type == MVT::i64) { // Expected 64-bit int operand
1262     // We don't allow fp literals in 64-bit integer instructions. It is
1263     // unclear how we should encode them.
1264     return false;
1265   }
1266 
1267   APFloat FPLiteral(APFloat::IEEEdouble(), APInt(64, Imm.Val));
1268   return canLosslesslyConvertToFPType(FPLiteral, type);
1269 }
1270 
1271 bool AMDGPUOperand::isRegClass(unsigned RCID) const {
1272   return isRegKind() && AsmParser->getMRI()->getRegClass(RCID).contains(getReg());
1273 }
1274 
1275 bool AMDGPUOperand::isSDWARegKind() const {
1276   if (AsmParser->isVI())
1277     return isVReg();
1278   else if (AsmParser->isGFX9())
1279     return isRegKind();
1280   else
1281     return false;
1282 }
1283 
1284 uint64_t AMDGPUOperand::applyInputFPModifiers(uint64_t Val, unsigned Size) const
1285 {
1286   assert(isImmTy(ImmTyNone) && Imm.Mods.hasFPModifiers());
1287   assert(Size == 2 || Size == 4 || Size == 8);
1288 
1289   const uint64_t FpSignMask = (1ULL << (Size * 8 - 1));
1290 
1291   if (Imm.Mods.Abs) {
1292     Val &= ~FpSignMask;
1293   }
1294   if (Imm.Mods.Neg) {
1295     Val ^= FpSignMask;
1296   }
1297 
1298   return Val;
1299 }
1300 
1301 void AMDGPUOperand::addImmOperands(MCInst &Inst, unsigned N, bool ApplyModifiers) const {
1302   if (AMDGPU::isSISrcOperand(AsmParser->getMII()->get(Inst.getOpcode()),
1303                              Inst.getNumOperands())) {
1304     addLiteralImmOperand(Inst, Imm.Val,
1305                          ApplyModifiers &
1306                          isImmTy(ImmTyNone) && Imm.Mods.hasFPModifiers());
1307   } else {
1308     assert(!isImmTy(ImmTyNone) || !hasModifiers());
1309     Inst.addOperand(MCOperand::createImm(Imm.Val));
1310   }
1311 }
1312 
1313 void AMDGPUOperand::addLiteralImmOperand(MCInst &Inst, int64_t Val, bool ApplyModifiers) const {
1314   const auto& InstDesc = AsmParser->getMII()->get(Inst.getOpcode());
1315   auto OpNum = Inst.getNumOperands();
1316   // Check that this operand accepts literals
1317   assert(AMDGPU::isSISrcOperand(InstDesc, OpNum));
1318 
1319   if (ApplyModifiers) {
1320     assert(AMDGPU::isSISrcFPOperand(InstDesc, OpNum));
1321     const unsigned Size = Imm.IsFPImm ? sizeof(double) : getOperandSize(InstDesc, OpNum);
1322     Val = applyInputFPModifiers(Val, Size);
1323   }
1324 
1325   APInt Literal(64, Val);
1326   uint8_t OpTy = InstDesc.OpInfo[OpNum].OperandType;
1327 
1328   if (Imm.IsFPImm) { // We got fp literal token
1329     switch (OpTy) {
1330     case AMDGPU::OPERAND_REG_IMM_INT64:
1331     case AMDGPU::OPERAND_REG_IMM_FP64:
1332     case AMDGPU::OPERAND_REG_INLINE_C_INT64:
1333     case AMDGPU::OPERAND_REG_INLINE_C_FP64:
1334       if (AMDGPU::isInlinableLiteral64(Literal.getZExtValue(),
1335                                        AsmParser->hasInv2PiInlineImm())) {
1336         Inst.addOperand(MCOperand::createImm(Literal.getZExtValue()));
1337         return;
1338       }
1339 
1340       // Non-inlineable
1341       if (AMDGPU::isSISrcFPOperand(InstDesc, OpNum)) { // Expected 64-bit fp operand
1342         // For fp operands we check if low 32 bits are zeros
1343         if (Literal.getLoBits(32) != 0) {
1344           const_cast<AMDGPUAsmParser *>(AsmParser)->Warning(Inst.getLoc(),
1345           "Can't encode literal as exact 64-bit floating-point operand. "
1346           "Low 32-bits will be set to zero");
1347         }
1348 
1349         Inst.addOperand(MCOperand::createImm(Literal.lshr(32).getZExtValue()));
1350         return;
1351       }
1352 
1353       // We don't allow fp literals in 64-bit integer instructions. It is
1354       // unclear how we should encode them. This case should be checked earlier
1355       // in predicate methods (isLiteralImm())
1356       llvm_unreachable("fp literal in 64-bit integer instruction.");
1357 
1358     case AMDGPU::OPERAND_REG_IMM_INT32:
1359     case AMDGPU::OPERAND_REG_IMM_FP32:
1360     case AMDGPU::OPERAND_REG_INLINE_C_INT32:
1361     case AMDGPU::OPERAND_REG_INLINE_C_FP32:
1362     case AMDGPU::OPERAND_REG_IMM_INT16:
1363     case AMDGPU::OPERAND_REG_IMM_FP16:
1364     case AMDGPU::OPERAND_REG_INLINE_C_INT16:
1365     case AMDGPU::OPERAND_REG_INLINE_C_FP16:
1366     case AMDGPU::OPERAND_REG_INLINE_C_V2INT16:
1367     case AMDGPU::OPERAND_REG_INLINE_C_V2FP16: {
1368       bool lost;
1369       APFloat FPLiteral(APFloat::IEEEdouble(), Literal);
1370       // Convert literal to single precision
1371       FPLiteral.convert(*getOpFltSemantics(OpTy),
1372                         APFloat::rmNearestTiesToEven, &lost);
1373       // We allow precision lost but not overflow or underflow. This should be
1374       // checked earlier in isLiteralImm()
1375 
1376       uint64_t ImmVal = FPLiteral.bitcastToAPInt().getZExtValue();
1377       if (OpTy == AMDGPU::OPERAND_REG_INLINE_C_V2INT16 ||
1378           OpTy == AMDGPU::OPERAND_REG_INLINE_C_V2FP16) {
1379         ImmVal |= (ImmVal << 16);
1380       }
1381 
1382       Inst.addOperand(MCOperand::createImm(ImmVal));
1383       return;
1384     }
1385     default:
1386       llvm_unreachable("invalid operand size");
1387     }
1388 
1389     return;
1390   }
1391 
1392    // We got int literal token.
1393   // Only sign extend inline immediates.
1394   // FIXME: No errors on truncation
1395   switch (OpTy) {
1396   case AMDGPU::OPERAND_REG_IMM_INT32:
1397   case AMDGPU::OPERAND_REG_IMM_FP32:
1398   case AMDGPU::OPERAND_REG_INLINE_C_INT32:
1399   case AMDGPU::OPERAND_REG_INLINE_C_FP32:
1400     if (isInt<32>(Val) &&
1401         AMDGPU::isInlinableLiteral32(static_cast<int32_t>(Val),
1402                                      AsmParser->hasInv2PiInlineImm())) {
1403       Inst.addOperand(MCOperand::createImm(Val));
1404       return;
1405     }
1406 
1407     Inst.addOperand(MCOperand::createImm(Val & 0xffffffff));
1408     return;
1409 
1410   case AMDGPU::OPERAND_REG_IMM_INT64:
1411   case AMDGPU::OPERAND_REG_IMM_FP64:
1412   case AMDGPU::OPERAND_REG_INLINE_C_INT64:
1413   case AMDGPU::OPERAND_REG_INLINE_C_FP64:
1414     if (AMDGPU::isInlinableLiteral64(Val, AsmParser->hasInv2PiInlineImm())) {
1415       Inst.addOperand(MCOperand::createImm(Val));
1416       return;
1417     }
1418 
1419     Inst.addOperand(MCOperand::createImm(Lo_32(Val)));
1420     return;
1421 
1422   case AMDGPU::OPERAND_REG_IMM_INT16:
1423   case AMDGPU::OPERAND_REG_IMM_FP16:
1424   case AMDGPU::OPERAND_REG_INLINE_C_INT16:
1425   case AMDGPU::OPERAND_REG_INLINE_C_FP16:
1426     if (isInt<16>(Val) &&
1427         AMDGPU::isInlinableLiteral16(static_cast<int16_t>(Val),
1428                                      AsmParser->hasInv2PiInlineImm())) {
1429       Inst.addOperand(MCOperand::createImm(Val));
1430       return;
1431     }
1432 
1433     Inst.addOperand(MCOperand::createImm(Val & 0xffff));
1434     return;
1435 
1436   case AMDGPU::OPERAND_REG_INLINE_C_V2INT16:
1437   case AMDGPU::OPERAND_REG_INLINE_C_V2FP16: {
1438     auto LiteralVal = static_cast<uint16_t>(Literal.getLoBits(16).getZExtValue());
1439     assert(AMDGPU::isInlinableLiteral16(LiteralVal,
1440                                         AsmParser->hasInv2PiInlineImm()));
1441 
1442     uint32_t ImmVal = static_cast<uint32_t>(LiteralVal) << 16 |
1443                       static_cast<uint32_t>(LiteralVal);
1444     Inst.addOperand(MCOperand::createImm(ImmVal));
1445     return;
1446   }
1447   default:
1448     llvm_unreachable("invalid operand size");
1449   }
1450 }
1451 
1452 template <unsigned Bitwidth>
1453 void AMDGPUOperand::addKImmFPOperands(MCInst &Inst, unsigned N) const {
1454   APInt Literal(64, Imm.Val);
1455 
1456   if (!Imm.IsFPImm) {
1457     // We got int literal token.
1458     Inst.addOperand(MCOperand::createImm(Literal.getLoBits(Bitwidth).getZExtValue()));
1459     return;
1460   }
1461 
1462   bool Lost;
1463   APFloat FPLiteral(APFloat::IEEEdouble(), Literal);
1464   FPLiteral.convert(*getFltSemantics(Bitwidth / 8),
1465                     APFloat::rmNearestTiesToEven, &Lost);
1466   Inst.addOperand(MCOperand::createImm(FPLiteral.bitcastToAPInt().getZExtValue()));
1467 }
1468 
1469 void AMDGPUOperand::addRegOperands(MCInst &Inst, unsigned N) const {
1470   Inst.addOperand(MCOperand::createReg(AMDGPU::getMCReg(getReg(), AsmParser->getSTI())));
1471 }
1472 
1473 //===----------------------------------------------------------------------===//
1474 // AsmParser
1475 //===----------------------------------------------------------------------===//
1476 
1477 static int getRegClass(RegisterKind Is, unsigned RegWidth) {
1478   if (Is == IS_VGPR) {
1479     switch (RegWidth) {
1480       default: return -1;
1481       case 1: return AMDGPU::VGPR_32RegClassID;
1482       case 2: return AMDGPU::VReg_64RegClassID;
1483       case 3: return AMDGPU::VReg_96RegClassID;
1484       case 4: return AMDGPU::VReg_128RegClassID;
1485       case 8: return AMDGPU::VReg_256RegClassID;
1486       case 16: return AMDGPU::VReg_512RegClassID;
1487     }
1488   } else if (Is == IS_TTMP) {
1489     switch (RegWidth) {
1490       default: return -1;
1491       case 1: return AMDGPU::TTMP_32RegClassID;
1492       case 2: return AMDGPU::TTMP_64RegClassID;
1493       case 4: return AMDGPU::TTMP_128RegClassID;
1494     }
1495   } else if (Is == IS_SGPR) {
1496     switch (RegWidth) {
1497       default: return -1;
1498       case 1: return AMDGPU::SGPR_32RegClassID;
1499       case 2: return AMDGPU::SGPR_64RegClassID;
1500       case 4: return AMDGPU::SGPR_128RegClassID;
1501       case 8: return AMDGPU::SReg_256RegClassID;
1502       case 16: return AMDGPU::SReg_512RegClassID;
1503     }
1504   }
1505   return -1;
1506 }
1507 
1508 static unsigned getSpecialRegForName(StringRef RegName) {
1509   return StringSwitch<unsigned>(RegName)
1510     .Case("exec", AMDGPU::EXEC)
1511     .Case("vcc", AMDGPU::VCC)
1512     .Case("flat_scratch", AMDGPU::FLAT_SCR)
1513     .Case("m0", AMDGPU::M0)
1514     .Case("scc", AMDGPU::SCC)
1515     .Case("tba", AMDGPU::TBA)
1516     .Case("tma", AMDGPU::TMA)
1517     .Case("flat_scratch_lo", AMDGPU::FLAT_SCR_LO)
1518     .Case("flat_scratch_hi", AMDGPU::FLAT_SCR_HI)
1519     .Case("vcc_lo", AMDGPU::VCC_LO)
1520     .Case("vcc_hi", AMDGPU::VCC_HI)
1521     .Case("exec_lo", AMDGPU::EXEC_LO)
1522     .Case("exec_hi", AMDGPU::EXEC_HI)
1523     .Case("tma_lo", AMDGPU::TMA_LO)
1524     .Case("tma_hi", AMDGPU::TMA_HI)
1525     .Case("tba_lo", AMDGPU::TBA_LO)
1526     .Case("tba_hi", AMDGPU::TBA_HI)
1527     .Default(0);
1528 }
1529 
1530 bool AMDGPUAsmParser::ParseRegister(unsigned &RegNo, SMLoc &StartLoc,
1531                                     SMLoc &EndLoc) {
1532   auto R = parseRegister();
1533   if (!R) return true;
1534   assert(R->isReg());
1535   RegNo = R->getReg();
1536   StartLoc = R->getStartLoc();
1537   EndLoc = R->getEndLoc();
1538   return false;
1539 }
1540 
1541 bool AMDGPUAsmParser::AddNextRegisterToList(unsigned &Reg, unsigned &RegWidth,
1542                                             RegisterKind RegKind, unsigned Reg1,
1543                                             unsigned RegNum) {
1544   switch (RegKind) {
1545   case IS_SPECIAL:
1546     if (Reg == AMDGPU::EXEC_LO && Reg1 == AMDGPU::EXEC_HI) {
1547       Reg = AMDGPU::EXEC;
1548       RegWidth = 2;
1549       return true;
1550     }
1551     if (Reg == AMDGPU::FLAT_SCR_LO && Reg1 == AMDGPU::FLAT_SCR_HI) {
1552       Reg = AMDGPU::FLAT_SCR;
1553       RegWidth = 2;
1554       return true;
1555     }
1556     if (Reg == AMDGPU::VCC_LO && Reg1 == AMDGPU::VCC_HI) {
1557       Reg = AMDGPU::VCC;
1558       RegWidth = 2;
1559       return true;
1560     }
1561     if (Reg == AMDGPU::TBA_LO && Reg1 == AMDGPU::TBA_HI) {
1562       Reg = AMDGPU::TBA;
1563       RegWidth = 2;
1564       return true;
1565     }
1566     if (Reg == AMDGPU::TMA_LO && Reg1 == AMDGPU::TMA_HI) {
1567       Reg = AMDGPU::TMA;
1568       RegWidth = 2;
1569       return true;
1570     }
1571     return false;
1572   case IS_VGPR:
1573   case IS_SGPR:
1574   case IS_TTMP:
1575     if (Reg1 != Reg + RegWidth) {
1576       return false;
1577     }
1578     RegWidth++;
1579     return true;
1580   default:
1581     llvm_unreachable("unexpected register kind");
1582   }
1583 }
1584 
1585 bool AMDGPUAsmParser::ParseAMDGPURegister(RegisterKind &RegKind, unsigned &Reg,
1586                                           unsigned &RegNum, unsigned &RegWidth,
1587                                           unsigned *DwordRegIndex) {
1588   if (DwordRegIndex) { *DwordRegIndex = 0; }
1589   const MCRegisterInfo *TRI = getContext().getRegisterInfo();
1590   if (getLexer().is(AsmToken::Identifier)) {
1591     StringRef RegName = Parser.getTok().getString();
1592     if ((Reg = getSpecialRegForName(RegName))) {
1593       Parser.Lex();
1594       RegKind = IS_SPECIAL;
1595     } else {
1596       unsigned RegNumIndex = 0;
1597       if (RegName[0] == 'v') {
1598         RegNumIndex = 1;
1599         RegKind = IS_VGPR;
1600       } else if (RegName[0] == 's') {
1601         RegNumIndex = 1;
1602         RegKind = IS_SGPR;
1603       } else if (RegName.startswith("ttmp")) {
1604         RegNumIndex = strlen("ttmp");
1605         RegKind = IS_TTMP;
1606       } else {
1607         return false;
1608       }
1609       if (RegName.size() > RegNumIndex) {
1610         // Single 32-bit register: vXX.
1611         if (RegName.substr(RegNumIndex).getAsInteger(10, RegNum))
1612           return false;
1613         Parser.Lex();
1614         RegWidth = 1;
1615       } else {
1616         // Range of registers: v[XX:YY]. ":YY" is optional.
1617         Parser.Lex();
1618         int64_t RegLo, RegHi;
1619         if (getLexer().isNot(AsmToken::LBrac))
1620           return false;
1621         Parser.Lex();
1622 
1623         if (getParser().parseAbsoluteExpression(RegLo))
1624           return false;
1625 
1626         const bool isRBrace = getLexer().is(AsmToken::RBrac);
1627         if (!isRBrace && getLexer().isNot(AsmToken::Colon))
1628           return false;
1629         Parser.Lex();
1630 
1631         if (isRBrace) {
1632           RegHi = RegLo;
1633         } else {
1634           if (getParser().parseAbsoluteExpression(RegHi))
1635             return false;
1636 
1637           if (getLexer().isNot(AsmToken::RBrac))
1638             return false;
1639           Parser.Lex();
1640         }
1641         RegNum = (unsigned) RegLo;
1642         RegWidth = (RegHi - RegLo) + 1;
1643       }
1644     }
1645   } else if (getLexer().is(AsmToken::LBrac)) {
1646     // List of consecutive registers: [s0,s1,s2,s3]
1647     Parser.Lex();
1648     if (!ParseAMDGPURegister(RegKind, Reg, RegNum, RegWidth, nullptr))
1649       return false;
1650     if (RegWidth != 1)
1651       return false;
1652     RegisterKind RegKind1;
1653     unsigned Reg1, RegNum1, RegWidth1;
1654     do {
1655       if (getLexer().is(AsmToken::Comma)) {
1656         Parser.Lex();
1657       } else if (getLexer().is(AsmToken::RBrac)) {
1658         Parser.Lex();
1659         break;
1660       } else if (ParseAMDGPURegister(RegKind1, Reg1, RegNum1, RegWidth1, nullptr)) {
1661         if (RegWidth1 != 1) {
1662           return false;
1663         }
1664         if (RegKind1 != RegKind) {
1665           return false;
1666         }
1667         if (!AddNextRegisterToList(Reg, RegWidth, RegKind1, Reg1, RegNum1)) {
1668           return false;
1669         }
1670       } else {
1671         return false;
1672       }
1673     } while (true);
1674   } else {
1675     return false;
1676   }
1677   switch (RegKind) {
1678   case IS_SPECIAL:
1679     RegNum = 0;
1680     RegWidth = 1;
1681     break;
1682   case IS_VGPR:
1683   case IS_SGPR:
1684   case IS_TTMP:
1685   {
1686     unsigned Size = 1;
1687     if (RegKind == IS_SGPR || RegKind == IS_TTMP) {
1688       // SGPR and TTMP registers must be aligned. Max required alignment is 4 dwords.
1689       Size = std::min(RegWidth, 4u);
1690     }
1691     if (RegNum % Size != 0)
1692       return false;
1693     if (DwordRegIndex) { *DwordRegIndex = RegNum; }
1694     RegNum = RegNum / Size;
1695     int RCID = getRegClass(RegKind, RegWidth);
1696     if (RCID == -1)
1697       return false;
1698     const MCRegisterClass RC = TRI->getRegClass(RCID);
1699     if (RegNum >= RC.getNumRegs())
1700       return false;
1701     Reg = RC.getRegister(RegNum);
1702     break;
1703   }
1704 
1705   default:
1706     llvm_unreachable("unexpected register kind");
1707   }
1708 
1709   if (!subtargetHasRegister(*TRI, Reg))
1710     return false;
1711   return true;
1712 }
1713 
1714 std::unique_ptr<AMDGPUOperand> AMDGPUAsmParser::parseRegister() {
1715   const auto &Tok = Parser.getTok();
1716   SMLoc StartLoc = Tok.getLoc();
1717   SMLoc EndLoc = Tok.getEndLoc();
1718   RegisterKind RegKind;
1719   unsigned Reg, RegNum, RegWidth, DwordRegIndex;
1720 
1721   if (!ParseAMDGPURegister(RegKind, Reg, RegNum, RegWidth, &DwordRegIndex)) {
1722     return nullptr;
1723   }
1724   KernelScope.usesRegister(RegKind, DwordRegIndex, RegWidth);
1725   return AMDGPUOperand::CreateReg(this, Reg, StartLoc, EndLoc, false);
1726 }
1727 
1728 bool
1729 AMDGPUAsmParser::parseAbsoluteExpr(int64_t &Val, bool AbsMod) {
1730   if (AbsMod && getLexer().peekTok().is(AsmToken::Pipe) &&
1731       (getLexer().getKind() == AsmToken::Integer ||
1732        getLexer().getKind() == AsmToken::Real)) {
1733     // This is a workaround for handling operands like these:
1734     //     |1.0|
1735     //     |-1|
1736     // This syntax is not compatible with syntax of standard
1737     // MC expressions (due to the trailing '|').
1738 
1739     SMLoc EndLoc;
1740     const MCExpr *Expr;
1741 
1742     if (getParser().parsePrimaryExpr(Expr, EndLoc)) {
1743       return true;
1744     }
1745 
1746     return !Expr->evaluateAsAbsolute(Val);
1747   }
1748 
1749   return getParser().parseAbsoluteExpression(Val);
1750 }
1751 
1752 OperandMatchResultTy
1753 AMDGPUAsmParser::parseImm(OperandVector &Operands, bool AbsMod) {
1754   // TODO: add syntactic sugar for 1/(2*PI)
1755   bool Minus = false;
1756   if (getLexer().getKind() == AsmToken::Minus) {
1757     Minus = true;
1758     Parser.Lex();
1759   }
1760 
1761   SMLoc S = Parser.getTok().getLoc();
1762   switch(getLexer().getKind()) {
1763   case AsmToken::Integer: {
1764     int64_t IntVal;
1765     if (parseAbsoluteExpr(IntVal, AbsMod))
1766       return MatchOperand_ParseFail;
1767     if (Minus)
1768       IntVal *= -1;
1769     Operands.push_back(AMDGPUOperand::CreateImm(this, IntVal, S));
1770     return MatchOperand_Success;
1771   }
1772   case AsmToken::Real: {
1773     int64_t IntVal;
1774     if (parseAbsoluteExpr(IntVal, AbsMod))
1775       return MatchOperand_ParseFail;
1776 
1777     APFloat F(BitsToDouble(IntVal));
1778     if (Minus)
1779       F.changeSign();
1780     Operands.push_back(
1781         AMDGPUOperand::CreateImm(this, F.bitcastToAPInt().getZExtValue(), S,
1782                                  AMDGPUOperand::ImmTyNone, true));
1783     return MatchOperand_Success;
1784   }
1785   default:
1786     return Minus ? MatchOperand_ParseFail : MatchOperand_NoMatch;
1787   }
1788 }
1789 
1790 OperandMatchResultTy
1791 AMDGPUAsmParser::parseReg(OperandVector &Operands) {
1792   if (auto R = parseRegister()) {
1793     assert(R->isReg());
1794     R->Reg.IsForcedVOP3 = isForcedVOP3();
1795     Operands.push_back(std::move(R));
1796     return MatchOperand_Success;
1797   }
1798   return MatchOperand_NoMatch;
1799 }
1800 
1801 OperandMatchResultTy
1802 AMDGPUAsmParser::parseRegOrImm(OperandVector &Operands, bool AbsMod) {
1803   auto res = parseImm(Operands, AbsMod);
1804   if (res != MatchOperand_NoMatch) {
1805     return res;
1806   }
1807 
1808   return parseReg(Operands);
1809 }
1810 
1811 OperandMatchResultTy
1812 AMDGPUAsmParser::parseRegOrImmWithFPInputMods(OperandVector &Operands,
1813                                               bool AllowImm) {
1814   bool Negate = false, Negate2 = false, Abs = false, Abs2 = false;
1815 
1816   if (getLexer().getKind()== AsmToken::Minus) {
1817     const AsmToken NextToken = getLexer().peekTok();
1818 
1819     // Disable ambiguous constructs like '--1' etc. Should use neg(-1) instead.
1820     if (NextToken.is(AsmToken::Minus)) {
1821       Error(Parser.getTok().getLoc(), "invalid syntax, expected 'neg' modifier");
1822       return MatchOperand_ParseFail;
1823     }
1824 
1825     // '-' followed by an integer literal N should be interpreted as integer
1826     // negation rather than a floating-point NEG modifier applied to N.
1827     // Beside being contr-intuitive, such use of floating-point NEG modifier
1828     // results in different meaning of integer literals used with VOP1/2/C
1829     // and VOP3, for example:
1830     //    v_exp_f32_e32 v5, -1 // VOP1: src0 = 0xFFFFFFFF
1831     //    v_exp_f32_e64 v5, -1 // VOP3: src0 = 0x80000001
1832     // Negative fp literals should be handled likewise for unifomtity
1833     if (!NextToken.is(AsmToken::Integer) && !NextToken.is(AsmToken::Real)) {
1834       Parser.Lex();
1835       Negate = true;
1836     }
1837   }
1838 
1839   if (getLexer().getKind() == AsmToken::Identifier &&
1840       Parser.getTok().getString() == "neg") {
1841     if (Negate) {
1842       Error(Parser.getTok().getLoc(), "expected register or immediate");
1843       return MatchOperand_ParseFail;
1844     }
1845     Parser.Lex();
1846     Negate2 = true;
1847     if (getLexer().isNot(AsmToken::LParen)) {
1848       Error(Parser.getTok().getLoc(), "expected left paren after neg");
1849       return MatchOperand_ParseFail;
1850     }
1851     Parser.Lex();
1852   }
1853 
1854   if (getLexer().getKind() == AsmToken::Identifier &&
1855       Parser.getTok().getString() == "abs") {
1856     Parser.Lex();
1857     Abs2 = true;
1858     if (getLexer().isNot(AsmToken::LParen)) {
1859       Error(Parser.getTok().getLoc(), "expected left paren after abs");
1860       return MatchOperand_ParseFail;
1861     }
1862     Parser.Lex();
1863   }
1864 
1865   if (getLexer().getKind() == AsmToken::Pipe) {
1866     if (Abs2) {
1867       Error(Parser.getTok().getLoc(), "expected register or immediate");
1868       return MatchOperand_ParseFail;
1869     }
1870     Parser.Lex();
1871     Abs = true;
1872   }
1873 
1874   OperandMatchResultTy Res;
1875   if (AllowImm) {
1876     Res = parseRegOrImm(Operands, Abs);
1877   } else {
1878     Res = parseReg(Operands);
1879   }
1880   if (Res != MatchOperand_Success) {
1881     return Res;
1882   }
1883 
1884   AMDGPUOperand::Modifiers Mods;
1885   if (Abs) {
1886     if (getLexer().getKind() != AsmToken::Pipe) {
1887       Error(Parser.getTok().getLoc(), "expected vertical bar");
1888       return MatchOperand_ParseFail;
1889     }
1890     Parser.Lex();
1891     Mods.Abs = true;
1892   }
1893   if (Abs2) {
1894     if (getLexer().isNot(AsmToken::RParen)) {
1895       Error(Parser.getTok().getLoc(), "expected closing parentheses");
1896       return MatchOperand_ParseFail;
1897     }
1898     Parser.Lex();
1899     Mods.Abs = true;
1900   }
1901 
1902   if (Negate) {
1903     Mods.Neg = true;
1904   } else if (Negate2) {
1905     if (getLexer().isNot(AsmToken::RParen)) {
1906       Error(Parser.getTok().getLoc(), "expected closing parentheses");
1907       return MatchOperand_ParseFail;
1908     }
1909     Parser.Lex();
1910     Mods.Neg = true;
1911   }
1912 
1913   if (Mods.hasFPModifiers()) {
1914     AMDGPUOperand &Op = static_cast<AMDGPUOperand &>(*Operands.back());
1915     Op.setModifiers(Mods);
1916   }
1917   return MatchOperand_Success;
1918 }
1919 
1920 OperandMatchResultTy
1921 AMDGPUAsmParser::parseRegOrImmWithIntInputMods(OperandVector &Operands,
1922                                                bool AllowImm) {
1923   bool Sext = false;
1924 
1925   if (getLexer().getKind() == AsmToken::Identifier &&
1926       Parser.getTok().getString() == "sext") {
1927     Parser.Lex();
1928     Sext = true;
1929     if (getLexer().isNot(AsmToken::LParen)) {
1930       Error(Parser.getTok().getLoc(), "expected left paren after sext");
1931       return MatchOperand_ParseFail;
1932     }
1933     Parser.Lex();
1934   }
1935 
1936   OperandMatchResultTy Res;
1937   if (AllowImm) {
1938     Res = parseRegOrImm(Operands);
1939   } else {
1940     Res = parseReg(Operands);
1941   }
1942   if (Res != MatchOperand_Success) {
1943     return Res;
1944   }
1945 
1946   AMDGPUOperand::Modifiers Mods;
1947   if (Sext) {
1948     if (getLexer().isNot(AsmToken::RParen)) {
1949       Error(Parser.getTok().getLoc(), "expected closing parentheses");
1950       return MatchOperand_ParseFail;
1951     }
1952     Parser.Lex();
1953     Mods.Sext = true;
1954   }
1955 
1956   if (Mods.hasIntModifiers()) {
1957     AMDGPUOperand &Op = static_cast<AMDGPUOperand &>(*Operands.back());
1958     Op.setModifiers(Mods);
1959   }
1960 
1961   return MatchOperand_Success;
1962 }
1963 
1964 OperandMatchResultTy
1965 AMDGPUAsmParser::parseRegWithFPInputMods(OperandVector &Operands) {
1966   return parseRegOrImmWithFPInputMods(Operands, false);
1967 }
1968 
1969 OperandMatchResultTy
1970 AMDGPUAsmParser::parseRegWithIntInputMods(OperandVector &Operands) {
1971   return parseRegOrImmWithIntInputMods(Operands, false);
1972 }
1973 
1974 OperandMatchResultTy AMDGPUAsmParser::parseVReg32OrOff(OperandVector &Operands) {
1975   std::unique_ptr<AMDGPUOperand> Reg = parseRegister();
1976   if (Reg) {
1977     Operands.push_back(std::move(Reg));
1978     return MatchOperand_Success;
1979   }
1980 
1981   const AsmToken &Tok = Parser.getTok();
1982   if (Tok.getString() == "off") {
1983     Operands.push_back(AMDGPUOperand::CreateImm(this, 0, Tok.getLoc(),
1984                                                 AMDGPUOperand::ImmTyOff, false));
1985     Parser.Lex();
1986     return MatchOperand_Success;
1987   }
1988 
1989   return MatchOperand_NoMatch;
1990 }
1991 
1992 unsigned AMDGPUAsmParser::checkTargetMatchPredicate(MCInst &Inst) {
1993   uint64_t TSFlags = MII.get(Inst.getOpcode()).TSFlags;
1994 
1995   if ((getForcedEncodingSize() == 32 && (TSFlags & SIInstrFlags::VOP3)) ||
1996       (getForcedEncodingSize() == 64 && !(TSFlags & SIInstrFlags::VOP3)) ||
1997       (isForcedDPP() && !(TSFlags & SIInstrFlags::DPP)) ||
1998       (isForcedSDWA() && !(TSFlags & SIInstrFlags::SDWA)) )
1999     return Match_InvalidOperand;
2000 
2001   if ((TSFlags & SIInstrFlags::VOP3) &&
2002       (TSFlags & SIInstrFlags::VOPAsmPrefer32Bit) &&
2003       getForcedEncodingSize() != 64)
2004     return Match_PreferE32;
2005 
2006   if (Inst.getOpcode() == AMDGPU::V_MAC_F32_sdwa_vi ||
2007       Inst.getOpcode() == AMDGPU::V_MAC_F16_sdwa_vi) {
2008     // v_mac_f32/16 allow only dst_sel == DWORD;
2009     auto OpNum =
2010         AMDGPU::getNamedOperandIdx(Inst.getOpcode(), AMDGPU::OpName::dst_sel);
2011     const auto &Op = Inst.getOperand(OpNum);
2012     if (!Op.isImm() || Op.getImm() != AMDGPU::SDWA::SdwaSel::DWORD) {
2013       return Match_InvalidOperand;
2014     }
2015   }
2016 
2017   if ((TSFlags & SIInstrFlags::FLAT) && !hasFlatOffsets()) {
2018     // FIXME: Produces error without correct column reported.
2019     auto OpNum =
2020         AMDGPU::getNamedOperandIdx(Inst.getOpcode(), AMDGPU::OpName::offset);
2021     const auto &Op = Inst.getOperand(OpNum);
2022     if (Op.getImm() != 0)
2023       return Match_InvalidOperand;
2024   }
2025 
2026   return Match_Success;
2027 }
2028 
2029 // What asm variants we should check
2030 ArrayRef<unsigned> AMDGPUAsmParser::getMatchedVariants() const {
2031   if (getForcedEncodingSize() == 32) {
2032     static const unsigned Variants[] = {AMDGPUAsmVariants::DEFAULT};
2033     return makeArrayRef(Variants);
2034   }
2035 
2036   if (isForcedVOP3()) {
2037     static const unsigned Variants[] = {AMDGPUAsmVariants::VOP3};
2038     return makeArrayRef(Variants);
2039   }
2040 
2041   if (isForcedSDWA()) {
2042     static const unsigned Variants[] = {AMDGPUAsmVariants::SDWA,
2043                                         AMDGPUAsmVariants::SDWA9};
2044     return makeArrayRef(Variants);
2045   }
2046 
2047   if (isForcedDPP()) {
2048     static const unsigned Variants[] = {AMDGPUAsmVariants::DPP};
2049     return makeArrayRef(Variants);
2050   }
2051 
2052   static const unsigned Variants[] = {
2053     AMDGPUAsmVariants::DEFAULT, AMDGPUAsmVariants::VOP3,
2054     AMDGPUAsmVariants::SDWA, AMDGPUAsmVariants::SDWA9, AMDGPUAsmVariants::DPP
2055   };
2056 
2057   return makeArrayRef(Variants);
2058 }
2059 
2060 unsigned AMDGPUAsmParser::findImplicitSGPRReadInVOP(const MCInst &Inst) const {
2061   const MCInstrDesc &Desc = MII.get(Inst.getOpcode());
2062   const unsigned Num = Desc.getNumImplicitUses();
2063   for (unsigned i = 0; i < Num; ++i) {
2064     unsigned Reg = Desc.ImplicitUses[i];
2065     switch (Reg) {
2066     case AMDGPU::FLAT_SCR:
2067     case AMDGPU::VCC:
2068     case AMDGPU::M0:
2069       return Reg;
2070     default:
2071       break;
2072     }
2073   }
2074   return AMDGPU::NoRegister;
2075 }
2076 
2077 // NB: This code is correct only when used to check constant
2078 // bus limitations because GFX7 support no f16 inline constants.
2079 // Note that there are no cases when a GFX7 opcode violates
2080 // constant bus limitations due to the use of an f16 constant.
2081 bool AMDGPUAsmParser::isInlineConstant(const MCInst &Inst,
2082                                        unsigned OpIdx) const {
2083   const MCInstrDesc &Desc = MII.get(Inst.getOpcode());
2084 
2085   if (!AMDGPU::isSISrcOperand(Desc, OpIdx)) {
2086     return false;
2087   }
2088 
2089   const MCOperand &MO = Inst.getOperand(OpIdx);
2090 
2091   int64_t Val = MO.getImm();
2092   auto OpSize = AMDGPU::getOperandSize(Desc, OpIdx);
2093 
2094   switch (OpSize) { // expected operand size
2095   case 8:
2096     return AMDGPU::isInlinableLiteral64(Val, hasInv2PiInlineImm());
2097   case 4:
2098     return AMDGPU::isInlinableLiteral32(Val, hasInv2PiInlineImm());
2099   case 2: {
2100     const unsigned OperandType = Desc.OpInfo[OpIdx].OperandType;
2101     if (OperandType == AMDGPU::OPERAND_REG_INLINE_C_V2INT16 ||
2102         OperandType == AMDGPU::OPERAND_REG_INLINE_C_V2FP16) {
2103       return AMDGPU::isInlinableLiteralV216(Val, hasInv2PiInlineImm());
2104     } else {
2105       return AMDGPU::isInlinableLiteral16(Val, hasInv2PiInlineImm());
2106     }
2107   }
2108   default:
2109     llvm_unreachable("invalid operand size");
2110   }
2111 }
2112 
2113 bool AMDGPUAsmParser::usesConstantBus(const MCInst &Inst, unsigned OpIdx) {
2114   const MCOperand &MO = Inst.getOperand(OpIdx);
2115   if (MO.isImm()) {
2116     return !isInlineConstant(Inst, OpIdx);
2117   }
2118   return !MO.isReg() ||
2119          isSGPR(mc2PseudoReg(MO.getReg()), getContext().getRegisterInfo());
2120 }
2121 
2122 bool AMDGPUAsmParser::validateConstantBusLimitations(const MCInst &Inst) {
2123   const unsigned Opcode = Inst.getOpcode();
2124   const MCInstrDesc &Desc = MII.get(Opcode);
2125   unsigned ConstantBusUseCount = 0;
2126 
2127   if (Desc.TSFlags &
2128       (SIInstrFlags::VOPC |
2129        SIInstrFlags::VOP1 | SIInstrFlags::VOP2 |
2130        SIInstrFlags::VOP3 | SIInstrFlags::VOP3P |
2131        SIInstrFlags::SDWA)) {
2132     // Check special imm operands (used by madmk, etc)
2133     if (AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::imm) != -1) {
2134       ++ConstantBusUseCount;
2135     }
2136 
2137     unsigned SGPRUsed = findImplicitSGPRReadInVOP(Inst);
2138     if (SGPRUsed != AMDGPU::NoRegister) {
2139       ++ConstantBusUseCount;
2140     }
2141 
2142     const int Src0Idx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::src0);
2143     const int Src1Idx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::src1);
2144     const int Src2Idx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::src2);
2145 
2146     const int OpIndices[] = { Src0Idx, Src1Idx, Src2Idx };
2147 
2148     for (int OpIdx : OpIndices) {
2149       if (OpIdx == -1) break;
2150 
2151       const MCOperand &MO = Inst.getOperand(OpIdx);
2152       if (usesConstantBus(Inst, OpIdx)) {
2153         if (MO.isReg()) {
2154           const unsigned Reg = mc2PseudoReg(MO.getReg());
2155           // Pairs of registers with a partial intersections like these
2156           //   s0, s[0:1]
2157           //   flat_scratch_lo, flat_scratch
2158           //   flat_scratch_lo, flat_scratch_hi
2159           // are theoretically valid but they are disabled anyway.
2160           // Note that this code mimics SIInstrInfo::verifyInstruction
2161           if (Reg != SGPRUsed) {
2162             ++ConstantBusUseCount;
2163           }
2164           SGPRUsed = Reg;
2165         } else { // Expression or a literal
2166           ++ConstantBusUseCount;
2167         }
2168       }
2169     }
2170   }
2171 
2172   return ConstantBusUseCount <= 1;
2173 }
2174 
2175 bool AMDGPUAsmParser::validateEarlyClobberLimitations(const MCInst &Inst) {
2176   const unsigned Opcode = Inst.getOpcode();
2177   const MCInstrDesc &Desc = MII.get(Opcode);
2178 
2179   const int DstIdx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::vdst);
2180   if (DstIdx == -1 ||
2181       Desc.getOperandConstraint(DstIdx, MCOI::EARLY_CLOBBER) == -1) {
2182     return true;
2183   }
2184 
2185   const MCRegisterInfo *TRI = getContext().getRegisterInfo();
2186 
2187   const int Src0Idx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::src0);
2188   const int Src1Idx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::src1);
2189   const int Src2Idx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::src2);
2190 
2191   assert(DstIdx != -1);
2192   const MCOperand &Dst = Inst.getOperand(DstIdx);
2193   assert(Dst.isReg());
2194   const unsigned DstReg = mc2PseudoReg(Dst.getReg());
2195 
2196   const int SrcIndices[] = { Src0Idx, Src1Idx, Src2Idx };
2197 
2198   for (int SrcIdx : SrcIndices) {
2199     if (SrcIdx == -1) break;
2200     const MCOperand &Src = Inst.getOperand(SrcIdx);
2201     if (Src.isReg()) {
2202       const unsigned SrcReg = mc2PseudoReg(Src.getReg());
2203       if (isRegIntersect(DstReg, SrcReg, TRI)) {
2204         return false;
2205       }
2206     }
2207   }
2208 
2209   return true;
2210 }
2211 
2212 bool AMDGPUAsmParser::validateIntClampSupported(const MCInst &Inst) {
2213 
2214   const unsigned Opc = Inst.getOpcode();
2215   const MCInstrDesc &Desc = MII.get(Opc);
2216 
2217   if ((Desc.TSFlags & SIInstrFlags::IntClamp) != 0 && !hasIntClamp()) {
2218     int ClampIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::clamp);
2219     assert(ClampIdx != -1);
2220     return Inst.getOperand(ClampIdx).getImm() == 0;
2221   }
2222 
2223   return true;
2224 }
2225 
2226 bool AMDGPUAsmParser::validateInstruction(const MCInst &Inst,
2227                                           const SMLoc &IDLoc) {
2228   if (!validateConstantBusLimitations(Inst)) {
2229     Error(IDLoc,
2230       "invalid operand (violates constant bus restrictions)");
2231     return false;
2232   }
2233   if (!validateEarlyClobberLimitations(Inst)) {
2234     Error(IDLoc,
2235       "destination must be different than all sources");
2236     return false;
2237   }
2238   if (!validateIntClampSupported(Inst)) {
2239     Error(IDLoc,
2240       "integer clamping is not supported on this GPU");
2241     return false;
2242   }
2243 
2244   return true;
2245 }
2246 
2247 bool AMDGPUAsmParser::MatchAndEmitInstruction(SMLoc IDLoc, unsigned &Opcode,
2248                                               OperandVector &Operands,
2249                                               MCStreamer &Out,
2250                                               uint64_t &ErrorInfo,
2251                                               bool MatchingInlineAsm) {
2252   MCInst Inst;
2253   unsigned Result = Match_Success;
2254   for (auto Variant : getMatchedVariants()) {
2255     uint64_t EI;
2256     auto R = MatchInstructionImpl(Operands, Inst, EI, MatchingInlineAsm,
2257                                   Variant);
2258     // We order match statuses from least to most specific. We use most specific
2259     // status as resulting
2260     // Match_MnemonicFail < Match_InvalidOperand < Match_MissingFeature < Match_PreferE32
2261     if ((R == Match_Success) ||
2262         (R == Match_PreferE32) ||
2263         (R == Match_MissingFeature && Result != Match_PreferE32) ||
2264         (R == Match_InvalidOperand && Result != Match_MissingFeature
2265                                    && Result != Match_PreferE32) ||
2266         (R == Match_MnemonicFail   && Result != Match_InvalidOperand
2267                                    && Result != Match_MissingFeature
2268                                    && Result != Match_PreferE32)) {
2269       Result = R;
2270       ErrorInfo = EI;
2271     }
2272     if (R == Match_Success)
2273       break;
2274   }
2275 
2276   switch (Result) {
2277   default: break;
2278   case Match_Success:
2279     if (!validateInstruction(Inst, IDLoc)) {
2280       return true;
2281     }
2282     Inst.setLoc(IDLoc);
2283     Out.EmitInstruction(Inst, getSTI());
2284     return false;
2285 
2286   case Match_MissingFeature:
2287     return Error(IDLoc, "instruction not supported on this GPU");
2288 
2289   case Match_MnemonicFail:
2290     return Error(IDLoc, "unrecognized instruction mnemonic");
2291 
2292   case Match_InvalidOperand: {
2293     SMLoc ErrorLoc = IDLoc;
2294     if (ErrorInfo != ~0ULL) {
2295       if (ErrorInfo >= Operands.size()) {
2296         return Error(IDLoc, "too few operands for instruction");
2297       }
2298       ErrorLoc = ((AMDGPUOperand &)*Operands[ErrorInfo]).getStartLoc();
2299       if (ErrorLoc == SMLoc())
2300         ErrorLoc = IDLoc;
2301     }
2302     return Error(ErrorLoc, "invalid operand for instruction");
2303   }
2304 
2305   case Match_PreferE32:
2306     return Error(IDLoc, "internal error: instruction without _e64 suffix "
2307                         "should be encoded as e32");
2308   }
2309   llvm_unreachable("Implement any new match types added!");
2310 }
2311 
2312 bool AMDGPUAsmParser::ParseAsAbsoluteExpression(uint32_t &Ret) {
2313   int64_t Tmp = -1;
2314   if (getLexer().isNot(AsmToken::Integer) && getLexer().isNot(AsmToken::Identifier)) {
2315     return true;
2316   }
2317   if (getParser().parseAbsoluteExpression(Tmp)) {
2318     return true;
2319   }
2320   Ret = static_cast<uint32_t>(Tmp);
2321   return false;
2322 }
2323 
2324 bool AMDGPUAsmParser::ParseDirectiveMajorMinor(uint32_t &Major,
2325                                                uint32_t &Minor) {
2326   if (ParseAsAbsoluteExpression(Major))
2327     return TokError("invalid major version");
2328 
2329   if (getLexer().isNot(AsmToken::Comma))
2330     return TokError("minor version number required, comma expected");
2331   Lex();
2332 
2333   if (ParseAsAbsoluteExpression(Minor))
2334     return TokError("invalid minor version");
2335 
2336   return false;
2337 }
2338 
2339 bool AMDGPUAsmParser::ParseDirectiveHSACodeObjectVersion() {
2340   uint32_t Major;
2341   uint32_t Minor;
2342 
2343   if (ParseDirectiveMajorMinor(Major, Minor))
2344     return true;
2345 
2346   getTargetStreamer().EmitDirectiveHSACodeObjectVersion(Major, Minor);
2347   return false;
2348 }
2349 
2350 bool AMDGPUAsmParser::ParseDirectiveHSACodeObjectISA() {
2351   uint32_t Major;
2352   uint32_t Minor;
2353   uint32_t Stepping;
2354   StringRef VendorName;
2355   StringRef ArchName;
2356 
2357   // If this directive has no arguments, then use the ISA version for the
2358   // targeted GPU.
2359   if (getLexer().is(AsmToken::EndOfStatement)) {
2360     AMDGPU::IsaInfo::IsaVersion ISA =
2361         AMDGPU::IsaInfo::getIsaVersion(getFeatureBits());
2362     getTargetStreamer().EmitDirectiveHSACodeObjectISA(ISA.Major, ISA.Minor,
2363                                                       ISA.Stepping,
2364                                                       "AMD", "AMDGPU");
2365     return false;
2366   }
2367 
2368   if (ParseDirectiveMajorMinor(Major, Minor))
2369     return true;
2370 
2371   if (getLexer().isNot(AsmToken::Comma))
2372     return TokError("stepping version number required, comma expected");
2373   Lex();
2374 
2375   if (ParseAsAbsoluteExpression(Stepping))
2376     return TokError("invalid stepping version");
2377 
2378   if (getLexer().isNot(AsmToken::Comma))
2379     return TokError("vendor name required, comma expected");
2380   Lex();
2381 
2382   if (getLexer().isNot(AsmToken::String))
2383     return TokError("invalid vendor name");
2384 
2385   VendorName = getLexer().getTok().getStringContents();
2386   Lex();
2387 
2388   if (getLexer().isNot(AsmToken::Comma))
2389     return TokError("arch name required, comma expected");
2390   Lex();
2391 
2392   if (getLexer().isNot(AsmToken::String))
2393     return TokError("invalid arch name");
2394 
2395   ArchName = getLexer().getTok().getStringContents();
2396   Lex();
2397 
2398   getTargetStreamer().EmitDirectiveHSACodeObjectISA(Major, Minor, Stepping,
2399                                                     VendorName, ArchName);
2400   return false;
2401 }
2402 
2403 bool AMDGPUAsmParser::ParseAMDKernelCodeTValue(StringRef ID,
2404                                                amd_kernel_code_t &Header) {
2405   SmallString<40> ErrStr;
2406   raw_svector_ostream Err(ErrStr);
2407   if (!parseAmdKernelCodeField(ID, getParser(), Header, Err)) {
2408     return TokError(Err.str());
2409   }
2410   Lex();
2411   return false;
2412 }
2413 
2414 bool AMDGPUAsmParser::ParseDirectiveAMDKernelCodeT() {
2415   amd_kernel_code_t Header;
2416   AMDGPU::initDefaultAMDKernelCodeT(Header, getFeatureBits());
2417 
2418   while (true) {
2419     // Lex EndOfStatement.  This is in a while loop, because lexing a comment
2420     // will set the current token to EndOfStatement.
2421     while(getLexer().is(AsmToken::EndOfStatement))
2422       Lex();
2423 
2424     if (getLexer().isNot(AsmToken::Identifier))
2425       return TokError("expected value identifier or .end_amd_kernel_code_t");
2426 
2427     StringRef ID = getLexer().getTok().getIdentifier();
2428     Lex();
2429 
2430     if (ID == ".end_amd_kernel_code_t")
2431       break;
2432 
2433     if (ParseAMDKernelCodeTValue(ID, Header))
2434       return true;
2435   }
2436 
2437   getTargetStreamer().EmitAMDKernelCodeT(Header);
2438 
2439   return false;
2440 }
2441 
2442 bool AMDGPUAsmParser::ParseDirectiveAMDGPUHsaKernel() {
2443   if (getLexer().isNot(AsmToken::Identifier))
2444     return TokError("expected symbol name");
2445 
2446   StringRef KernelName = Parser.getTok().getString();
2447 
2448   getTargetStreamer().EmitAMDGPUSymbolType(KernelName,
2449                                            ELF::STT_AMDGPU_HSA_KERNEL);
2450   Lex();
2451   KernelScope.initialize(getContext());
2452   return false;
2453 }
2454 
2455 bool AMDGPUAsmParser::ParseDirectiveHSAMetadata() {
2456   std::string HSAMetadataString;
2457   raw_string_ostream YamlStream(HSAMetadataString);
2458 
2459   getLexer().setSkipSpace(false);
2460 
2461   bool FoundEnd = false;
2462   while (!getLexer().is(AsmToken::Eof)) {
2463     while (getLexer().is(AsmToken::Space)) {
2464       YamlStream << getLexer().getTok().getString();
2465       Lex();
2466     }
2467 
2468     if (getLexer().is(AsmToken::Identifier)) {
2469       StringRef ID = getLexer().getTok().getIdentifier();
2470       if (ID == AMDGPU::HSAMD::AssemblerDirectiveEnd) {
2471         Lex();
2472         FoundEnd = true;
2473         break;
2474       }
2475     }
2476 
2477     YamlStream << Parser.parseStringToEndOfStatement()
2478                << getContext().getAsmInfo()->getSeparatorString();
2479 
2480     Parser.eatToEndOfStatement();
2481   }
2482 
2483   getLexer().setSkipSpace(true);
2484 
2485   if (getLexer().is(AsmToken::Eof) && !FoundEnd) {
2486     return TokError(Twine("expected directive ") +
2487                     Twine(HSAMD::AssemblerDirectiveEnd) + Twine("not found"));
2488   }
2489 
2490   YamlStream.flush();
2491 
2492   if (!getTargetStreamer().EmitHSAMetadata(HSAMetadataString))
2493     return Error(getParser().getTok().getLoc(), "invalid HSA metadata");
2494 
2495   return false;
2496 }
2497 
2498 bool AMDGPUAsmParser::ParseDirectivePALMetadata() {
2499   PALMD::Metadata PALMetadata;
2500   for (;;) {
2501     uint32_t Value;
2502     if (ParseAsAbsoluteExpression(Value)) {
2503       return TokError(Twine("invalid value in ") +
2504                       Twine(PALMD::AssemblerDirective));
2505     }
2506     PALMetadata.push_back(Value);
2507     if (getLexer().isNot(AsmToken::Comma))
2508       break;
2509     Lex();
2510   }
2511   getTargetStreamer().EmitPALMetadata(PALMetadata);
2512   return false;
2513 }
2514 
2515 bool AMDGPUAsmParser::ParseDirective(AsmToken DirectiveID) {
2516   StringRef IDVal = DirectiveID.getString();
2517 
2518   if (IDVal == ".hsa_code_object_version")
2519     return ParseDirectiveHSACodeObjectVersion();
2520 
2521   if (IDVal == ".hsa_code_object_isa")
2522     return ParseDirectiveHSACodeObjectISA();
2523 
2524   if (IDVal == ".amd_kernel_code_t")
2525     return ParseDirectiveAMDKernelCodeT();
2526 
2527   if (IDVal == ".amdgpu_hsa_kernel")
2528     return ParseDirectiveAMDGPUHsaKernel();
2529 
2530   if (IDVal == AMDGPU::HSAMD::AssemblerDirectiveBegin)
2531     return ParseDirectiveHSAMetadata();
2532 
2533   if (IDVal == PALMD::AssemblerDirective)
2534     return ParseDirectivePALMetadata();
2535 
2536   return true;
2537 }
2538 
2539 bool AMDGPUAsmParser::subtargetHasRegister(const MCRegisterInfo &MRI,
2540                                            unsigned RegNo) const {
2541   if (isCI())
2542     return true;
2543 
2544   if (isSI()) {
2545     // No flat_scr
2546     switch (RegNo) {
2547     case AMDGPU::FLAT_SCR:
2548     case AMDGPU::FLAT_SCR_LO:
2549     case AMDGPU::FLAT_SCR_HI:
2550       return false;
2551     default:
2552       return true;
2553     }
2554   }
2555 
2556   // VI only has 102 SGPRs, so make sure we aren't trying to use the 2 more that
2557   // SI/CI have.
2558   for (MCRegAliasIterator R(AMDGPU::SGPR102_SGPR103, &MRI, true);
2559        R.isValid(); ++R) {
2560     if (*R == RegNo)
2561       return false;
2562   }
2563 
2564   return true;
2565 }
2566 
2567 OperandMatchResultTy
2568 AMDGPUAsmParser::parseOperand(OperandVector &Operands, StringRef Mnemonic) {
2569   // Try to parse with a custom parser
2570   OperandMatchResultTy ResTy = MatchOperandParserImpl(Operands, Mnemonic);
2571 
2572   // If we successfully parsed the operand or if there as an error parsing,
2573   // we are done.
2574   //
2575   // If we are parsing after we reach EndOfStatement then this means we
2576   // are appending default values to the Operands list.  This is only done
2577   // by custom parser, so we shouldn't continue on to the generic parsing.
2578   if (ResTy == MatchOperand_Success || ResTy == MatchOperand_ParseFail ||
2579       getLexer().is(AsmToken::EndOfStatement))
2580     return ResTy;
2581 
2582   ResTy = parseRegOrImm(Operands);
2583 
2584   if (ResTy == MatchOperand_Success)
2585     return ResTy;
2586 
2587   const auto &Tok = Parser.getTok();
2588   SMLoc S = Tok.getLoc();
2589 
2590   const MCExpr *Expr = nullptr;
2591   if (!Parser.parseExpression(Expr)) {
2592     Operands.push_back(AMDGPUOperand::CreateExpr(this, Expr, S));
2593     return MatchOperand_Success;
2594   }
2595 
2596   // Possibly this is an instruction flag like 'gds'.
2597   if (Tok.getKind() == AsmToken::Identifier) {
2598     Operands.push_back(AMDGPUOperand::CreateToken(this, Tok.getString(), S));
2599     Parser.Lex();
2600     return MatchOperand_Success;
2601   }
2602 
2603   return MatchOperand_NoMatch;
2604 }
2605 
2606 StringRef AMDGPUAsmParser::parseMnemonicSuffix(StringRef Name) {
2607   // Clear any forced encodings from the previous instruction.
2608   setForcedEncodingSize(0);
2609   setForcedDPP(false);
2610   setForcedSDWA(false);
2611 
2612   if (Name.endswith("_e64")) {
2613     setForcedEncodingSize(64);
2614     return Name.substr(0, Name.size() - 4);
2615   } else if (Name.endswith("_e32")) {
2616     setForcedEncodingSize(32);
2617     return Name.substr(0, Name.size() - 4);
2618   } else if (Name.endswith("_dpp")) {
2619     setForcedDPP(true);
2620     return Name.substr(0, Name.size() - 4);
2621   } else if (Name.endswith("_sdwa")) {
2622     setForcedSDWA(true);
2623     return Name.substr(0, Name.size() - 5);
2624   }
2625   return Name;
2626 }
2627 
2628 bool AMDGPUAsmParser::ParseInstruction(ParseInstructionInfo &Info,
2629                                        StringRef Name,
2630                                        SMLoc NameLoc, OperandVector &Operands) {
2631   // Add the instruction mnemonic
2632   Name = parseMnemonicSuffix(Name);
2633   Operands.push_back(AMDGPUOperand::CreateToken(this, Name, NameLoc));
2634 
2635   while (!getLexer().is(AsmToken::EndOfStatement)) {
2636     OperandMatchResultTy Res = parseOperand(Operands, Name);
2637 
2638     // Eat the comma or space if there is one.
2639     if (getLexer().is(AsmToken::Comma))
2640       Parser.Lex();
2641 
2642     switch (Res) {
2643       case MatchOperand_Success: break;
2644       case MatchOperand_ParseFail:
2645         Error(getLexer().getLoc(), "failed parsing operand.");
2646         while (!getLexer().is(AsmToken::EndOfStatement)) {
2647           Parser.Lex();
2648         }
2649         return true;
2650       case MatchOperand_NoMatch:
2651         Error(getLexer().getLoc(), "not a valid operand.");
2652         while (!getLexer().is(AsmToken::EndOfStatement)) {
2653           Parser.Lex();
2654         }
2655         return true;
2656     }
2657   }
2658 
2659   return false;
2660 }
2661 
2662 //===----------------------------------------------------------------------===//
2663 // Utility functions
2664 //===----------------------------------------------------------------------===//
2665 
2666 OperandMatchResultTy
2667 AMDGPUAsmParser::parseIntWithPrefix(const char *Prefix, int64_t &Int) {
2668   switch(getLexer().getKind()) {
2669     default: return MatchOperand_NoMatch;
2670     case AsmToken::Identifier: {
2671       StringRef Name = Parser.getTok().getString();
2672       if (!Name.equals(Prefix)) {
2673         return MatchOperand_NoMatch;
2674       }
2675 
2676       Parser.Lex();
2677       if (getLexer().isNot(AsmToken::Colon))
2678         return MatchOperand_ParseFail;
2679 
2680       Parser.Lex();
2681 
2682       bool IsMinus = false;
2683       if (getLexer().getKind() == AsmToken::Minus) {
2684         Parser.Lex();
2685         IsMinus = true;
2686       }
2687 
2688       if (getLexer().isNot(AsmToken::Integer))
2689         return MatchOperand_ParseFail;
2690 
2691       if (getParser().parseAbsoluteExpression(Int))
2692         return MatchOperand_ParseFail;
2693 
2694       if (IsMinus)
2695         Int = -Int;
2696       break;
2697     }
2698   }
2699   return MatchOperand_Success;
2700 }
2701 
2702 OperandMatchResultTy
2703 AMDGPUAsmParser::parseIntWithPrefix(const char *Prefix, OperandVector &Operands,
2704                                     AMDGPUOperand::ImmTy ImmTy,
2705                                     bool (*ConvertResult)(int64_t&)) {
2706   SMLoc S = Parser.getTok().getLoc();
2707   int64_t Value = 0;
2708 
2709   OperandMatchResultTy Res = parseIntWithPrefix(Prefix, Value);
2710   if (Res != MatchOperand_Success)
2711     return Res;
2712 
2713   if (ConvertResult && !ConvertResult(Value)) {
2714     return MatchOperand_ParseFail;
2715   }
2716 
2717   Operands.push_back(AMDGPUOperand::CreateImm(this, Value, S, ImmTy));
2718   return MatchOperand_Success;
2719 }
2720 
2721 OperandMatchResultTy AMDGPUAsmParser::parseOperandArrayWithPrefix(
2722   const char *Prefix,
2723   OperandVector &Operands,
2724   AMDGPUOperand::ImmTy ImmTy,
2725   bool (*ConvertResult)(int64_t&)) {
2726   StringRef Name = Parser.getTok().getString();
2727   if (!Name.equals(Prefix))
2728     return MatchOperand_NoMatch;
2729 
2730   Parser.Lex();
2731   if (getLexer().isNot(AsmToken::Colon))
2732     return MatchOperand_ParseFail;
2733 
2734   Parser.Lex();
2735   if (getLexer().isNot(AsmToken::LBrac))
2736     return MatchOperand_ParseFail;
2737   Parser.Lex();
2738 
2739   unsigned Val = 0;
2740   SMLoc S = Parser.getTok().getLoc();
2741 
2742   // FIXME: How to verify the number of elements matches the number of src
2743   // operands?
2744   for (int I = 0; I < 4; ++I) {
2745     if (I != 0) {
2746       if (getLexer().is(AsmToken::RBrac))
2747         break;
2748 
2749       if (getLexer().isNot(AsmToken::Comma))
2750         return MatchOperand_ParseFail;
2751       Parser.Lex();
2752     }
2753 
2754     if (getLexer().isNot(AsmToken::Integer))
2755       return MatchOperand_ParseFail;
2756 
2757     int64_t Op;
2758     if (getParser().parseAbsoluteExpression(Op))
2759       return MatchOperand_ParseFail;
2760 
2761     if (Op != 0 && Op != 1)
2762       return MatchOperand_ParseFail;
2763     Val |= (Op << I);
2764   }
2765 
2766   Parser.Lex();
2767   Operands.push_back(AMDGPUOperand::CreateImm(this, Val, S, ImmTy));
2768   return MatchOperand_Success;
2769 }
2770 
2771 OperandMatchResultTy
2772 AMDGPUAsmParser::parseNamedBit(const char *Name, OperandVector &Operands,
2773                                AMDGPUOperand::ImmTy ImmTy) {
2774   int64_t Bit = 0;
2775   SMLoc S = Parser.getTok().getLoc();
2776 
2777   // We are at the end of the statement, and this is a default argument, so
2778   // use a default value.
2779   if (getLexer().isNot(AsmToken::EndOfStatement)) {
2780     switch(getLexer().getKind()) {
2781       case AsmToken::Identifier: {
2782         StringRef Tok = Parser.getTok().getString();
2783         if (Tok == Name) {
2784           Bit = 1;
2785           Parser.Lex();
2786         } else if (Tok.startswith("no") && Tok.endswith(Name)) {
2787           Bit = 0;
2788           Parser.Lex();
2789         } else {
2790           return MatchOperand_NoMatch;
2791         }
2792         break;
2793       }
2794       default:
2795         return MatchOperand_NoMatch;
2796     }
2797   }
2798 
2799   Operands.push_back(AMDGPUOperand::CreateImm(this, Bit, S, ImmTy));
2800   return MatchOperand_Success;
2801 }
2802 
2803 static void addOptionalImmOperand(
2804   MCInst& Inst, const OperandVector& Operands,
2805   AMDGPUAsmParser::OptionalImmIndexMap& OptionalIdx,
2806   AMDGPUOperand::ImmTy ImmT,
2807   int64_t Default = 0) {
2808   auto i = OptionalIdx.find(ImmT);
2809   if (i != OptionalIdx.end()) {
2810     unsigned Idx = i->second;
2811     ((AMDGPUOperand &)*Operands[Idx]).addImmOperands(Inst, 1);
2812   } else {
2813     Inst.addOperand(MCOperand::createImm(Default));
2814   }
2815 }
2816 
2817 OperandMatchResultTy
2818 AMDGPUAsmParser::parseStringWithPrefix(StringRef Prefix, StringRef &Value) {
2819   if (getLexer().isNot(AsmToken::Identifier)) {
2820     return MatchOperand_NoMatch;
2821   }
2822   StringRef Tok = Parser.getTok().getString();
2823   if (Tok != Prefix) {
2824     return MatchOperand_NoMatch;
2825   }
2826 
2827   Parser.Lex();
2828   if (getLexer().isNot(AsmToken::Colon)) {
2829     return MatchOperand_ParseFail;
2830   }
2831 
2832   Parser.Lex();
2833   if (getLexer().isNot(AsmToken::Identifier)) {
2834     return MatchOperand_ParseFail;
2835   }
2836 
2837   Value = Parser.getTok().getString();
2838   return MatchOperand_Success;
2839 }
2840 
2841 //===----------------------------------------------------------------------===//
2842 // ds
2843 //===----------------------------------------------------------------------===//
2844 
2845 void AMDGPUAsmParser::cvtDSOffset01(MCInst &Inst,
2846                                     const OperandVector &Operands) {
2847   OptionalImmIndexMap OptionalIdx;
2848 
2849   for (unsigned i = 1, e = Operands.size(); i != e; ++i) {
2850     AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[i]);
2851 
2852     // Add the register arguments
2853     if (Op.isReg()) {
2854       Op.addRegOperands(Inst, 1);
2855       continue;
2856     }
2857 
2858     // Handle optional arguments
2859     OptionalIdx[Op.getImmTy()] = i;
2860   }
2861 
2862   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyOffset0);
2863   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyOffset1);
2864   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyGDS);
2865 
2866   Inst.addOperand(MCOperand::createReg(AMDGPU::M0)); // m0
2867 }
2868 
2869 void AMDGPUAsmParser::cvtDSImpl(MCInst &Inst, const OperandVector &Operands,
2870                                 bool IsGdsHardcoded) {
2871   OptionalImmIndexMap OptionalIdx;
2872 
2873   for (unsigned i = 1, e = Operands.size(); i != e; ++i) {
2874     AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[i]);
2875 
2876     // Add the register arguments
2877     if (Op.isReg()) {
2878       Op.addRegOperands(Inst, 1);
2879       continue;
2880     }
2881 
2882     if (Op.isToken() && Op.getToken() == "gds") {
2883       IsGdsHardcoded = true;
2884       continue;
2885     }
2886 
2887     // Handle optional arguments
2888     OptionalIdx[Op.getImmTy()] = i;
2889   }
2890 
2891   AMDGPUOperand::ImmTy OffsetType =
2892     (Inst.getOpcode() == AMDGPU::DS_SWIZZLE_B32_si ||
2893      Inst.getOpcode() == AMDGPU::DS_SWIZZLE_B32_vi) ? AMDGPUOperand::ImmTySwizzle :
2894                                                       AMDGPUOperand::ImmTyOffset;
2895 
2896   addOptionalImmOperand(Inst, Operands, OptionalIdx, OffsetType);
2897 
2898   if (!IsGdsHardcoded) {
2899     addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyGDS);
2900   }
2901   Inst.addOperand(MCOperand::createReg(AMDGPU::M0)); // m0
2902 }
2903 
2904 void AMDGPUAsmParser::cvtExp(MCInst &Inst, const OperandVector &Operands) {
2905   OptionalImmIndexMap OptionalIdx;
2906 
2907   unsigned OperandIdx[4];
2908   unsigned EnMask = 0;
2909   int SrcIdx = 0;
2910 
2911   for (unsigned i = 1, e = Operands.size(); i != e; ++i) {
2912     AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[i]);
2913 
2914     // Add the register arguments
2915     if (Op.isReg()) {
2916       assert(SrcIdx < 4);
2917       OperandIdx[SrcIdx] = Inst.size();
2918       Op.addRegOperands(Inst, 1);
2919       ++SrcIdx;
2920       continue;
2921     }
2922 
2923     if (Op.isOff()) {
2924       assert(SrcIdx < 4);
2925       OperandIdx[SrcIdx] = Inst.size();
2926       Inst.addOperand(MCOperand::createReg(AMDGPU::NoRegister));
2927       ++SrcIdx;
2928       continue;
2929     }
2930 
2931     if (Op.isImm() && Op.getImmTy() == AMDGPUOperand::ImmTyExpTgt) {
2932       Op.addImmOperands(Inst, 1);
2933       continue;
2934     }
2935 
2936     if (Op.isToken() && Op.getToken() == "done")
2937       continue;
2938 
2939     // Handle optional arguments
2940     OptionalIdx[Op.getImmTy()] = i;
2941   }
2942 
2943   assert(SrcIdx == 4);
2944 
2945   bool Compr = false;
2946   if (OptionalIdx.find(AMDGPUOperand::ImmTyExpCompr) != OptionalIdx.end()) {
2947     Compr = true;
2948     Inst.getOperand(OperandIdx[1]) = Inst.getOperand(OperandIdx[2]);
2949     Inst.getOperand(OperandIdx[2]).setReg(AMDGPU::NoRegister);
2950     Inst.getOperand(OperandIdx[3]).setReg(AMDGPU::NoRegister);
2951   }
2952 
2953   for (auto i = 0; i < SrcIdx; ++i) {
2954     if (Inst.getOperand(OperandIdx[i]).getReg() != AMDGPU::NoRegister) {
2955       EnMask |= Compr? (0x3 << i * 2) : (0x1 << i);
2956     }
2957   }
2958 
2959   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyExpVM);
2960   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyExpCompr);
2961 
2962   Inst.addOperand(MCOperand::createImm(EnMask));
2963 }
2964 
2965 //===----------------------------------------------------------------------===//
2966 // s_waitcnt
2967 //===----------------------------------------------------------------------===//
2968 
2969 static bool
2970 encodeCnt(
2971   const AMDGPU::IsaInfo::IsaVersion ISA,
2972   int64_t &IntVal,
2973   int64_t CntVal,
2974   bool Saturate,
2975   unsigned (*encode)(const IsaInfo::IsaVersion &Version, unsigned, unsigned),
2976   unsigned (*decode)(const IsaInfo::IsaVersion &Version, unsigned))
2977 {
2978   bool Failed = false;
2979 
2980   IntVal = encode(ISA, IntVal, CntVal);
2981   if (CntVal != decode(ISA, IntVal)) {
2982     if (Saturate) {
2983       IntVal = encode(ISA, IntVal, -1);
2984     } else {
2985       Failed = true;
2986     }
2987   }
2988   return Failed;
2989 }
2990 
2991 bool AMDGPUAsmParser::parseCnt(int64_t &IntVal) {
2992   StringRef CntName = Parser.getTok().getString();
2993   int64_t CntVal;
2994 
2995   Parser.Lex();
2996   if (getLexer().isNot(AsmToken::LParen))
2997     return true;
2998 
2999   Parser.Lex();
3000   if (getLexer().isNot(AsmToken::Integer))
3001     return true;
3002 
3003   SMLoc ValLoc = Parser.getTok().getLoc();
3004   if (getParser().parseAbsoluteExpression(CntVal))
3005     return true;
3006 
3007   AMDGPU::IsaInfo::IsaVersion ISA =
3008       AMDGPU::IsaInfo::getIsaVersion(getFeatureBits());
3009 
3010   bool Failed = true;
3011   bool Sat = CntName.endswith("_sat");
3012 
3013   if (CntName == "vmcnt" || CntName == "vmcnt_sat") {
3014     Failed = encodeCnt(ISA, IntVal, CntVal, Sat, encodeVmcnt, decodeVmcnt);
3015   } else if (CntName == "expcnt" || CntName == "expcnt_sat") {
3016     Failed = encodeCnt(ISA, IntVal, CntVal, Sat, encodeExpcnt, decodeExpcnt);
3017   } else if (CntName == "lgkmcnt" || CntName == "lgkmcnt_sat") {
3018     Failed = encodeCnt(ISA, IntVal, CntVal, Sat, encodeLgkmcnt, decodeLgkmcnt);
3019   }
3020 
3021   if (Failed) {
3022     Error(ValLoc, "too large value for " + CntName);
3023     return true;
3024   }
3025 
3026   if (getLexer().isNot(AsmToken::RParen)) {
3027     return true;
3028   }
3029 
3030   Parser.Lex();
3031   if (getLexer().is(AsmToken::Amp) || getLexer().is(AsmToken::Comma)) {
3032     const AsmToken NextToken = getLexer().peekTok();
3033     if (NextToken.is(AsmToken::Identifier)) {
3034       Parser.Lex();
3035     }
3036   }
3037 
3038   return false;
3039 }
3040 
3041 OperandMatchResultTy
3042 AMDGPUAsmParser::parseSWaitCntOps(OperandVector &Operands) {
3043   AMDGPU::IsaInfo::IsaVersion ISA =
3044       AMDGPU::IsaInfo::getIsaVersion(getFeatureBits());
3045   int64_t Waitcnt = getWaitcntBitMask(ISA);
3046   SMLoc S = Parser.getTok().getLoc();
3047 
3048   switch(getLexer().getKind()) {
3049     default: return MatchOperand_ParseFail;
3050     case AsmToken::Integer:
3051       // The operand can be an integer value.
3052       if (getParser().parseAbsoluteExpression(Waitcnt))
3053         return MatchOperand_ParseFail;
3054       break;
3055 
3056     case AsmToken::Identifier:
3057       do {
3058         if (parseCnt(Waitcnt))
3059           return MatchOperand_ParseFail;
3060       } while(getLexer().isNot(AsmToken::EndOfStatement));
3061       break;
3062   }
3063   Operands.push_back(AMDGPUOperand::CreateImm(this, Waitcnt, S));
3064   return MatchOperand_Success;
3065 }
3066 
3067 bool AMDGPUAsmParser::parseHwregConstruct(OperandInfoTy &HwReg, int64_t &Offset,
3068                                           int64_t &Width) {
3069   using namespace llvm::AMDGPU::Hwreg;
3070 
3071   if (Parser.getTok().getString() != "hwreg")
3072     return true;
3073   Parser.Lex();
3074 
3075   if (getLexer().isNot(AsmToken::LParen))
3076     return true;
3077   Parser.Lex();
3078 
3079   if (getLexer().is(AsmToken::Identifier)) {
3080     HwReg.IsSymbolic = true;
3081     HwReg.Id = ID_UNKNOWN_;
3082     const StringRef tok = Parser.getTok().getString();
3083     for (int i = ID_SYMBOLIC_FIRST_; i < ID_SYMBOLIC_LAST_; ++i) {
3084       if (tok == IdSymbolic[i]) {
3085         HwReg.Id = i;
3086         break;
3087       }
3088     }
3089     Parser.Lex();
3090   } else {
3091     HwReg.IsSymbolic = false;
3092     if (getLexer().isNot(AsmToken::Integer))
3093       return true;
3094     if (getParser().parseAbsoluteExpression(HwReg.Id))
3095       return true;
3096   }
3097 
3098   if (getLexer().is(AsmToken::RParen)) {
3099     Parser.Lex();
3100     return false;
3101   }
3102 
3103   // optional params
3104   if (getLexer().isNot(AsmToken::Comma))
3105     return true;
3106   Parser.Lex();
3107 
3108   if (getLexer().isNot(AsmToken::Integer))
3109     return true;
3110   if (getParser().parseAbsoluteExpression(Offset))
3111     return true;
3112 
3113   if (getLexer().isNot(AsmToken::Comma))
3114     return true;
3115   Parser.Lex();
3116 
3117   if (getLexer().isNot(AsmToken::Integer))
3118     return true;
3119   if (getParser().parseAbsoluteExpression(Width))
3120     return true;
3121 
3122   if (getLexer().isNot(AsmToken::RParen))
3123     return true;
3124   Parser.Lex();
3125 
3126   return false;
3127 }
3128 
3129 OperandMatchResultTy AMDGPUAsmParser::parseHwreg(OperandVector &Operands) {
3130   using namespace llvm::AMDGPU::Hwreg;
3131 
3132   int64_t Imm16Val = 0;
3133   SMLoc S = Parser.getTok().getLoc();
3134 
3135   switch(getLexer().getKind()) {
3136     default: return MatchOperand_NoMatch;
3137     case AsmToken::Integer:
3138       // The operand can be an integer value.
3139       if (getParser().parseAbsoluteExpression(Imm16Val))
3140         return MatchOperand_NoMatch;
3141       if (Imm16Val < 0 || !isUInt<16>(Imm16Val)) {
3142         Error(S, "invalid immediate: only 16-bit values are legal");
3143         // Do not return error code, but create an imm operand anyway and proceed
3144         // to the next operand, if any. That avoids unneccessary error messages.
3145       }
3146       break;
3147 
3148     case AsmToken::Identifier: {
3149         OperandInfoTy HwReg(ID_UNKNOWN_);
3150         int64_t Offset = OFFSET_DEFAULT_;
3151         int64_t Width = WIDTH_M1_DEFAULT_ + 1;
3152         if (parseHwregConstruct(HwReg, Offset, Width))
3153           return MatchOperand_ParseFail;
3154         if (HwReg.Id < 0 || !isUInt<ID_WIDTH_>(HwReg.Id)) {
3155           if (HwReg.IsSymbolic)
3156             Error(S, "invalid symbolic name of hardware register");
3157           else
3158             Error(S, "invalid code of hardware register: only 6-bit values are legal");
3159         }
3160         if (Offset < 0 || !isUInt<OFFSET_WIDTH_>(Offset))
3161           Error(S, "invalid bit offset: only 5-bit values are legal");
3162         if ((Width-1) < 0 || !isUInt<WIDTH_M1_WIDTH_>(Width-1))
3163           Error(S, "invalid bitfield width: only values from 1 to 32 are legal");
3164         Imm16Val = (HwReg.Id << ID_SHIFT_) | (Offset << OFFSET_SHIFT_) | ((Width-1) << WIDTH_M1_SHIFT_);
3165       }
3166       break;
3167   }
3168   Operands.push_back(AMDGPUOperand::CreateImm(this, Imm16Val, S, AMDGPUOperand::ImmTyHwreg));
3169   return MatchOperand_Success;
3170 }
3171 
3172 bool AMDGPUOperand::isSWaitCnt() const {
3173   return isImm();
3174 }
3175 
3176 bool AMDGPUOperand::isHwreg() const {
3177   return isImmTy(ImmTyHwreg);
3178 }
3179 
3180 bool AMDGPUAsmParser::parseSendMsgConstruct(OperandInfoTy &Msg, OperandInfoTy &Operation, int64_t &StreamId) {
3181   using namespace llvm::AMDGPU::SendMsg;
3182 
3183   if (Parser.getTok().getString() != "sendmsg")
3184     return true;
3185   Parser.Lex();
3186 
3187   if (getLexer().isNot(AsmToken::LParen))
3188     return true;
3189   Parser.Lex();
3190 
3191   if (getLexer().is(AsmToken::Identifier)) {
3192     Msg.IsSymbolic = true;
3193     Msg.Id = ID_UNKNOWN_;
3194     const std::string tok = Parser.getTok().getString();
3195     for (int i = ID_GAPS_FIRST_; i < ID_GAPS_LAST_; ++i) {
3196       switch(i) {
3197         default: continue; // Omit gaps.
3198         case ID_INTERRUPT: case ID_GS: case ID_GS_DONE:  case ID_SYSMSG: break;
3199       }
3200       if (tok == IdSymbolic[i]) {
3201         Msg.Id = i;
3202         break;
3203       }
3204     }
3205     Parser.Lex();
3206   } else {
3207     Msg.IsSymbolic = false;
3208     if (getLexer().isNot(AsmToken::Integer))
3209       return true;
3210     if (getParser().parseAbsoluteExpression(Msg.Id))
3211       return true;
3212     if (getLexer().is(AsmToken::Integer))
3213       if (getParser().parseAbsoluteExpression(Msg.Id))
3214         Msg.Id = ID_UNKNOWN_;
3215   }
3216   if (Msg.Id == ID_UNKNOWN_) // Don't know how to parse the rest.
3217     return false;
3218 
3219   if (!(Msg.Id == ID_GS || Msg.Id == ID_GS_DONE || Msg.Id == ID_SYSMSG)) {
3220     if (getLexer().isNot(AsmToken::RParen))
3221       return true;
3222     Parser.Lex();
3223     return false;
3224   }
3225 
3226   if (getLexer().isNot(AsmToken::Comma))
3227     return true;
3228   Parser.Lex();
3229 
3230   assert(Msg.Id == ID_GS || Msg.Id == ID_GS_DONE || Msg.Id == ID_SYSMSG);
3231   Operation.Id = ID_UNKNOWN_;
3232   if (getLexer().is(AsmToken::Identifier)) {
3233     Operation.IsSymbolic = true;
3234     const char* const *S = (Msg.Id == ID_SYSMSG) ? OpSysSymbolic : OpGsSymbolic;
3235     const int F = (Msg.Id == ID_SYSMSG) ? OP_SYS_FIRST_ : OP_GS_FIRST_;
3236     const int L = (Msg.Id == ID_SYSMSG) ? OP_SYS_LAST_ : OP_GS_LAST_;
3237     const StringRef Tok = Parser.getTok().getString();
3238     for (int i = F; i < L; ++i) {
3239       if (Tok == S[i]) {
3240         Operation.Id = i;
3241         break;
3242       }
3243     }
3244     Parser.Lex();
3245   } else {
3246     Operation.IsSymbolic = false;
3247     if (getLexer().isNot(AsmToken::Integer))
3248       return true;
3249     if (getParser().parseAbsoluteExpression(Operation.Id))
3250       return true;
3251   }
3252 
3253   if ((Msg.Id == ID_GS || Msg.Id == ID_GS_DONE) && Operation.Id != OP_GS_NOP) {
3254     // Stream id is optional.
3255     if (getLexer().is(AsmToken::RParen)) {
3256       Parser.Lex();
3257       return false;
3258     }
3259 
3260     if (getLexer().isNot(AsmToken::Comma))
3261       return true;
3262     Parser.Lex();
3263 
3264     if (getLexer().isNot(AsmToken::Integer))
3265       return true;
3266     if (getParser().parseAbsoluteExpression(StreamId))
3267       return true;
3268   }
3269 
3270   if (getLexer().isNot(AsmToken::RParen))
3271     return true;
3272   Parser.Lex();
3273   return false;
3274 }
3275 
3276 OperandMatchResultTy AMDGPUAsmParser::parseInterpSlot(OperandVector &Operands) {
3277   if (getLexer().getKind() != AsmToken::Identifier)
3278     return MatchOperand_NoMatch;
3279 
3280   StringRef Str = Parser.getTok().getString();
3281   int Slot = StringSwitch<int>(Str)
3282     .Case("p10", 0)
3283     .Case("p20", 1)
3284     .Case("p0", 2)
3285     .Default(-1);
3286 
3287   SMLoc S = Parser.getTok().getLoc();
3288   if (Slot == -1)
3289     return MatchOperand_ParseFail;
3290 
3291   Parser.Lex();
3292   Operands.push_back(AMDGPUOperand::CreateImm(this, Slot, S,
3293                                               AMDGPUOperand::ImmTyInterpSlot));
3294   return MatchOperand_Success;
3295 }
3296 
3297 OperandMatchResultTy AMDGPUAsmParser::parseInterpAttr(OperandVector &Operands) {
3298   if (getLexer().getKind() != AsmToken::Identifier)
3299     return MatchOperand_NoMatch;
3300 
3301   StringRef Str = Parser.getTok().getString();
3302   if (!Str.startswith("attr"))
3303     return MatchOperand_NoMatch;
3304 
3305   StringRef Chan = Str.take_back(2);
3306   int AttrChan = StringSwitch<int>(Chan)
3307     .Case(".x", 0)
3308     .Case(".y", 1)
3309     .Case(".z", 2)
3310     .Case(".w", 3)
3311     .Default(-1);
3312   if (AttrChan == -1)
3313     return MatchOperand_ParseFail;
3314 
3315   Str = Str.drop_back(2).drop_front(4);
3316 
3317   uint8_t Attr;
3318   if (Str.getAsInteger(10, Attr))
3319     return MatchOperand_ParseFail;
3320 
3321   SMLoc S = Parser.getTok().getLoc();
3322   Parser.Lex();
3323   if (Attr > 63) {
3324     Error(S, "out of bounds attr");
3325     return MatchOperand_Success;
3326   }
3327 
3328   SMLoc SChan = SMLoc::getFromPointer(Chan.data());
3329 
3330   Operands.push_back(AMDGPUOperand::CreateImm(this, Attr, S,
3331                                               AMDGPUOperand::ImmTyInterpAttr));
3332   Operands.push_back(AMDGPUOperand::CreateImm(this, AttrChan, SChan,
3333                                               AMDGPUOperand::ImmTyAttrChan));
3334   return MatchOperand_Success;
3335 }
3336 
3337 void AMDGPUAsmParser::errorExpTgt() {
3338   Error(Parser.getTok().getLoc(), "invalid exp target");
3339 }
3340 
3341 OperandMatchResultTy AMDGPUAsmParser::parseExpTgtImpl(StringRef Str,
3342                                                       uint8_t &Val) {
3343   if (Str == "null") {
3344     Val = 9;
3345     return MatchOperand_Success;
3346   }
3347 
3348   if (Str.startswith("mrt")) {
3349     Str = Str.drop_front(3);
3350     if (Str == "z") { // == mrtz
3351       Val = 8;
3352       return MatchOperand_Success;
3353     }
3354 
3355     if (Str.getAsInteger(10, Val))
3356       return MatchOperand_ParseFail;
3357 
3358     if (Val > 7)
3359       errorExpTgt();
3360 
3361     return MatchOperand_Success;
3362   }
3363 
3364   if (Str.startswith("pos")) {
3365     Str = Str.drop_front(3);
3366     if (Str.getAsInteger(10, Val))
3367       return MatchOperand_ParseFail;
3368 
3369     if (Val > 3)
3370       errorExpTgt();
3371 
3372     Val += 12;
3373     return MatchOperand_Success;
3374   }
3375 
3376   if (Str.startswith("param")) {
3377     Str = Str.drop_front(5);
3378     if (Str.getAsInteger(10, Val))
3379       return MatchOperand_ParseFail;
3380 
3381     if (Val >= 32)
3382       errorExpTgt();
3383 
3384     Val += 32;
3385     return MatchOperand_Success;
3386   }
3387 
3388   if (Str.startswith("invalid_target_")) {
3389     Str = Str.drop_front(15);
3390     if (Str.getAsInteger(10, Val))
3391       return MatchOperand_ParseFail;
3392 
3393     errorExpTgt();
3394     return MatchOperand_Success;
3395   }
3396 
3397   return MatchOperand_NoMatch;
3398 }
3399 
3400 OperandMatchResultTy AMDGPUAsmParser::parseExpTgt(OperandVector &Operands) {
3401   uint8_t Val;
3402   StringRef Str = Parser.getTok().getString();
3403 
3404   auto Res = parseExpTgtImpl(Str, Val);
3405   if (Res != MatchOperand_Success)
3406     return Res;
3407 
3408   SMLoc S = Parser.getTok().getLoc();
3409   Parser.Lex();
3410 
3411   Operands.push_back(AMDGPUOperand::CreateImm(this, Val, S,
3412                                               AMDGPUOperand::ImmTyExpTgt));
3413   return MatchOperand_Success;
3414 }
3415 
3416 OperandMatchResultTy
3417 AMDGPUAsmParser::parseSendMsgOp(OperandVector &Operands) {
3418   using namespace llvm::AMDGPU::SendMsg;
3419 
3420   int64_t Imm16Val = 0;
3421   SMLoc S = Parser.getTok().getLoc();
3422 
3423   switch(getLexer().getKind()) {
3424   default:
3425     return MatchOperand_NoMatch;
3426   case AsmToken::Integer:
3427     // The operand can be an integer value.
3428     if (getParser().parseAbsoluteExpression(Imm16Val))
3429       return MatchOperand_NoMatch;
3430     if (Imm16Val < 0 || !isUInt<16>(Imm16Val)) {
3431       Error(S, "invalid immediate: only 16-bit values are legal");
3432       // Do not return error code, but create an imm operand anyway and proceed
3433       // to the next operand, if any. That avoids unneccessary error messages.
3434     }
3435     break;
3436   case AsmToken::Identifier: {
3437       OperandInfoTy Msg(ID_UNKNOWN_);
3438       OperandInfoTy Operation(OP_UNKNOWN_);
3439       int64_t StreamId = STREAM_ID_DEFAULT_;
3440       if (parseSendMsgConstruct(Msg, Operation, StreamId))
3441         return MatchOperand_ParseFail;
3442       do {
3443         // Validate and encode message ID.
3444         if (! ((ID_INTERRUPT <= Msg.Id && Msg.Id <= ID_GS_DONE)
3445                 || Msg.Id == ID_SYSMSG)) {
3446           if (Msg.IsSymbolic)
3447             Error(S, "invalid/unsupported symbolic name of message");
3448           else
3449             Error(S, "invalid/unsupported code of message");
3450           break;
3451         }
3452         Imm16Val = (Msg.Id << ID_SHIFT_);
3453         // Validate and encode operation ID.
3454         if (Msg.Id == ID_GS || Msg.Id == ID_GS_DONE) {
3455           if (! (OP_GS_FIRST_ <= Operation.Id && Operation.Id < OP_GS_LAST_)) {
3456             if (Operation.IsSymbolic)
3457               Error(S, "invalid symbolic name of GS_OP");
3458             else
3459               Error(S, "invalid code of GS_OP: only 2-bit values are legal");
3460             break;
3461           }
3462           if (Operation.Id == OP_GS_NOP
3463               && Msg.Id != ID_GS_DONE) {
3464             Error(S, "invalid GS_OP: NOP is for GS_DONE only");
3465             break;
3466           }
3467           Imm16Val |= (Operation.Id << OP_SHIFT_);
3468         }
3469         if (Msg.Id == ID_SYSMSG) {
3470           if (! (OP_SYS_FIRST_ <= Operation.Id && Operation.Id < OP_SYS_LAST_)) {
3471             if (Operation.IsSymbolic)
3472               Error(S, "invalid/unsupported symbolic name of SYSMSG_OP");
3473             else
3474               Error(S, "invalid/unsupported code of SYSMSG_OP");
3475             break;
3476           }
3477           Imm16Val |= (Operation.Id << OP_SHIFT_);
3478         }
3479         // Validate and encode stream ID.
3480         if ((Msg.Id == ID_GS || Msg.Id == ID_GS_DONE) && Operation.Id != OP_GS_NOP) {
3481           if (! (STREAM_ID_FIRST_ <= StreamId && StreamId < STREAM_ID_LAST_)) {
3482             Error(S, "invalid stream id: only 2-bit values are legal");
3483             break;
3484           }
3485           Imm16Val |= (StreamId << STREAM_ID_SHIFT_);
3486         }
3487       } while (false);
3488     }
3489     break;
3490   }
3491   Operands.push_back(AMDGPUOperand::CreateImm(this, Imm16Val, S, AMDGPUOperand::ImmTySendMsg));
3492   return MatchOperand_Success;
3493 }
3494 
3495 bool AMDGPUOperand::isSendMsg() const {
3496   return isImmTy(ImmTySendMsg);
3497 }
3498 
3499 //===----------------------------------------------------------------------===//
3500 // parser helpers
3501 //===----------------------------------------------------------------------===//
3502 
3503 bool
3504 AMDGPUAsmParser::trySkipId(const StringRef Id) {
3505   if (getLexer().getKind() == AsmToken::Identifier &&
3506       Parser.getTok().getString() == Id) {
3507     Parser.Lex();
3508     return true;
3509   }
3510   return false;
3511 }
3512 
3513 bool
3514 AMDGPUAsmParser::trySkipToken(const AsmToken::TokenKind Kind) {
3515   if (getLexer().getKind() == Kind) {
3516     Parser.Lex();
3517     return true;
3518   }
3519   return false;
3520 }
3521 
3522 bool
3523 AMDGPUAsmParser::skipToken(const AsmToken::TokenKind Kind,
3524                            const StringRef ErrMsg) {
3525   if (!trySkipToken(Kind)) {
3526     Error(Parser.getTok().getLoc(), ErrMsg);
3527     return false;
3528   }
3529   return true;
3530 }
3531 
3532 bool
3533 AMDGPUAsmParser::parseExpr(int64_t &Imm) {
3534   return !getParser().parseAbsoluteExpression(Imm);
3535 }
3536 
3537 bool
3538 AMDGPUAsmParser::parseString(StringRef &Val, const StringRef ErrMsg) {
3539   SMLoc S = Parser.getTok().getLoc();
3540   if (getLexer().getKind() == AsmToken::String) {
3541     Val = Parser.getTok().getStringContents();
3542     Parser.Lex();
3543     return true;
3544   } else {
3545     Error(S, ErrMsg);
3546     return false;
3547   }
3548 }
3549 
3550 //===----------------------------------------------------------------------===//
3551 // swizzle
3552 //===----------------------------------------------------------------------===//
3553 
3554 LLVM_READNONE
3555 static unsigned
3556 encodeBitmaskPerm(const unsigned AndMask,
3557                   const unsigned OrMask,
3558                   const unsigned XorMask) {
3559   using namespace llvm::AMDGPU::Swizzle;
3560 
3561   return BITMASK_PERM_ENC |
3562          (AndMask << BITMASK_AND_SHIFT) |
3563          (OrMask  << BITMASK_OR_SHIFT)  |
3564          (XorMask << BITMASK_XOR_SHIFT);
3565 }
3566 
3567 bool
3568 AMDGPUAsmParser::parseSwizzleOperands(const unsigned OpNum, int64_t* Op,
3569                                       const unsigned MinVal,
3570                                       const unsigned MaxVal,
3571                                       const StringRef ErrMsg) {
3572   for (unsigned i = 0; i < OpNum; ++i) {
3573     if (!skipToken(AsmToken::Comma, "expected a comma")){
3574       return false;
3575     }
3576     SMLoc ExprLoc = Parser.getTok().getLoc();
3577     if (!parseExpr(Op[i])) {
3578       return false;
3579     }
3580     if (Op[i] < MinVal || Op[i] > MaxVal) {
3581       Error(ExprLoc, ErrMsg);
3582       return false;
3583     }
3584   }
3585 
3586   return true;
3587 }
3588 
3589 bool
3590 AMDGPUAsmParser::parseSwizzleQuadPerm(int64_t &Imm) {
3591   using namespace llvm::AMDGPU::Swizzle;
3592 
3593   int64_t Lane[LANE_NUM];
3594   if (parseSwizzleOperands(LANE_NUM, Lane, 0, LANE_MAX,
3595                            "expected a 2-bit lane id")) {
3596     Imm = QUAD_PERM_ENC;
3597     for (auto i = 0; i < LANE_NUM; ++i) {
3598       Imm |= Lane[i] << (LANE_SHIFT * i);
3599     }
3600     return true;
3601   }
3602   return false;
3603 }
3604 
3605 bool
3606 AMDGPUAsmParser::parseSwizzleBroadcast(int64_t &Imm) {
3607   using namespace llvm::AMDGPU::Swizzle;
3608 
3609   SMLoc S = Parser.getTok().getLoc();
3610   int64_t GroupSize;
3611   int64_t LaneIdx;
3612 
3613   if (!parseSwizzleOperands(1, &GroupSize,
3614                             2, 32,
3615                             "group size must be in the interval [2,32]")) {
3616     return false;
3617   }
3618   if (!isPowerOf2_64(GroupSize)) {
3619     Error(S, "group size must be a power of two");
3620     return false;
3621   }
3622   if (parseSwizzleOperands(1, &LaneIdx,
3623                            0, GroupSize - 1,
3624                            "lane id must be in the interval [0,group size - 1]")) {
3625     Imm = encodeBitmaskPerm(BITMASK_MAX - GroupSize + 1, LaneIdx, 0);
3626     return true;
3627   }
3628   return false;
3629 }
3630 
3631 bool
3632 AMDGPUAsmParser::parseSwizzleReverse(int64_t &Imm) {
3633   using namespace llvm::AMDGPU::Swizzle;
3634 
3635   SMLoc S = Parser.getTok().getLoc();
3636   int64_t GroupSize;
3637 
3638   if (!parseSwizzleOperands(1, &GroupSize,
3639       2, 32, "group size must be in the interval [2,32]")) {
3640     return false;
3641   }
3642   if (!isPowerOf2_64(GroupSize)) {
3643     Error(S, "group size must be a power of two");
3644     return false;
3645   }
3646 
3647   Imm = encodeBitmaskPerm(BITMASK_MAX, 0, GroupSize - 1);
3648   return true;
3649 }
3650 
3651 bool
3652 AMDGPUAsmParser::parseSwizzleSwap(int64_t &Imm) {
3653   using namespace llvm::AMDGPU::Swizzle;
3654 
3655   SMLoc S = Parser.getTok().getLoc();
3656   int64_t GroupSize;
3657 
3658   if (!parseSwizzleOperands(1, &GroupSize,
3659       1, 16, "group size must be in the interval [1,16]")) {
3660     return false;
3661   }
3662   if (!isPowerOf2_64(GroupSize)) {
3663     Error(S, "group size must be a power of two");
3664     return false;
3665   }
3666 
3667   Imm = encodeBitmaskPerm(BITMASK_MAX, 0, GroupSize);
3668   return true;
3669 }
3670 
3671 bool
3672 AMDGPUAsmParser::parseSwizzleBitmaskPerm(int64_t &Imm) {
3673   using namespace llvm::AMDGPU::Swizzle;
3674 
3675   if (!skipToken(AsmToken::Comma, "expected a comma")) {
3676     return false;
3677   }
3678 
3679   StringRef Ctl;
3680   SMLoc StrLoc = Parser.getTok().getLoc();
3681   if (!parseString(Ctl)) {
3682     return false;
3683   }
3684   if (Ctl.size() != BITMASK_WIDTH) {
3685     Error(StrLoc, "expected a 5-character mask");
3686     return false;
3687   }
3688 
3689   unsigned AndMask = 0;
3690   unsigned OrMask = 0;
3691   unsigned XorMask = 0;
3692 
3693   for (size_t i = 0; i < Ctl.size(); ++i) {
3694     unsigned Mask = 1 << (BITMASK_WIDTH - 1 - i);
3695     switch(Ctl[i]) {
3696     default:
3697       Error(StrLoc, "invalid mask");
3698       return false;
3699     case '0':
3700       break;
3701     case '1':
3702       OrMask |= Mask;
3703       break;
3704     case 'p':
3705       AndMask |= Mask;
3706       break;
3707     case 'i':
3708       AndMask |= Mask;
3709       XorMask |= Mask;
3710       break;
3711     }
3712   }
3713 
3714   Imm = encodeBitmaskPerm(AndMask, OrMask, XorMask);
3715   return true;
3716 }
3717 
3718 bool
3719 AMDGPUAsmParser::parseSwizzleOffset(int64_t &Imm) {
3720 
3721   SMLoc OffsetLoc = Parser.getTok().getLoc();
3722 
3723   if (!parseExpr(Imm)) {
3724     return false;
3725   }
3726   if (!isUInt<16>(Imm)) {
3727     Error(OffsetLoc, "expected a 16-bit offset");
3728     return false;
3729   }
3730   return true;
3731 }
3732 
3733 bool
3734 AMDGPUAsmParser::parseSwizzleMacro(int64_t &Imm) {
3735   using namespace llvm::AMDGPU::Swizzle;
3736 
3737   if (skipToken(AsmToken::LParen, "expected a left parentheses")) {
3738 
3739     SMLoc ModeLoc = Parser.getTok().getLoc();
3740     bool Ok = false;
3741 
3742     if (trySkipId(IdSymbolic[ID_QUAD_PERM])) {
3743       Ok = parseSwizzleQuadPerm(Imm);
3744     } else if (trySkipId(IdSymbolic[ID_BITMASK_PERM])) {
3745       Ok = parseSwizzleBitmaskPerm(Imm);
3746     } else if (trySkipId(IdSymbolic[ID_BROADCAST])) {
3747       Ok = parseSwizzleBroadcast(Imm);
3748     } else if (trySkipId(IdSymbolic[ID_SWAP])) {
3749       Ok = parseSwizzleSwap(Imm);
3750     } else if (trySkipId(IdSymbolic[ID_REVERSE])) {
3751       Ok = parseSwizzleReverse(Imm);
3752     } else {
3753       Error(ModeLoc, "expected a swizzle mode");
3754     }
3755 
3756     return Ok && skipToken(AsmToken::RParen, "expected a closing parentheses");
3757   }
3758 
3759   return false;
3760 }
3761 
3762 OperandMatchResultTy
3763 AMDGPUAsmParser::parseSwizzleOp(OperandVector &Operands) {
3764   SMLoc S = Parser.getTok().getLoc();
3765   int64_t Imm = 0;
3766 
3767   if (trySkipId("offset")) {
3768 
3769     bool Ok = false;
3770     if (skipToken(AsmToken::Colon, "expected a colon")) {
3771       if (trySkipId("swizzle")) {
3772         Ok = parseSwizzleMacro(Imm);
3773       } else {
3774         Ok = parseSwizzleOffset(Imm);
3775       }
3776     }
3777 
3778     Operands.push_back(AMDGPUOperand::CreateImm(this, Imm, S, AMDGPUOperand::ImmTySwizzle));
3779 
3780     return Ok? MatchOperand_Success : MatchOperand_ParseFail;
3781   } else {
3782     return MatchOperand_NoMatch;
3783   }
3784 }
3785 
3786 bool
3787 AMDGPUOperand::isSwizzle() const {
3788   return isImmTy(ImmTySwizzle);
3789 }
3790 
3791 //===----------------------------------------------------------------------===//
3792 // sopp branch targets
3793 //===----------------------------------------------------------------------===//
3794 
3795 OperandMatchResultTy
3796 AMDGPUAsmParser::parseSOppBrTarget(OperandVector &Operands) {
3797   SMLoc S = Parser.getTok().getLoc();
3798 
3799   switch (getLexer().getKind()) {
3800     default: return MatchOperand_ParseFail;
3801     case AsmToken::Integer: {
3802       int64_t Imm;
3803       if (getParser().parseAbsoluteExpression(Imm))
3804         return MatchOperand_ParseFail;
3805       Operands.push_back(AMDGPUOperand::CreateImm(this, Imm, S));
3806       return MatchOperand_Success;
3807     }
3808 
3809     case AsmToken::Identifier:
3810       Operands.push_back(AMDGPUOperand::CreateExpr(this,
3811           MCSymbolRefExpr::create(getContext().getOrCreateSymbol(
3812                                   Parser.getTok().getString()), getContext()), S));
3813       Parser.Lex();
3814       return MatchOperand_Success;
3815   }
3816 }
3817 
3818 //===----------------------------------------------------------------------===//
3819 // mubuf
3820 //===----------------------------------------------------------------------===//
3821 
3822 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultGLC() const {
3823   return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTyGLC);
3824 }
3825 
3826 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultSLC() const {
3827   return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTySLC);
3828 }
3829 
3830 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultTFE() const {
3831   return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTyTFE);
3832 }
3833 
3834 void AMDGPUAsmParser::cvtMubufImpl(MCInst &Inst,
3835                                const OperandVector &Operands,
3836                                bool IsAtomic, bool IsAtomicReturn) {
3837   OptionalImmIndexMap OptionalIdx;
3838   assert(IsAtomicReturn ? IsAtomic : true);
3839 
3840   for (unsigned i = 1, e = Operands.size(); i != e; ++i) {
3841     AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[i]);
3842 
3843     // Add the register arguments
3844     if (Op.isReg()) {
3845       Op.addRegOperands(Inst, 1);
3846       continue;
3847     }
3848 
3849     // Handle the case where soffset is an immediate
3850     if (Op.isImm() && Op.getImmTy() == AMDGPUOperand::ImmTyNone) {
3851       Op.addImmOperands(Inst, 1);
3852       continue;
3853     }
3854 
3855     // Handle tokens like 'offen' which are sometimes hard-coded into the
3856     // asm string.  There are no MCInst operands for these.
3857     if (Op.isToken()) {
3858       continue;
3859     }
3860     assert(Op.isImm());
3861 
3862     // Handle optional arguments
3863     OptionalIdx[Op.getImmTy()] = i;
3864   }
3865 
3866   // Copy $vdata_in operand and insert as $vdata for MUBUF_Atomic RTN insns.
3867   if (IsAtomicReturn) {
3868     MCInst::iterator I = Inst.begin(); // $vdata_in is always at the beginning.
3869     Inst.insert(I, *I);
3870   }
3871 
3872   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyOffset);
3873   if (!IsAtomic) { // glc is hard-coded.
3874     addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyGLC);
3875   }
3876   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySLC);
3877   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyTFE);
3878 }
3879 
3880 void AMDGPUAsmParser::cvtMtbuf(MCInst &Inst, const OperandVector &Operands) {
3881   OptionalImmIndexMap OptionalIdx;
3882 
3883   for (unsigned i = 1, e = Operands.size(); i != e; ++i) {
3884     AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[i]);
3885 
3886     // Add the register arguments
3887     if (Op.isReg()) {
3888       Op.addRegOperands(Inst, 1);
3889       continue;
3890     }
3891 
3892     // Handle the case where soffset is an immediate
3893     if (Op.isImm() && Op.getImmTy() == AMDGPUOperand::ImmTyNone) {
3894       Op.addImmOperands(Inst, 1);
3895       continue;
3896     }
3897 
3898     // Handle tokens like 'offen' which are sometimes hard-coded into the
3899     // asm string.  There are no MCInst operands for these.
3900     if (Op.isToken()) {
3901       continue;
3902     }
3903     assert(Op.isImm());
3904 
3905     // Handle optional arguments
3906     OptionalIdx[Op.getImmTy()] = i;
3907   }
3908 
3909   addOptionalImmOperand(Inst, Operands, OptionalIdx,
3910                         AMDGPUOperand::ImmTyOffset);
3911   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyDFMT);
3912   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyNFMT);
3913   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyGLC);
3914   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySLC);
3915   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyTFE);
3916 }
3917 
3918 //===----------------------------------------------------------------------===//
3919 // mimg
3920 //===----------------------------------------------------------------------===//
3921 
3922 void AMDGPUAsmParser::cvtMIMG(MCInst &Inst, const OperandVector &Operands,
3923                               bool IsAtomic) {
3924   unsigned I = 1;
3925   const MCInstrDesc &Desc = MII.get(Inst.getOpcode());
3926   for (unsigned J = 0; J < Desc.getNumDefs(); ++J) {
3927     ((AMDGPUOperand &)*Operands[I++]).addRegOperands(Inst, 1);
3928   }
3929 
3930   if (IsAtomic) {
3931     // Add src, same as dst
3932     ((AMDGPUOperand &)*Operands[I]).addRegOperands(Inst, 1);
3933   }
3934 
3935   OptionalImmIndexMap OptionalIdx;
3936 
3937   for (unsigned E = Operands.size(); I != E; ++I) {
3938     AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[I]);
3939 
3940     // Add the register arguments
3941     if (Op.isRegOrImm()) {
3942       Op.addRegOrImmOperands(Inst, 1);
3943       continue;
3944     } else if (Op.isImmModifier()) {
3945       OptionalIdx[Op.getImmTy()] = I;
3946     } else {
3947       llvm_unreachable("unexpected operand type");
3948     }
3949   }
3950 
3951   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyDMask);
3952   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyUNorm);
3953   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyGLC);
3954   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyDA);
3955   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyR128);
3956   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyTFE);
3957   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyLWE);
3958   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySLC);
3959 }
3960 
3961 void AMDGPUAsmParser::cvtMIMGAtomic(MCInst &Inst, const OperandVector &Operands) {
3962   cvtMIMG(Inst, Operands, true);
3963 }
3964 
3965 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultDMask() const {
3966   return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTyDMask);
3967 }
3968 
3969 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultUNorm() const {
3970   return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTyUNorm);
3971 }
3972 
3973 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultDA() const {
3974   return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTyDA);
3975 }
3976 
3977 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultR128() const {
3978   return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTyR128);
3979 }
3980 
3981 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultLWE() const {
3982   return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTyLWE);
3983 }
3984 
3985 //===----------------------------------------------------------------------===//
3986 // smrd
3987 //===----------------------------------------------------------------------===//
3988 
3989 bool AMDGPUOperand::isSMRDOffset8() const {
3990   return isImm() && isUInt<8>(getImm());
3991 }
3992 
3993 bool AMDGPUOperand::isSMRDOffset20() const {
3994   return isImm() && isUInt<20>(getImm());
3995 }
3996 
3997 bool AMDGPUOperand::isSMRDLiteralOffset() const {
3998   // 32-bit literals are only supported on CI and we only want to use them
3999   // when the offset is > 8-bits.
4000   return isImm() && !isUInt<8>(getImm()) && isUInt<32>(getImm());
4001 }
4002 
4003 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultSMRDOffset8() const {
4004   return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTyOffset);
4005 }
4006 
4007 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultSMRDOffset20() const {
4008   return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTyOffset);
4009 }
4010 
4011 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultSMRDLiteralOffset() const {
4012   return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTyOffset);
4013 }
4014 
4015 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultOffsetU12() const {
4016   return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTyOffset);
4017 }
4018 
4019 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultOffsetS13() const {
4020   return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTyOffset);
4021 }
4022 
4023 //===----------------------------------------------------------------------===//
4024 // vop3
4025 //===----------------------------------------------------------------------===//
4026 
4027 static bool ConvertOmodMul(int64_t &Mul) {
4028   if (Mul != 1 && Mul != 2 && Mul != 4)
4029     return false;
4030 
4031   Mul >>= 1;
4032   return true;
4033 }
4034 
4035 static bool ConvertOmodDiv(int64_t &Div) {
4036   if (Div == 1) {
4037     Div = 0;
4038     return true;
4039   }
4040 
4041   if (Div == 2) {
4042     Div = 3;
4043     return true;
4044   }
4045 
4046   return false;
4047 }
4048 
4049 static bool ConvertBoundCtrl(int64_t &BoundCtrl) {
4050   if (BoundCtrl == 0) {
4051     BoundCtrl = 1;
4052     return true;
4053   }
4054 
4055   if (BoundCtrl == -1) {
4056     BoundCtrl = 0;
4057     return true;
4058   }
4059 
4060   return false;
4061 }
4062 
4063 // Note: the order in this table matches the order of operands in AsmString.
4064 static const OptionalOperand AMDGPUOptionalOperandTable[] = {
4065   {"offen",   AMDGPUOperand::ImmTyOffen, true, nullptr},
4066   {"idxen",   AMDGPUOperand::ImmTyIdxen, true, nullptr},
4067   {"addr64",  AMDGPUOperand::ImmTyAddr64, true, nullptr},
4068   {"offset0", AMDGPUOperand::ImmTyOffset0, false, nullptr},
4069   {"offset1", AMDGPUOperand::ImmTyOffset1, false, nullptr},
4070   {"gds",     AMDGPUOperand::ImmTyGDS, true, nullptr},
4071   {"offset",  AMDGPUOperand::ImmTyOffset, false, nullptr},
4072   {"dfmt",    AMDGPUOperand::ImmTyDFMT, false, nullptr},
4073   {"nfmt",    AMDGPUOperand::ImmTyNFMT, false, nullptr},
4074   {"glc",     AMDGPUOperand::ImmTyGLC, true, nullptr},
4075   {"slc",     AMDGPUOperand::ImmTySLC, true, nullptr},
4076   {"tfe",     AMDGPUOperand::ImmTyTFE, true, nullptr},
4077   {"high",    AMDGPUOperand::ImmTyHigh, true, nullptr},
4078   {"clamp",   AMDGPUOperand::ImmTyClampSI, true, nullptr},
4079   {"omod",    AMDGPUOperand::ImmTyOModSI, false, ConvertOmodMul},
4080   {"unorm",   AMDGPUOperand::ImmTyUNorm, true, nullptr},
4081   {"da",      AMDGPUOperand::ImmTyDA,    true, nullptr},
4082   {"r128",    AMDGPUOperand::ImmTyR128,  true, nullptr},
4083   {"lwe",     AMDGPUOperand::ImmTyLWE,   true, nullptr},
4084   {"dmask",   AMDGPUOperand::ImmTyDMask, false, nullptr},
4085   {"row_mask",   AMDGPUOperand::ImmTyDppRowMask, false, nullptr},
4086   {"bank_mask",  AMDGPUOperand::ImmTyDppBankMask, false, nullptr},
4087   {"bound_ctrl", AMDGPUOperand::ImmTyDppBoundCtrl, false, ConvertBoundCtrl},
4088   {"dst_sel",    AMDGPUOperand::ImmTySdwaDstSel, false, nullptr},
4089   {"src0_sel",   AMDGPUOperand::ImmTySdwaSrc0Sel, false, nullptr},
4090   {"src1_sel",   AMDGPUOperand::ImmTySdwaSrc1Sel, false, nullptr},
4091   {"dst_unused", AMDGPUOperand::ImmTySdwaDstUnused, false, nullptr},
4092   {"compr", AMDGPUOperand::ImmTyExpCompr, true, nullptr },
4093   {"vm", AMDGPUOperand::ImmTyExpVM, true, nullptr},
4094   {"op_sel", AMDGPUOperand::ImmTyOpSel, false, nullptr},
4095   {"op_sel_hi", AMDGPUOperand::ImmTyOpSelHi, false, nullptr},
4096   {"neg_lo", AMDGPUOperand::ImmTyNegLo, false, nullptr},
4097   {"neg_hi", AMDGPUOperand::ImmTyNegHi, false, nullptr}
4098 };
4099 
4100 OperandMatchResultTy AMDGPUAsmParser::parseOptionalOperand(OperandVector &Operands) {
4101   OperandMatchResultTy res;
4102   for (const OptionalOperand &Op : AMDGPUOptionalOperandTable) {
4103     // try to parse any optional operand here
4104     if (Op.IsBit) {
4105       res = parseNamedBit(Op.Name, Operands, Op.Type);
4106     } else if (Op.Type == AMDGPUOperand::ImmTyOModSI) {
4107       res = parseOModOperand(Operands);
4108     } else if (Op.Type == AMDGPUOperand::ImmTySdwaDstSel ||
4109                Op.Type == AMDGPUOperand::ImmTySdwaSrc0Sel ||
4110                Op.Type == AMDGPUOperand::ImmTySdwaSrc1Sel) {
4111       res = parseSDWASel(Operands, Op.Name, Op.Type);
4112     } else if (Op.Type == AMDGPUOperand::ImmTySdwaDstUnused) {
4113       res = parseSDWADstUnused(Operands);
4114     } else if (Op.Type == AMDGPUOperand::ImmTyOpSel ||
4115                Op.Type == AMDGPUOperand::ImmTyOpSelHi ||
4116                Op.Type == AMDGPUOperand::ImmTyNegLo ||
4117                Op.Type == AMDGPUOperand::ImmTyNegHi) {
4118       res = parseOperandArrayWithPrefix(Op.Name, Operands, Op.Type,
4119                                         Op.ConvertResult);
4120     } else {
4121       res = parseIntWithPrefix(Op.Name, Operands, Op.Type, Op.ConvertResult);
4122     }
4123     if (res != MatchOperand_NoMatch) {
4124       return res;
4125     }
4126   }
4127   return MatchOperand_NoMatch;
4128 }
4129 
4130 OperandMatchResultTy AMDGPUAsmParser::parseOModOperand(OperandVector &Operands) {
4131   StringRef Name = Parser.getTok().getString();
4132   if (Name == "mul") {
4133     return parseIntWithPrefix("mul", Operands,
4134                               AMDGPUOperand::ImmTyOModSI, ConvertOmodMul);
4135   }
4136 
4137   if (Name == "div") {
4138     return parseIntWithPrefix("div", Operands,
4139                               AMDGPUOperand::ImmTyOModSI, ConvertOmodDiv);
4140   }
4141 
4142   return MatchOperand_NoMatch;
4143 }
4144 
4145 void AMDGPUAsmParser::cvtVOP3OpSel(MCInst &Inst, const OperandVector &Operands) {
4146   cvtVOP3P(Inst, Operands);
4147 
4148   int Opc = Inst.getOpcode();
4149 
4150   int SrcNum;
4151   const int Ops[] = { AMDGPU::OpName::src0,
4152                       AMDGPU::OpName::src1,
4153                       AMDGPU::OpName::src2 };
4154   for (SrcNum = 0;
4155        SrcNum < 3 && AMDGPU::getNamedOperandIdx(Opc, Ops[SrcNum]) != -1;
4156        ++SrcNum);
4157   assert(SrcNum > 0);
4158 
4159   int OpSelIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::op_sel);
4160   unsigned OpSel = Inst.getOperand(OpSelIdx).getImm();
4161 
4162   if ((OpSel & (1 << SrcNum)) != 0) {
4163     int ModIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src0_modifiers);
4164     uint32_t ModVal = Inst.getOperand(ModIdx).getImm();
4165     Inst.getOperand(ModIdx).setImm(ModVal | SISrcMods::DST_OP_SEL);
4166   }
4167 }
4168 
4169 static bool isRegOrImmWithInputMods(const MCInstrDesc &Desc, unsigned OpNum) {
4170       // 1. This operand is input modifiers
4171   return Desc.OpInfo[OpNum].OperandType == AMDGPU::OPERAND_INPUT_MODS
4172       // 2. This is not last operand
4173       && Desc.NumOperands > (OpNum + 1)
4174       // 3. Next operand is register class
4175       && Desc.OpInfo[OpNum + 1].RegClass != -1
4176       // 4. Next register is not tied to any other operand
4177       && Desc.getOperandConstraint(OpNum + 1, MCOI::OperandConstraint::TIED_TO) == -1;
4178 }
4179 
4180 void AMDGPUAsmParser::cvtVOP3Interp(MCInst &Inst, const OperandVector &Operands)
4181 {
4182   OptionalImmIndexMap OptionalIdx;
4183   unsigned Opc = Inst.getOpcode();
4184 
4185   unsigned I = 1;
4186   const MCInstrDesc &Desc = MII.get(Inst.getOpcode());
4187   for (unsigned J = 0; J < Desc.getNumDefs(); ++J) {
4188     ((AMDGPUOperand &)*Operands[I++]).addRegOperands(Inst, 1);
4189   }
4190 
4191   for (unsigned E = Operands.size(); I != E; ++I) {
4192     AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[I]);
4193     if (isRegOrImmWithInputMods(Desc, Inst.getNumOperands())) {
4194       Op.addRegOrImmWithFPInputModsOperands(Inst, 2);
4195     } else if (Op.isInterpSlot() ||
4196                Op.isInterpAttr() ||
4197                Op.isAttrChan()) {
4198       Inst.addOperand(MCOperand::createImm(Op.Imm.Val));
4199     } else if (Op.isImmModifier()) {
4200       OptionalIdx[Op.getImmTy()] = I;
4201     } else {
4202       llvm_unreachable("unhandled operand type");
4203     }
4204   }
4205 
4206   if (AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::high) != -1) {
4207     addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyHigh);
4208   }
4209 
4210   if (AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::clamp) != -1) {
4211     addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyClampSI);
4212   }
4213 
4214   if (AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::omod) != -1) {
4215     addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyOModSI);
4216   }
4217 }
4218 
4219 void AMDGPUAsmParser::cvtVOP3(MCInst &Inst, const OperandVector &Operands,
4220                               OptionalImmIndexMap &OptionalIdx) {
4221   unsigned Opc = Inst.getOpcode();
4222 
4223   unsigned I = 1;
4224   const MCInstrDesc &Desc = MII.get(Inst.getOpcode());
4225   for (unsigned J = 0; J < Desc.getNumDefs(); ++J) {
4226     ((AMDGPUOperand &)*Operands[I++]).addRegOperands(Inst, 1);
4227   }
4228 
4229   if (AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src0_modifiers) != -1) {
4230     // This instruction has src modifiers
4231     for (unsigned E = Operands.size(); I != E; ++I) {
4232       AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[I]);
4233       if (isRegOrImmWithInputMods(Desc, Inst.getNumOperands())) {
4234         Op.addRegOrImmWithFPInputModsOperands(Inst, 2);
4235       } else if (Op.isImmModifier()) {
4236         OptionalIdx[Op.getImmTy()] = I;
4237       } else if (Op.isRegOrImm()) {
4238         Op.addRegOrImmOperands(Inst, 1);
4239       } else {
4240         llvm_unreachable("unhandled operand type");
4241       }
4242     }
4243   } else {
4244     // No src modifiers
4245     for (unsigned E = Operands.size(); I != E; ++I) {
4246       AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[I]);
4247       if (Op.isMod()) {
4248         OptionalIdx[Op.getImmTy()] = I;
4249       } else {
4250         Op.addRegOrImmOperands(Inst, 1);
4251       }
4252     }
4253   }
4254 
4255   if (AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::clamp) != -1) {
4256     addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyClampSI);
4257   }
4258 
4259   if (AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::omod) != -1) {
4260     addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyOModSI);
4261   }
4262 
4263   // special case v_mac_{f16, f32}:
4264   // it has src2 register operand that is tied to dst operand
4265   // we don't allow modifiers for this operand in assembler so src2_modifiers
4266   // should be 0
4267   if (Opc == AMDGPU::V_MAC_F32_e64_si || Opc == AMDGPU::V_MAC_F32_e64_vi ||
4268       Opc == AMDGPU::V_MAC_F16_e64_vi) {
4269     auto it = Inst.begin();
4270     std::advance(it, AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src2_modifiers));
4271     it = Inst.insert(it, MCOperand::createImm(0)); // no modifiers for src2
4272     ++it;
4273     Inst.insert(it, Inst.getOperand(0)); // src2 = dst
4274   }
4275 }
4276 
4277 void AMDGPUAsmParser::cvtVOP3(MCInst &Inst, const OperandVector &Operands) {
4278   OptionalImmIndexMap OptionalIdx;
4279   cvtVOP3(Inst, Operands, OptionalIdx);
4280 }
4281 
4282 void AMDGPUAsmParser::cvtVOP3PImpl(MCInst &Inst,
4283                                    const OperandVector &Operands,
4284                                    bool IsPacked) {
4285   OptionalImmIndexMap OptIdx;
4286   int Opc = Inst.getOpcode();
4287 
4288   cvtVOP3(Inst, Operands, OptIdx);
4289 
4290   if (AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::vdst_in) != -1) {
4291     assert(!IsPacked);
4292     Inst.addOperand(Inst.getOperand(0));
4293   }
4294 
4295   // FIXME: This is messy. Parse the modifiers as if it was a normal VOP3
4296   // instruction, and then figure out where to actually put the modifiers
4297 
4298   addOptionalImmOperand(Inst, Operands, OptIdx, AMDGPUOperand::ImmTyOpSel);
4299 
4300   int OpSelHiIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::op_sel_hi);
4301   if (OpSelHiIdx != -1) {
4302     // TODO: Should we change the printing to match?
4303     int DefaultVal = IsPacked ? -1 : 0;
4304     addOptionalImmOperand(Inst, Operands, OptIdx, AMDGPUOperand::ImmTyOpSelHi,
4305                           DefaultVal);
4306   }
4307 
4308   int NegLoIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::neg_lo);
4309   if (NegLoIdx != -1) {
4310     assert(IsPacked);
4311     addOptionalImmOperand(Inst, Operands, OptIdx, AMDGPUOperand::ImmTyNegLo);
4312     addOptionalImmOperand(Inst, Operands, OptIdx, AMDGPUOperand::ImmTyNegHi);
4313   }
4314 
4315   const int Ops[] = { AMDGPU::OpName::src0,
4316                       AMDGPU::OpName::src1,
4317                       AMDGPU::OpName::src2 };
4318   const int ModOps[] = { AMDGPU::OpName::src0_modifiers,
4319                          AMDGPU::OpName::src1_modifiers,
4320                          AMDGPU::OpName::src2_modifiers };
4321 
4322   int OpSelIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::op_sel);
4323 
4324   unsigned OpSel = Inst.getOperand(OpSelIdx).getImm();
4325   unsigned OpSelHi = 0;
4326   unsigned NegLo = 0;
4327   unsigned NegHi = 0;
4328 
4329   if (OpSelHiIdx != -1) {
4330     OpSelHi = Inst.getOperand(OpSelHiIdx).getImm();
4331   }
4332 
4333   if (NegLoIdx != -1) {
4334     int NegHiIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::neg_hi);
4335     NegLo = Inst.getOperand(NegLoIdx).getImm();
4336     NegHi = Inst.getOperand(NegHiIdx).getImm();
4337   }
4338 
4339   for (int J = 0; J < 3; ++J) {
4340     int OpIdx = AMDGPU::getNamedOperandIdx(Opc, Ops[J]);
4341     if (OpIdx == -1)
4342       break;
4343 
4344     uint32_t ModVal = 0;
4345 
4346     if ((OpSel & (1 << J)) != 0)
4347       ModVal |= SISrcMods::OP_SEL_0;
4348 
4349     if ((OpSelHi & (1 << J)) != 0)
4350       ModVal |= SISrcMods::OP_SEL_1;
4351 
4352     if ((NegLo & (1 << J)) != 0)
4353       ModVal |= SISrcMods::NEG;
4354 
4355     if ((NegHi & (1 << J)) != 0)
4356       ModVal |= SISrcMods::NEG_HI;
4357 
4358     int ModIdx = AMDGPU::getNamedOperandIdx(Opc, ModOps[J]);
4359 
4360     Inst.getOperand(ModIdx).setImm(Inst.getOperand(ModIdx).getImm() | ModVal);
4361   }
4362 }
4363 
4364 void AMDGPUAsmParser::cvtVOP3P(MCInst &Inst, const OperandVector &Operands) {
4365   cvtVOP3PImpl(Inst, Operands, true);
4366 }
4367 
4368 void AMDGPUAsmParser::cvtVOP3P_NotPacked(MCInst &Inst,
4369                                          const OperandVector &Operands) {
4370   cvtVOP3PImpl(Inst, Operands, false);
4371 }
4372 
4373 //===----------------------------------------------------------------------===//
4374 // dpp
4375 //===----------------------------------------------------------------------===//
4376 
4377 bool AMDGPUOperand::isDPPCtrl() const {
4378   bool result = isImm() && getImmTy() == ImmTyDppCtrl && isUInt<9>(getImm());
4379   if (result) {
4380     int64_t Imm = getImm();
4381     return ((Imm >= 0x000) && (Imm <= 0x0ff)) ||
4382            ((Imm >= 0x101) && (Imm <= 0x10f)) ||
4383            ((Imm >= 0x111) && (Imm <= 0x11f)) ||
4384            ((Imm >= 0x121) && (Imm <= 0x12f)) ||
4385            (Imm == 0x130) ||
4386            (Imm == 0x134) ||
4387            (Imm == 0x138) ||
4388            (Imm == 0x13c) ||
4389            (Imm == 0x140) ||
4390            (Imm == 0x141) ||
4391            (Imm == 0x142) ||
4392            (Imm == 0x143);
4393   }
4394   return false;
4395 }
4396 
4397 bool AMDGPUOperand::isGPRIdxMode() const {
4398   return isImm() && isUInt<4>(getImm());
4399 }
4400 
4401 bool AMDGPUOperand::isS16Imm() const {
4402   return isImm() && (isInt<16>(getImm()) || isUInt<16>(getImm()));
4403 }
4404 
4405 bool AMDGPUOperand::isU16Imm() const {
4406   return isImm() && isUInt<16>(getImm());
4407 }
4408 
4409 OperandMatchResultTy
4410 AMDGPUAsmParser::parseDPPCtrl(OperandVector &Operands) {
4411   SMLoc S = Parser.getTok().getLoc();
4412   StringRef Prefix;
4413   int64_t Int;
4414 
4415   if (getLexer().getKind() == AsmToken::Identifier) {
4416     Prefix = Parser.getTok().getString();
4417   } else {
4418     return MatchOperand_NoMatch;
4419   }
4420 
4421   if (Prefix == "row_mirror") {
4422     Int = 0x140;
4423     Parser.Lex();
4424   } else if (Prefix == "row_half_mirror") {
4425     Int = 0x141;
4426     Parser.Lex();
4427   } else {
4428     // Check to prevent parseDPPCtrlOps from eating invalid tokens
4429     if (Prefix != "quad_perm"
4430         && Prefix != "row_shl"
4431         && Prefix != "row_shr"
4432         && Prefix != "row_ror"
4433         && Prefix != "wave_shl"
4434         && Prefix != "wave_rol"
4435         && Prefix != "wave_shr"
4436         && Prefix != "wave_ror"
4437         && Prefix != "row_bcast") {
4438       return MatchOperand_NoMatch;
4439     }
4440 
4441     Parser.Lex();
4442     if (getLexer().isNot(AsmToken::Colon))
4443       return MatchOperand_ParseFail;
4444 
4445     if (Prefix == "quad_perm") {
4446       // quad_perm:[%d,%d,%d,%d]
4447       Parser.Lex();
4448       if (getLexer().isNot(AsmToken::LBrac))
4449         return MatchOperand_ParseFail;
4450       Parser.Lex();
4451 
4452       if (getParser().parseAbsoluteExpression(Int) || !(0 <= Int && Int <=3))
4453         return MatchOperand_ParseFail;
4454 
4455       for (int i = 0; i < 3; ++i) {
4456         if (getLexer().isNot(AsmToken::Comma))
4457           return MatchOperand_ParseFail;
4458         Parser.Lex();
4459 
4460         int64_t Temp;
4461         if (getParser().parseAbsoluteExpression(Temp) || !(0 <= Temp && Temp <=3))
4462           return MatchOperand_ParseFail;
4463         const int shift = i*2 + 2;
4464         Int += (Temp << shift);
4465       }
4466 
4467       if (getLexer().isNot(AsmToken::RBrac))
4468         return MatchOperand_ParseFail;
4469       Parser.Lex();
4470     } else {
4471       // sel:%d
4472       Parser.Lex();
4473       if (getParser().parseAbsoluteExpression(Int))
4474         return MatchOperand_ParseFail;
4475 
4476       if (Prefix == "row_shl" && 1 <= Int && Int <= 15) {
4477         Int |= 0x100;
4478       } else if (Prefix == "row_shr" && 1 <= Int && Int <= 15) {
4479         Int |= 0x110;
4480       } else if (Prefix == "row_ror" && 1 <= Int && Int <= 15) {
4481         Int |= 0x120;
4482       } else if (Prefix == "wave_shl" && 1 == Int) {
4483         Int = 0x130;
4484       } else if (Prefix == "wave_rol" && 1 == Int) {
4485         Int = 0x134;
4486       } else if (Prefix == "wave_shr" && 1 == Int) {
4487         Int = 0x138;
4488       } else if (Prefix == "wave_ror" && 1 == Int) {
4489         Int = 0x13C;
4490       } else if (Prefix == "row_bcast") {
4491         if (Int == 15) {
4492           Int = 0x142;
4493         } else if (Int == 31) {
4494           Int = 0x143;
4495         } else {
4496           return MatchOperand_ParseFail;
4497         }
4498       } else {
4499         return MatchOperand_ParseFail;
4500       }
4501     }
4502   }
4503 
4504   Operands.push_back(AMDGPUOperand::CreateImm(this, Int, S, AMDGPUOperand::ImmTyDppCtrl));
4505   return MatchOperand_Success;
4506 }
4507 
4508 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultRowMask() const {
4509   return AMDGPUOperand::CreateImm(this, 0xf, SMLoc(), AMDGPUOperand::ImmTyDppRowMask);
4510 }
4511 
4512 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultBankMask() const {
4513   return AMDGPUOperand::CreateImm(this, 0xf, SMLoc(), AMDGPUOperand::ImmTyDppBankMask);
4514 }
4515 
4516 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultBoundCtrl() const {
4517   return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTyDppBoundCtrl);
4518 }
4519 
4520 void AMDGPUAsmParser::cvtDPP(MCInst &Inst, const OperandVector &Operands) {
4521   OptionalImmIndexMap OptionalIdx;
4522 
4523   unsigned I = 1;
4524   const MCInstrDesc &Desc = MII.get(Inst.getOpcode());
4525   for (unsigned J = 0; J < Desc.getNumDefs(); ++J) {
4526     ((AMDGPUOperand &)*Operands[I++]).addRegOperands(Inst, 1);
4527   }
4528 
4529   // All DPP instructions with at least one source operand have a fake "old"
4530   // source at the beginning that's tied to the dst operand. Handle it here.
4531   if (Desc.getNumOperands() >= 2)
4532     Inst.addOperand(Inst.getOperand(0));
4533 
4534   for (unsigned E = Operands.size(); I != E; ++I) {
4535     AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[I]);
4536     // Add the register arguments
4537     if (Op.isReg() && Op.Reg.RegNo == AMDGPU::VCC) {
4538       // VOP2b (v_add_u32, v_sub_u32 ...) dpp use "vcc" token.
4539       // Skip it.
4540       continue;
4541     } if (isRegOrImmWithInputMods(Desc, Inst.getNumOperands())) {
4542       Op.addRegWithFPInputModsOperands(Inst, 2);
4543     } else if (Op.isDPPCtrl()) {
4544       Op.addImmOperands(Inst, 1);
4545     } else if (Op.isImm()) {
4546       // Handle optional arguments
4547       OptionalIdx[Op.getImmTy()] = I;
4548     } else {
4549       llvm_unreachable("Invalid operand type");
4550     }
4551   }
4552 
4553   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyDppRowMask, 0xf);
4554   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyDppBankMask, 0xf);
4555   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyDppBoundCtrl);
4556 }
4557 
4558 //===----------------------------------------------------------------------===//
4559 // sdwa
4560 //===----------------------------------------------------------------------===//
4561 
4562 OperandMatchResultTy
4563 AMDGPUAsmParser::parseSDWASel(OperandVector &Operands, StringRef Prefix,
4564                               AMDGPUOperand::ImmTy Type) {
4565   using namespace llvm::AMDGPU::SDWA;
4566 
4567   SMLoc S = Parser.getTok().getLoc();
4568   StringRef Value;
4569   OperandMatchResultTy res;
4570 
4571   res = parseStringWithPrefix(Prefix, Value);
4572   if (res != MatchOperand_Success) {
4573     return res;
4574   }
4575 
4576   int64_t Int;
4577   Int = StringSwitch<int64_t>(Value)
4578         .Case("BYTE_0", SdwaSel::BYTE_0)
4579         .Case("BYTE_1", SdwaSel::BYTE_1)
4580         .Case("BYTE_2", SdwaSel::BYTE_2)
4581         .Case("BYTE_3", SdwaSel::BYTE_3)
4582         .Case("WORD_0", SdwaSel::WORD_0)
4583         .Case("WORD_1", SdwaSel::WORD_1)
4584         .Case("DWORD", SdwaSel::DWORD)
4585         .Default(0xffffffff);
4586   Parser.Lex(); // eat last token
4587 
4588   if (Int == 0xffffffff) {
4589     return MatchOperand_ParseFail;
4590   }
4591 
4592   Operands.push_back(AMDGPUOperand::CreateImm(this, Int, S, Type));
4593   return MatchOperand_Success;
4594 }
4595 
4596 OperandMatchResultTy
4597 AMDGPUAsmParser::parseSDWADstUnused(OperandVector &Operands) {
4598   using namespace llvm::AMDGPU::SDWA;
4599 
4600   SMLoc S = Parser.getTok().getLoc();
4601   StringRef Value;
4602   OperandMatchResultTy res;
4603 
4604   res = parseStringWithPrefix("dst_unused", Value);
4605   if (res != MatchOperand_Success) {
4606     return res;
4607   }
4608 
4609   int64_t Int;
4610   Int = StringSwitch<int64_t>(Value)
4611         .Case("UNUSED_PAD", DstUnused::UNUSED_PAD)
4612         .Case("UNUSED_SEXT", DstUnused::UNUSED_SEXT)
4613         .Case("UNUSED_PRESERVE", DstUnused::UNUSED_PRESERVE)
4614         .Default(0xffffffff);
4615   Parser.Lex(); // eat last token
4616 
4617   if (Int == 0xffffffff) {
4618     return MatchOperand_ParseFail;
4619   }
4620 
4621   Operands.push_back(AMDGPUOperand::CreateImm(this, Int, S, AMDGPUOperand::ImmTySdwaDstUnused));
4622   return MatchOperand_Success;
4623 }
4624 
4625 void AMDGPUAsmParser::cvtSdwaVOP1(MCInst &Inst, const OperandVector &Operands) {
4626   cvtSDWA(Inst, Operands, SIInstrFlags::VOP1);
4627 }
4628 
4629 void AMDGPUAsmParser::cvtSdwaVOP2(MCInst &Inst, const OperandVector &Operands) {
4630   cvtSDWA(Inst, Operands, SIInstrFlags::VOP2);
4631 }
4632 
4633 void AMDGPUAsmParser::cvtSdwaVOP2b(MCInst &Inst, const OperandVector &Operands) {
4634   cvtSDWA(Inst, Operands, SIInstrFlags::VOP2, true);
4635 }
4636 
4637 void AMDGPUAsmParser::cvtSdwaVOPC(MCInst &Inst, const OperandVector &Operands) {
4638   cvtSDWA(Inst, Operands, SIInstrFlags::VOPC, isVI());
4639 }
4640 
4641 void AMDGPUAsmParser::cvtSDWA(MCInst &Inst, const OperandVector &Operands,
4642                               uint64_t BasicInstType, bool skipVcc) {
4643   using namespace llvm::AMDGPU::SDWA;
4644 
4645   OptionalImmIndexMap OptionalIdx;
4646   bool skippedVcc = false;
4647 
4648   unsigned I = 1;
4649   const MCInstrDesc &Desc = MII.get(Inst.getOpcode());
4650   for (unsigned J = 0; J < Desc.getNumDefs(); ++J) {
4651     ((AMDGPUOperand &)*Operands[I++]).addRegOperands(Inst, 1);
4652   }
4653 
4654   for (unsigned E = Operands.size(); I != E; ++I) {
4655     AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[I]);
4656     if (skipVcc && !skippedVcc && Op.isReg() && Op.Reg.RegNo == AMDGPU::VCC) {
4657       // VOP2b (v_add_u32, v_sub_u32 ...) sdwa use "vcc" token as dst.
4658       // Skip it if it's 2nd (e.g. v_add_i32_sdwa v1, vcc, v2, v3)
4659       // or 4th (v_addc_u32_sdwa v1, vcc, v2, v3, vcc) operand.
4660       // Skip VCC only if we didn't skip it on previous iteration.
4661       if (BasicInstType == SIInstrFlags::VOP2 &&
4662           (Inst.getNumOperands() == 1 || Inst.getNumOperands() == 5)) {
4663         skippedVcc = true;
4664         continue;
4665       } else if (BasicInstType == SIInstrFlags::VOPC &&
4666                  Inst.getNumOperands() == 0) {
4667         skippedVcc = true;
4668         continue;
4669       }
4670     }
4671     if (isRegOrImmWithInputMods(Desc, Inst.getNumOperands())) {
4672       Op.addRegWithInputModsOperands(Inst, 2);
4673     } else if (Op.isImm()) {
4674       // Handle optional arguments
4675       OptionalIdx[Op.getImmTy()] = I;
4676     } else {
4677       llvm_unreachable("Invalid operand type");
4678     }
4679     skippedVcc = false;
4680   }
4681 
4682   if (Inst.getOpcode() != AMDGPU::V_NOP_sdwa_gfx9 &&
4683       Inst.getOpcode() != AMDGPU::V_NOP_sdwa_vi) {
4684     // v_nop_sdwa_sdwa_vi/gfx9 has no optional sdwa arguments
4685     switch (BasicInstType) {
4686     case SIInstrFlags::VOP1:
4687       addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyClampSI, 0);
4688       if (AMDGPU::getNamedOperandIdx(Inst.getOpcode(), AMDGPU::OpName::omod) != -1) {
4689         addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyOModSI, 0);
4690       }
4691       addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySdwaDstSel, SdwaSel::DWORD);
4692       addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySdwaDstUnused, DstUnused::UNUSED_PRESERVE);
4693       addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySdwaSrc0Sel, SdwaSel::DWORD);
4694       break;
4695 
4696     case SIInstrFlags::VOP2:
4697       addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyClampSI, 0);
4698       if (AMDGPU::getNamedOperandIdx(Inst.getOpcode(), AMDGPU::OpName::omod) != -1) {
4699         addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyOModSI, 0);
4700       }
4701       addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySdwaDstSel, SdwaSel::DWORD);
4702       addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySdwaDstUnused, DstUnused::UNUSED_PRESERVE);
4703       addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySdwaSrc0Sel, SdwaSel::DWORD);
4704       addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySdwaSrc1Sel, SdwaSel::DWORD);
4705       break;
4706 
4707     case SIInstrFlags::VOPC:
4708       addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyClampSI, 0);
4709       addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySdwaSrc0Sel, SdwaSel::DWORD);
4710       addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySdwaSrc1Sel, SdwaSel::DWORD);
4711       break;
4712 
4713     default:
4714       llvm_unreachable("Invalid instruction type. Only VOP1, VOP2 and VOPC allowed");
4715     }
4716   }
4717 
4718   // special case v_mac_{f16, f32}:
4719   // it has src2 register operand that is tied to dst operand
4720   if (Inst.getOpcode() == AMDGPU::V_MAC_F32_sdwa_vi ||
4721       Inst.getOpcode() == AMDGPU::V_MAC_F16_sdwa_vi)  {
4722     auto it = Inst.begin();
4723     std::advance(
4724       it, AMDGPU::getNamedOperandIdx(Inst.getOpcode(), AMDGPU::OpName::src2));
4725     Inst.insert(it, Inst.getOperand(0)); // src2 = dst
4726   }
4727 }
4728 
4729 /// Force static initialization.
4730 extern "C" void LLVMInitializeAMDGPUAsmParser() {
4731   RegisterMCAsmParser<AMDGPUAsmParser> A(getTheAMDGPUTarget());
4732   RegisterMCAsmParser<AMDGPUAsmParser> B(getTheGCNTarget());
4733 }
4734 
4735 #define GET_REGISTER_MATCHER
4736 #define GET_MATCHER_IMPLEMENTATION
4737 #include "AMDGPUGenAsmMatcher.inc"
4738 
4739 // This fuction should be defined after auto-generated include so that we have
4740 // MatchClassKind enum defined
4741 unsigned AMDGPUAsmParser::validateTargetOperandClass(MCParsedAsmOperand &Op,
4742                                                      unsigned Kind) {
4743   // Tokens like "glc" would be parsed as immediate operands in ParseOperand().
4744   // But MatchInstructionImpl() expects to meet token and fails to validate
4745   // operand. This method checks if we are given immediate operand but expect to
4746   // get corresponding token.
4747   AMDGPUOperand &Operand = (AMDGPUOperand&)Op;
4748   switch (Kind) {
4749   case MCK_addr64:
4750     return Operand.isAddr64() ? Match_Success : Match_InvalidOperand;
4751   case MCK_gds:
4752     return Operand.isGDS() ? Match_Success : Match_InvalidOperand;
4753   case MCK_glc:
4754     return Operand.isGLC() ? Match_Success : Match_InvalidOperand;
4755   case MCK_idxen:
4756     return Operand.isIdxen() ? Match_Success : Match_InvalidOperand;
4757   case MCK_offen:
4758     return Operand.isOffen() ? Match_Success : Match_InvalidOperand;
4759   case MCK_SSrcB32:
4760     // When operands have expression values, they will return true for isToken,
4761     // because it is not possible to distinguish between a token and an
4762     // expression at parse time. MatchInstructionImpl() will always try to
4763     // match an operand as a token, when isToken returns true, and when the
4764     // name of the expression is not a valid token, the match will fail,
4765     // so we need to handle it here.
4766     return Operand.isSSrcB32() ? Match_Success : Match_InvalidOperand;
4767   case MCK_SSrcF32:
4768     return Operand.isSSrcF32() ? Match_Success : Match_InvalidOperand;
4769   case MCK_SoppBrTarget:
4770     return Operand.isSoppBrTarget() ? Match_Success : Match_InvalidOperand;
4771   case MCK_VReg32OrOff:
4772     return Operand.isVReg32OrOff() ? Match_Success : Match_InvalidOperand;
4773   case MCK_InterpSlot:
4774     return Operand.isInterpSlot() ? Match_Success : Match_InvalidOperand;
4775   case MCK_Attr:
4776     return Operand.isInterpAttr() ? Match_Success : Match_InvalidOperand;
4777   case MCK_AttrChan:
4778     return Operand.isAttrChan() ? Match_Success : Match_InvalidOperand;
4779   default:
4780     return Match_InvalidOperand;
4781   }
4782 }
4783