1 //===-- AMDGPUAsmParser.cpp - Parse SI asm to MCInst instructions ---------===//
2 //
3 //                     The LLVM Compiler Infrastructure
4 //
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
7 //
8 //===----------------------------------------------------------------------===//
9 
10 #include "AMDKernelCodeT.h"
11 #include "MCTargetDesc/AMDGPUMCTargetDesc.h"
12 #include "MCTargetDesc/AMDGPUTargetStreamer.h"
13 #include "SIDefines.h"
14 #include "Utils/AMDGPUBaseInfo.h"
15 #include "Utils/AMDKernelCodeTUtils.h"
16 #include "Utils/AMDGPUAsmUtils.h"
17 #include "llvm/ADT/APFloat.h"
18 #include "llvm/ADT/APInt.h"
19 #include "llvm/ADT/ArrayRef.h"
20 #include "llvm/ADT/SmallBitVector.h"
21 #include "llvm/ADT/SmallString.h"
22 #include "llvm/ADT/STLExtras.h"
23 #include "llvm/ADT/StringRef.h"
24 #include "llvm/ADT/StringSwitch.h"
25 #include "llvm/ADT/Twine.h"
26 #include "llvm/CodeGen/MachineValueType.h"
27 #include "llvm/MC/MCAsmInfo.h"
28 #include "llvm/MC/MCContext.h"
29 #include "llvm/MC/MCExpr.h"
30 #include "llvm/MC/MCInst.h"
31 #include "llvm/MC/MCInstrDesc.h"
32 #include "llvm/MC/MCInstrInfo.h"
33 #include "llvm/MC/MCParser/MCAsmLexer.h"
34 #include "llvm/MC/MCParser/MCAsmParser.h"
35 #include "llvm/MC/MCParser/MCAsmParserExtension.h"
36 #include "llvm/MC/MCParser/MCParsedAsmOperand.h"
37 #include "llvm/MC/MCParser/MCTargetAsmParser.h"
38 #include "llvm/MC/MCRegisterInfo.h"
39 #include "llvm/MC/MCStreamer.h"
40 #include "llvm/MC/MCSubtargetInfo.h"
41 #include "llvm/MC/MCSymbol.h"
42 #include "llvm/Support/Casting.h"
43 #include "llvm/Support/ELF.h"
44 #include "llvm/Support/ErrorHandling.h"
45 #include "llvm/Support/MathExtras.h"
46 #include "llvm/Support/raw_ostream.h"
47 #include "llvm/Support/SMLoc.h"
48 #include "llvm/Support/TargetRegistry.h"
49 #include <algorithm>
50 #include <cassert>
51 #include <cstdint>
52 #include <cstring>
53 #include <iterator>
54 #include <map>
55 #include <memory>
56 #include <string>
57 
58 using namespace llvm;
59 using namespace llvm::AMDGPU;
60 
61 namespace {
62 
63 class AMDGPUAsmParser;
64 
65 enum RegisterKind { IS_UNKNOWN, IS_VGPR, IS_SGPR, IS_TTMP, IS_SPECIAL };
66 
67 //===----------------------------------------------------------------------===//
68 // Operand
69 //===----------------------------------------------------------------------===//
70 
71 class AMDGPUOperand : public MCParsedAsmOperand {
72   enum KindTy {
73     Token,
74     Immediate,
75     Register,
76     Expression
77   } Kind;
78 
79   SMLoc StartLoc, EndLoc;
80   const AMDGPUAsmParser *AsmParser;
81 
82 public:
83   AMDGPUOperand(KindTy Kind_, const AMDGPUAsmParser *AsmParser_)
84     : MCParsedAsmOperand(), Kind(Kind_), AsmParser(AsmParser_) {}
85 
86   typedef std::unique_ptr<AMDGPUOperand> Ptr;
87 
88   struct Modifiers {
89     bool Abs = false;
90     bool Neg = false;
91     bool Sext = false;
92 
93     bool hasFPModifiers() const { return Abs || Neg; }
94     bool hasIntModifiers() const { return Sext; }
95     bool hasModifiers() const { return hasFPModifiers() || hasIntModifiers(); }
96 
97     int64_t getFPModifiersOperand() const {
98       int64_t Operand = 0;
99       Operand |= Abs ? SISrcMods::ABS : 0;
100       Operand |= Neg ? SISrcMods::NEG : 0;
101       return Operand;
102     }
103 
104     int64_t getIntModifiersOperand() const {
105       int64_t Operand = 0;
106       Operand |= Sext ? SISrcMods::SEXT : 0;
107       return Operand;
108     }
109 
110     int64_t getModifiersOperand() const {
111       assert(!(hasFPModifiers() && hasIntModifiers())
112            && "fp and int modifiers should not be used simultaneously");
113       if (hasFPModifiers()) {
114         return getFPModifiersOperand();
115       } else if (hasIntModifiers()) {
116         return getIntModifiersOperand();
117       } else {
118         return 0;
119       }
120     }
121 
122     friend raw_ostream &operator <<(raw_ostream &OS, AMDGPUOperand::Modifiers Mods);
123   };
124 
125   enum ImmTy {
126     ImmTyNone,
127     ImmTyGDS,
128     ImmTyOffen,
129     ImmTyIdxen,
130     ImmTyAddr64,
131     ImmTyOffset,
132     ImmTyOffset0,
133     ImmTyOffset1,
134     ImmTyGLC,
135     ImmTySLC,
136     ImmTyTFE,
137     ImmTyClampSI,
138     ImmTyOModSI,
139     ImmTyDppCtrl,
140     ImmTyDppRowMask,
141     ImmTyDppBankMask,
142     ImmTyDppBoundCtrl,
143     ImmTySdwaDstSel,
144     ImmTySdwaSrc0Sel,
145     ImmTySdwaSrc1Sel,
146     ImmTySdwaDstUnused,
147     ImmTyDMask,
148     ImmTyUNorm,
149     ImmTyDA,
150     ImmTyR128,
151     ImmTyLWE,
152     ImmTyExpTgt,
153     ImmTyExpCompr,
154     ImmTyExpVM,
155     ImmTyHwreg,
156     ImmTyOff,
157     ImmTySendMsg,
158     ImmTyInterpSlot,
159     ImmTyInterpAttr,
160     ImmTyAttrChan,
161     ImmTyOpSel,
162     ImmTyOpSelHi,
163     ImmTyNegLo,
164     ImmTyNegHi
165   };
166 
167   struct TokOp {
168     const char *Data;
169     unsigned Length;
170   };
171 
172   struct ImmOp {
173     int64_t Val;
174     ImmTy Type;
175     bool IsFPImm;
176     Modifiers Mods;
177   };
178 
179   struct RegOp {
180     unsigned RegNo;
181     bool IsForcedVOP3;
182     Modifiers Mods;
183   };
184 
185   union {
186     TokOp Tok;
187     ImmOp Imm;
188     RegOp Reg;
189     const MCExpr *Expr;
190   };
191 
192   bool isToken() const override {
193     if (Kind == Token)
194       return true;
195 
196     if (Kind != Expression || !Expr)
197       return false;
198 
199     // When parsing operands, we can't always tell if something was meant to be
200     // a token, like 'gds', or an expression that references a global variable.
201     // In this case, we assume the string is an expression, and if we need to
202     // interpret is a token, then we treat the symbol name as the token.
203     return isa<MCSymbolRefExpr>(Expr);
204   }
205 
206   bool isImm() const override {
207     return Kind == Immediate;
208   }
209 
210   bool isInlinableImm(MVT type) const;
211   bool isLiteralImm(MVT type) const;
212 
213   bool isRegKind() const {
214     return Kind == Register;
215   }
216 
217   bool isReg() const override {
218     return isRegKind() && !hasModifiers();
219   }
220 
221   bool isRegOrImmWithInputMods(MVT type) const {
222     return isRegKind() || isInlinableImm(type);
223   }
224 
225   bool isRegOrImmWithInt16InputMods() const {
226     return isRegOrImmWithInputMods(MVT::i16);
227   }
228 
229   bool isRegOrImmWithInt32InputMods() const {
230     return isRegOrImmWithInputMods(MVT::i32);
231   }
232 
233   bool isRegOrImmWithInt64InputMods() const {
234     return isRegOrImmWithInputMods(MVT::i64);
235   }
236 
237   bool isRegOrImmWithFP16InputMods() const {
238     return isRegOrImmWithInputMods(MVT::f16);
239   }
240 
241   bool isRegOrImmWithFP32InputMods() const {
242     return isRegOrImmWithInputMods(MVT::f32);
243   }
244 
245   bool isRegOrImmWithFP64InputMods() const {
246     return isRegOrImmWithInputMods(MVT::f64);
247   }
248 
249   bool isVReg() const {
250     return isRegClass(AMDGPU::VGPR_32RegClassID) ||
251            isRegClass(AMDGPU::VReg_64RegClassID) ||
252            isRegClass(AMDGPU::VReg_96RegClassID) ||
253            isRegClass(AMDGPU::VReg_128RegClassID) ||
254            isRegClass(AMDGPU::VReg_256RegClassID) ||
255            isRegClass(AMDGPU::VReg_512RegClassID);
256   }
257 
258   bool isVReg32OrOff() const {
259     return isOff() || isRegClass(AMDGPU::VGPR_32RegClassID);
260   }
261 
262   bool isImmTy(ImmTy ImmT) const {
263     return isImm() && Imm.Type == ImmT;
264   }
265 
266   bool isImmModifier() const {
267     return isImm() && Imm.Type != ImmTyNone;
268   }
269 
270   bool isClampSI() const { return isImmTy(ImmTyClampSI); }
271   bool isOModSI() const { return isImmTy(ImmTyOModSI); }
272   bool isDMask() const { return isImmTy(ImmTyDMask); }
273   bool isUNorm() const { return isImmTy(ImmTyUNorm); }
274   bool isDA() const { return isImmTy(ImmTyDA); }
275   bool isR128() const { return isImmTy(ImmTyUNorm); }
276   bool isLWE() const { return isImmTy(ImmTyLWE); }
277   bool isOff() const { return isImmTy(ImmTyOff); }
278   bool isExpTgt() const { return isImmTy(ImmTyExpTgt); }
279   bool isExpVM() const { return isImmTy(ImmTyExpVM); }
280   bool isExpCompr() const { return isImmTy(ImmTyExpCompr); }
281   bool isOffen() const { return isImmTy(ImmTyOffen); }
282   bool isIdxen() const { return isImmTy(ImmTyIdxen); }
283   bool isAddr64() const { return isImmTy(ImmTyAddr64); }
284   bool isOffset() const { return isImmTy(ImmTyOffset) && isUInt<16>(getImm()); }
285   bool isOffset0() const { return isImmTy(ImmTyOffset0) && isUInt<16>(getImm()); }
286   bool isOffset1() const { return isImmTy(ImmTyOffset1) && isUInt<8>(getImm()); }
287   bool isGDS() const { return isImmTy(ImmTyGDS); }
288   bool isGLC() const { return isImmTy(ImmTyGLC); }
289   bool isSLC() const { return isImmTy(ImmTySLC); }
290   bool isTFE() const { return isImmTy(ImmTyTFE); }
291   bool isBankMask() const { return isImmTy(ImmTyDppBankMask); }
292   bool isRowMask() const { return isImmTy(ImmTyDppRowMask); }
293   bool isBoundCtrl() const { return isImmTy(ImmTyDppBoundCtrl); }
294   bool isSDWADstSel() const { return isImmTy(ImmTySdwaDstSel); }
295   bool isSDWASrc0Sel() const { return isImmTy(ImmTySdwaSrc0Sel); }
296   bool isSDWASrc1Sel() const { return isImmTy(ImmTySdwaSrc1Sel); }
297   bool isSDWADstUnused() const { return isImmTy(ImmTySdwaDstUnused); }
298   bool isInterpSlot() const { return isImmTy(ImmTyInterpSlot); }
299   bool isInterpAttr() const { return isImmTy(ImmTyInterpAttr); }
300   bool isAttrChan() const { return isImmTy(ImmTyAttrChan); }
301   bool isOpSel() const { return isImmTy(ImmTyOpSel); }
302   bool isOpSelHi() const { return isImmTy(ImmTyOpSelHi); }
303   bool isNegLo() const { return isImmTy(ImmTyNegLo); }
304   bool isNegHi() const { return isImmTy(ImmTyNegHi); }
305 
306   bool isMod() const {
307     return isClampSI() || isOModSI();
308   }
309 
310   bool isRegOrImm() const {
311     return isReg() || isImm();
312   }
313 
314   bool isRegClass(unsigned RCID) const;
315 
316   bool isRegOrInlineNoMods(unsigned RCID, MVT type) const {
317     return (isRegClass(RCID) || isInlinableImm(type)) && !hasModifiers();
318   }
319 
320   bool isSCSrcB16() const {
321     return isRegOrInlineNoMods(AMDGPU::SReg_32RegClassID, MVT::i16);
322   }
323 
324   bool isSCSrcV2B16() const {
325     return isSCSrcB16();
326   }
327 
328   bool isSCSrcB32() const {
329     return isRegOrInlineNoMods(AMDGPU::SReg_32RegClassID, MVT::i32);
330   }
331 
332   bool isSCSrcB64() const {
333     return isRegOrInlineNoMods(AMDGPU::SReg_64RegClassID, MVT::i64);
334   }
335 
336   bool isSCSrcF16() const {
337     return isRegOrInlineNoMods(AMDGPU::SReg_32RegClassID, MVT::f16);
338   }
339 
340   bool isSCSrcV2F16() const {
341     return isSCSrcF16();
342   }
343 
344   bool isSCSrcF32() const {
345     return isRegOrInlineNoMods(AMDGPU::SReg_32RegClassID, MVT::f32);
346   }
347 
348   bool isSCSrcF64() const {
349     return isRegOrInlineNoMods(AMDGPU::SReg_64RegClassID, MVT::f64);
350   }
351 
352   bool isSSrcB32() const {
353     return isSCSrcB32() || isLiteralImm(MVT::i32) || isExpr();
354   }
355 
356   bool isSSrcB16() const {
357     return isSCSrcB16() || isLiteralImm(MVT::i16);
358   }
359 
360   bool isSSrcV2B16() const {
361     llvm_unreachable("cannot happen");
362     return isSSrcB16();
363   }
364 
365   bool isSSrcB64() const {
366     // TODO: Find out how SALU supports extension of 32-bit literals to 64 bits.
367     // See isVSrc64().
368     return isSCSrcB64() || isLiteralImm(MVT::i64);
369   }
370 
371   bool isSSrcF32() const {
372     return isSCSrcB32() || isLiteralImm(MVT::f32) || isExpr();
373   }
374 
375   bool isSSrcF64() const {
376     return isSCSrcB64() || isLiteralImm(MVT::f64);
377   }
378 
379   bool isSSrcF16() const {
380     return isSCSrcB16() || isLiteralImm(MVT::f16);
381   }
382 
383   bool isSSrcV2F16() const {
384     llvm_unreachable("cannot happen");
385     return isSSrcF16();
386   }
387 
388   bool isVCSrcB32() const {
389     return isRegOrInlineNoMods(AMDGPU::VS_32RegClassID, MVT::i32);
390   }
391 
392   bool isVCSrcB64() const {
393     return isRegOrInlineNoMods(AMDGPU::VS_64RegClassID, MVT::i64);
394   }
395 
396   bool isVCSrcB16() const {
397     return isRegOrInlineNoMods(AMDGPU::VS_32RegClassID, MVT::i16);
398   }
399 
400   bool isVCSrcV2B16() const {
401     return isVCSrcB16();
402   }
403 
404   bool isVCSrcF32() const {
405     return isRegOrInlineNoMods(AMDGPU::VS_32RegClassID, MVT::f32);
406   }
407 
408   bool isVCSrcF64() const {
409     return isRegOrInlineNoMods(AMDGPU::VS_64RegClassID, MVT::f64);
410   }
411 
412   bool isVCSrcF16() const {
413     return isRegOrInlineNoMods(AMDGPU::VS_32RegClassID, MVT::f16);
414   }
415 
416   bool isVCSrcV2F16() const {
417     return isVCSrcF16();
418   }
419 
420   bool isVSrcB32() const {
421     return isVCSrcF32() || isLiteralImm(MVT::i32);
422   }
423 
424   bool isVSrcB64() const {
425     return isVCSrcF64() || isLiteralImm(MVT::i64);
426   }
427 
428   bool isVSrcB16() const {
429     return isVCSrcF16() || isLiteralImm(MVT::i16);
430   }
431 
432   bool isVSrcV2B16() const {
433     llvm_unreachable("cannot happen");
434     return isVSrcB16();
435   }
436 
437   bool isVSrcF32() const {
438     return isVCSrcF32() || isLiteralImm(MVT::f32);
439   }
440 
441   bool isVSrcF64() const {
442     return isVCSrcF64() || isLiteralImm(MVT::f64);
443   }
444 
445   bool isVSrcF16() const {
446     return isVCSrcF16() || isLiteralImm(MVT::f16);
447   }
448 
449   bool isVSrcV2F16() const {
450     llvm_unreachable("cannot happen");
451     return isVSrcF16();
452   }
453 
454   bool isKImmFP32() const {
455     return isLiteralImm(MVT::f32);
456   }
457 
458   bool isKImmFP16() const {
459     return isLiteralImm(MVT::f16);
460   }
461 
462   bool isMem() const override {
463     return false;
464   }
465 
466   bool isExpr() const {
467     return Kind == Expression;
468   }
469 
470   bool isSoppBrTarget() const {
471     return isExpr() || isImm();
472   }
473 
474   bool isSWaitCnt() const;
475   bool isHwreg() const;
476   bool isSendMsg() const;
477   bool isSMRDOffset8() const;
478   bool isSMRDOffset20() const;
479   bool isSMRDLiteralOffset() const;
480   bool isDPPCtrl() const;
481   bool isGPRIdxMode() const;
482 
483   StringRef getExpressionAsToken() const {
484     assert(isExpr());
485     const MCSymbolRefExpr *S = cast<MCSymbolRefExpr>(Expr);
486     return S->getSymbol().getName();
487   }
488 
489   StringRef getToken() const {
490     assert(isToken());
491 
492     if (Kind == Expression)
493       return getExpressionAsToken();
494 
495     return StringRef(Tok.Data, Tok.Length);
496   }
497 
498   int64_t getImm() const {
499     assert(isImm());
500     return Imm.Val;
501   }
502 
503   ImmTy getImmTy() const {
504     assert(isImm());
505     return Imm.Type;
506   }
507 
508   unsigned getReg() const override {
509     return Reg.RegNo;
510   }
511 
512   SMLoc getStartLoc() const override {
513     return StartLoc;
514   }
515 
516   SMLoc getEndLoc() const override {
517     return EndLoc;
518   }
519 
520   Modifiers getModifiers() const {
521     assert(isRegKind() || isImmTy(ImmTyNone));
522     return isRegKind() ? Reg.Mods : Imm.Mods;
523   }
524 
525   void setModifiers(Modifiers Mods) {
526     assert(isRegKind() || isImmTy(ImmTyNone));
527     if (isRegKind())
528       Reg.Mods = Mods;
529     else
530       Imm.Mods = Mods;
531   }
532 
533   bool hasModifiers() const {
534     return getModifiers().hasModifiers();
535   }
536 
537   bool hasFPModifiers() const {
538     return getModifiers().hasFPModifiers();
539   }
540 
541   bool hasIntModifiers() const {
542     return getModifiers().hasIntModifiers();
543   }
544 
545   uint64_t applyInputFPModifiers(uint64_t Val, unsigned Size) const;
546 
547   void addImmOperands(MCInst &Inst, unsigned N, bool ApplyModifiers = true) const;
548 
549   void addLiteralImmOperand(MCInst &Inst, int64_t Val, bool ApplyModifiers) const;
550 
551   template <unsigned Bitwidth>
552   void addKImmFPOperands(MCInst &Inst, unsigned N) const;
553 
554   void addKImmFP16Operands(MCInst &Inst, unsigned N) const {
555     addKImmFPOperands<16>(Inst, N);
556   }
557 
558   void addKImmFP32Operands(MCInst &Inst, unsigned N) const {
559     addKImmFPOperands<32>(Inst, N);
560   }
561 
562   void addRegOperands(MCInst &Inst, unsigned N) const;
563 
564   void addRegOrImmOperands(MCInst &Inst, unsigned N) const {
565     if (isRegKind())
566       addRegOperands(Inst, N);
567     else if (isExpr())
568       Inst.addOperand(MCOperand::createExpr(Expr));
569     else
570       addImmOperands(Inst, N);
571   }
572 
573   void addRegOrImmWithInputModsOperands(MCInst &Inst, unsigned N) const {
574     Modifiers Mods = getModifiers();
575     Inst.addOperand(MCOperand::createImm(Mods.getModifiersOperand()));
576     if (isRegKind()) {
577       addRegOperands(Inst, N);
578     } else {
579       addImmOperands(Inst, N, false);
580     }
581   }
582 
583   void addRegOrImmWithFPInputModsOperands(MCInst &Inst, unsigned N) const {
584     assert(!hasIntModifiers());
585     addRegOrImmWithInputModsOperands(Inst, N);
586   }
587 
588   void addRegOrImmWithIntInputModsOperands(MCInst &Inst, unsigned N) const {
589     assert(!hasFPModifiers());
590     addRegOrImmWithInputModsOperands(Inst, N);
591   }
592 
593   void addRegWithInputModsOperands(MCInst &Inst, unsigned N) const {
594     Modifiers Mods = getModifiers();
595     Inst.addOperand(MCOperand::createImm(Mods.getModifiersOperand()));
596     assert(isRegKind());
597     addRegOperands(Inst, N);
598   }
599 
600   void addRegWithFPInputModsOperands(MCInst &Inst, unsigned N) const {
601     assert(!hasIntModifiers());
602     addRegWithInputModsOperands(Inst, N);
603   }
604 
605   void addRegWithIntInputModsOperands(MCInst &Inst, unsigned N) const {
606     assert(!hasFPModifiers());
607     addRegWithInputModsOperands(Inst, N);
608   }
609 
610   void addSoppBrTargetOperands(MCInst &Inst, unsigned N) const {
611     if (isImm())
612       addImmOperands(Inst, N);
613     else {
614       assert(isExpr());
615       Inst.addOperand(MCOperand::createExpr(Expr));
616     }
617   }
618 
619   static void printImmTy(raw_ostream& OS, ImmTy Type) {
620     switch (Type) {
621     case ImmTyNone: OS << "None"; break;
622     case ImmTyGDS: OS << "GDS"; break;
623     case ImmTyOffen: OS << "Offen"; break;
624     case ImmTyIdxen: OS << "Idxen"; break;
625     case ImmTyAddr64: OS << "Addr64"; break;
626     case ImmTyOffset: OS << "Offset"; break;
627     case ImmTyOffset0: OS << "Offset0"; break;
628     case ImmTyOffset1: OS << "Offset1"; break;
629     case ImmTyGLC: OS << "GLC"; break;
630     case ImmTySLC: OS << "SLC"; break;
631     case ImmTyTFE: OS << "TFE"; break;
632     case ImmTyClampSI: OS << "ClampSI"; break;
633     case ImmTyOModSI: OS << "OModSI"; break;
634     case ImmTyDppCtrl: OS << "DppCtrl"; break;
635     case ImmTyDppRowMask: OS << "DppRowMask"; break;
636     case ImmTyDppBankMask: OS << "DppBankMask"; break;
637     case ImmTyDppBoundCtrl: OS << "DppBoundCtrl"; break;
638     case ImmTySdwaDstSel: OS << "SdwaDstSel"; break;
639     case ImmTySdwaSrc0Sel: OS << "SdwaSrc0Sel"; break;
640     case ImmTySdwaSrc1Sel: OS << "SdwaSrc1Sel"; break;
641     case ImmTySdwaDstUnused: OS << "SdwaDstUnused"; break;
642     case ImmTyDMask: OS << "DMask"; break;
643     case ImmTyUNorm: OS << "UNorm"; break;
644     case ImmTyDA: OS << "DA"; break;
645     case ImmTyR128: OS << "R128"; break;
646     case ImmTyLWE: OS << "LWE"; break;
647     case ImmTyOff: OS << "Off"; break;
648     case ImmTyExpTgt: OS << "ExpTgt"; break;
649     case ImmTyExpCompr: OS << "ExpCompr"; break;
650     case ImmTyExpVM: OS << "ExpVM"; break;
651     case ImmTyHwreg: OS << "Hwreg"; break;
652     case ImmTySendMsg: OS << "SendMsg"; break;
653     case ImmTyInterpSlot: OS << "InterpSlot"; break;
654     case ImmTyInterpAttr: OS << "InterpAttr"; break;
655     case ImmTyAttrChan: OS << "AttrChan"; break;
656     case ImmTyOpSel: OS << "OpSel"; break;
657     case ImmTyOpSelHi: OS << "OpSelHi"; break;
658     case ImmTyNegLo: OS << "NegLo"; break;
659     case ImmTyNegHi: OS << "NegHi"; break;
660     }
661   }
662 
663   void print(raw_ostream &OS) const override {
664     switch (Kind) {
665     case Register:
666       OS << "<register " << getReg() << " mods: " << Reg.Mods << '>';
667       break;
668     case Immediate:
669       OS << '<' << getImm();
670       if (getImmTy() != ImmTyNone) {
671         OS << " type: "; printImmTy(OS, getImmTy());
672       }
673       OS << " mods: " << Imm.Mods << '>';
674       break;
675     case Token:
676       OS << '\'' << getToken() << '\'';
677       break;
678     case Expression:
679       OS << "<expr " << *Expr << '>';
680       break;
681     }
682   }
683 
684   static AMDGPUOperand::Ptr CreateImm(const AMDGPUAsmParser *AsmParser,
685                                       int64_t Val, SMLoc Loc,
686                                       ImmTy Type = ImmTyNone,
687                                       bool IsFPImm = false) {
688     auto Op = llvm::make_unique<AMDGPUOperand>(Immediate, AsmParser);
689     Op->Imm.Val = Val;
690     Op->Imm.IsFPImm = IsFPImm;
691     Op->Imm.Type = Type;
692     Op->Imm.Mods = Modifiers();
693     Op->StartLoc = Loc;
694     Op->EndLoc = Loc;
695     return Op;
696   }
697 
698   static AMDGPUOperand::Ptr CreateToken(const AMDGPUAsmParser *AsmParser,
699                                         StringRef Str, SMLoc Loc,
700                                         bool HasExplicitEncodingSize = true) {
701     auto Res = llvm::make_unique<AMDGPUOperand>(Token, AsmParser);
702     Res->Tok.Data = Str.data();
703     Res->Tok.Length = Str.size();
704     Res->StartLoc = Loc;
705     Res->EndLoc = Loc;
706     return Res;
707   }
708 
709   static AMDGPUOperand::Ptr CreateReg(const AMDGPUAsmParser *AsmParser,
710                                       unsigned RegNo, SMLoc S,
711                                       SMLoc E,
712                                       bool ForceVOP3) {
713     auto Op = llvm::make_unique<AMDGPUOperand>(Register, AsmParser);
714     Op->Reg.RegNo = RegNo;
715     Op->Reg.Mods = Modifiers();
716     Op->Reg.IsForcedVOP3 = ForceVOP3;
717     Op->StartLoc = S;
718     Op->EndLoc = E;
719     return Op;
720   }
721 
722   static AMDGPUOperand::Ptr CreateExpr(const AMDGPUAsmParser *AsmParser,
723                                        const class MCExpr *Expr, SMLoc S) {
724     auto Op = llvm::make_unique<AMDGPUOperand>(Expression, AsmParser);
725     Op->Expr = Expr;
726     Op->StartLoc = S;
727     Op->EndLoc = S;
728     return Op;
729   }
730 };
731 
732 raw_ostream &operator <<(raw_ostream &OS, AMDGPUOperand::Modifiers Mods) {
733   OS << "abs:" << Mods.Abs << " neg: " << Mods.Neg << " sext:" << Mods.Sext;
734   return OS;
735 }
736 
737 //===----------------------------------------------------------------------===//
738 // AsmParser
739 //===----------------------------------------------------------------------===//
740 
741 // Holds info related to the current kernel, e.g. count of SGPRs used.
742 // Kernel scope begins at .amdgpu_hsa_kernel directive, ends at next
743 // .amdgpu_hsa_kernel or at EOF.
744 class KernelScopeInfo {
745   int SgprIndexUnusedMin = -1;
746   int VgprIndexUnusedMin = -1;
747   MCContext *Ctx = nullptr;
748 
749   void usesSgprAt(int i) {
750     if (i >= SgprIndexUnusedMin) {
751       SgprIndexUnusedMin = ++i;
752       if (Ctx) {
753         MCSymbol * const Sym = Ctx->getOrCreateSymbol(Twine(".kernel.sgpr_count"));
754         Sym->setVariableValue(MCConstantExpr::create(SgprIndexUnusedMin, *Ctx));
755       }
756     }
757   }
758 
759   void usesVgprAt(int i) {
760     if (i >= VgprIndexUnusedMin) {
761       VgprIndexUnusedMin = ++i;
762       if (Ctx) {
763         MCSymbol * const Sym = Ctx->getOrCreateSymbol(Twine(".kernel.vgpr_count"));
764         Sym->setVariableValue(MCConstantExpr::create(VgprIndexUnusedMin, *Ctx));
765       }
766     }
767   }
768 
769 public:
770   KernelScopeInfo() = default;
771 
772   void initialize(MCContext &Context) {
773     Ctx = &Context;
774     usesSgprAt(SgprIndexUnusedMin = -1);
775     usesVgprAt(VgprIndexUnusedMin = -1);
776   }
777 
778   void usesRegister(RegisterKind RegKind, unsigned DwordRegIndex, unsigned RegWidth) {
779     switch (RegKind) {
780       case IS_SGPR: usesSgprAt(DwordRegIndex + RegWidth - 1); break;
781       case IS_VGPR: usesVgprAt(DwordRegIndex + RegWidth - 1); break;
782       default: break;
783     }
784   }
785 };
786 
787 class AMDGPUAsmParser : public MCTargetAsmParser {
788   const MCInstrInfo &MII;
789   MCAsmParser &Parser;
790 
791   unsigned ForcedEncodingSize = 0;
792   bool ForcedDPP = false;
793   bool ForcedSDWA = false;
794   KernelScopeInfo KernelScope;
795 
796   /// @name Auto-generated Match Functions
797   /// {
798 
799 #define GET_ASSEMBLER_HEADER
800 #include "AMDGPUGenAsmMatcher.inc"
801 
802   /// }
803 
804 private:
805   bool ParseAsAbsoluteExpression(uint32_t &Ret);
806   bool ParseDirectiveMajorMinor(uint32_t &Major, uint32_t &Minor);
807   bool ParseDirectiveHSACodeObjectVersion();
808   bool ParseDirectiveHSACodeObjectISA();
809   bool ParseDirectiveCodeObjectMetadata();
810   bool ParseAMDKernelCodeTValue(StringRef ID, amd_kernel_code_t &Header);
811   bool ParseDirectiveAMDKernelCodeT();
812   bool ParseSectionDirectiveHSAText();
813   bool subtargetHasRegister(const MCRegisterInfo &MRI, unsigned RegNo) const;
814   bool ParseDirectiveAMDGPUHsaKernel();
815   bool ParseDirectiveAMDGPUHsaModuleGlobal();
816   bool ParseDirectiveAMDGPUHsaProgramGlobal();
817   bool ParseSectionDirectiveHSADataGlobalAgent();
818   bool ParseSectionDirectiveHSADataGlobalProgram();
819   bool ParseSectionDirectiveHSARodataReadonlyAgent();
820   bool AddNextRegisterToList(unsigned& Reg, unsigned& RegWidth,
821                              RegisterKind RegKind, unsigned Reg1,
822                              unsigned RegNum);
823   bool ParseAMDGPURegister(RegisterKind& RegKind, unsigned& Reg,
824                            unsigned& RegNum, unsigned& RegWidth,
825                            unsigned *DwordRegIndex);
826   void cvtMubufImpl(MCInst &Inst, const OperandVector &Operands,
827                     bool IsAtomic, bool IsAtomicReturn);
828   void cvtDSImpl(MCInst &Inst, const OperandVector &Operands,
829                  bool IsGdsHardcoded);
830 
831 public:
832   enum AMDGPUMatchResultTy {
833     Match_PreferE32 = FIRST_TARGET_MATCH_RESULT_TY
834   };
835 
836   typedef std::map<AMDGPUOperand::ImmTy, unsigned> OptionalImmIndexMap;
837 
838   AMDGPUAsmParser(const MCSubtargetInfo &STI, MCAsmParser &_Parser,
839                const MCInstrInfo &MII,
840                const MCTargetOptions &Options)
841       : MCTargetAsmParser(Options, STI), MII(MII), Parser(_Parser) {
842     MCAsmParserExtension::Initialize(Parser);
843 
844     if (getFeatureBits().none()) {
845       // Set default features.
846       copySTI().ToggleFeature("SOUTHERN_ISLANDS");
847     }
848 
849     setAvailableFeatures(ComputeAvailableFeatures(getFeatureBits()));
850 
851     {
852       // TODO: make those pre-defined variables read-only.
853       // Currently there is none suitable machinery in the core llvm-mc for this.
854       // MCSymbol::isRedefinable is intended for another purpose, and
855       // AsmParser::parseDirectiveSet() cannot be specialized for specific target.
856       AMDGPU::IsaInfo::IsaVersion ISA =
857           AMDGPU::IsaInfo::getIsaVersion(getFeatureBits());
858       MCContext &Ctx = getContext();
859       MCSymbol *Sym =
860           Ctx.getOrCreateSymbol(Twine(".option.machine_version_major"));
861       Sym->setVariableValue(MCConstantExpr::create(ISA.Major, Ctx));
862       Sym = Ctx.getOrCreateSymbol(Twine(".option.machine_version_minor"));
863       Sym->setVariableValue(MCConstantExpr::create(ISA.Minor, Ctx));
864       Sym = Ctx.getOrCreateSymbol(Twine(".option.machine_version_stepping"));
865       Sym->setVariableValue(MCConstantExpr::create(ISA.Stepping, Ctx));
866     }
867     KernelScope.initialize(getContext());
868   }
869 
870   bool isSI() const {
871     return AMDGPU::isSI(getSTI());
872   }
873 
874   bool isCI() const {
875     return AMDGPU::isCI(getSTI());
876   }
877 
878   bool isVI() const {
879     return AMDGPU::isVI(getSTI());
880   }
881 
882   bool hasInv2PiInlineImm() const {
883     return getFeatureBits()[AMDGPU::FeatureInv2PiInlineImm];
884   }
885 
886   bool hasSGPR102_SGPR103() const {
887     return !isVI();
888   }
889 
890   AMDGPUTargetStreamer &getTargetStreamer() {
891     MCTargetStreamer &TS = *getParser().getStreamer().getTargetStreamer();
892     return static_cast<AMDGPUTargetStreamer &>(TS);
893   }
894 
895   const MCRegisterInfo *getMRI() const {
896     // We need this const_cast because for some reason getContext() is not const
897     // in MCAsmParser.
898     return const_cast<AMDGPUAsmParser*>(this)->getContext().getRegisterInfo();
899   }
900 
901   const MCInstrInfo *getMII() const {
902     return &MII;
903   }
904 
905   const FeatureBitset &getFeatureBits() const {
906     return getSTI().getFeatureBits();
907   }
908 
909   void setForcedEncodingSize(unsigned Size) { ForcedEncodingSize = Size; }
910   void setForcedDPP(bool ForceDPP_) { ForcedDPP = ForceDPP_; }
911   void setForcedSDWA(bool ForceSDWA_) { ForcedSDWA = ForceSDWA_; }
912 
913   unsigned getForcedEncodingSize() const { return ForcedEncodingSize; }
914   bool isForcedVOP3() const { return ForcedEncodingSize == 64; }
915   bool isForcedDPP() const { return ForcedDPP; }
916   bool isForcedSDWA() const { return ForcedSDWA; }
917   ArrayRef<unsigned> getMatchedVariants() const;
918 
919   std::unique_ptr<AMDGPUOperand> parseRegister();
920   bool ParseRegister(unsigned &RegNo, SMLoc &StartLoc, SMLoc &EndLoc) override;
921   unsigned checkTargetMatchPredicate(MCInst &Inst) override;
922   unsigned validateTargetOperandClass(MCParsedAsmOperand &Op,
923                                       unsigned Kind) override;
924   bool MatchAndEmitInstruction(SMLoc IDLoc, unsigned &Opcode,
925                                OperandVector &Operands, MCStreamer &Out,
926                                uint64_t &ErrorInfo,
927                                bool MatchingInlineAsm) override;
928   bool ParseDirective(AsmToken DirectiveID) override;
929   OperandMatchResultTy parseOperand(OperandVector &Operands, StringRef Mnemonic);
930   StringRef parseMnemonicSuffix(StringRef Name);
931   bool ParseInstruction(ParseInstructionInfo &Info, StringRef Name,
932                         SMLoc NameLoc, OperandVector &Operands) override;
933   //bool ProcessInstruction(MCInst &Inst);
934 
935   OperandMatchResultTy parseIntWithPrefix(const char *Prefix, int64_t &Int);
936 
937   OperandMatchResultTy
938   parseIntWithPrefix(const char *Prefix, OperandVector &Operands,
939                      AMDGPUOperand::ImmTy ImmTy = AMDGPUOperand::ImmTyNone,
940                      bool (*ConvertResult)(int64_t &) = nullptr);
941 
942   OperandMatchResultTy parseOperandArrayWithPrefix(
943     const char *Prefix,
944     OperandVector &Operands,
945     AMDGPUOperand::ImmTy ImmTy = AMDGPUOperand::ImmTyNone,
946     bool (*ConvertResult)(int64_t&) = nullptr);
947 
948   OperandMatchResultTy
949   parseNamedBit(const char *Name, OperandVector &Operands,
950                 AMDGPUOperand::ImmTy ImmTy = AMDGPUOperand::ImmTyNone);
951   OperandMatchResultTy parseStringWithPrefix(StringRef Prefix,
952                                              StringRef &Value);
953 
954   bool parseAbsoluteExpr(int64_t &Val, bool AbsMod = false);
955   OperandMatchResultTy parseImm(OperandVector &Operands, bool AbsMod = false);
956   OperandMatchResultTy parseReg(OperandVector &Operands);
957   OperandMatchResultTy parseRegOrImm(OperandVector &Operands, bool AbsMod = false);
958   OperandMatchResultTy parseRegOrImmWithFPInputMods(OperandVector &Operands, bool AllowImm = true);
959   OperandMatchResultTy parseRegOrImmWithIntInputMods(OperandVector &Operands, bool AllowImm = true);
960   OperandMatchResultTy parseRegWithFPInputMods(OperandVector &Operands);
961   OperandMatchResultTy parseRegWithIntInputMods(OperandVector &Operands);
962   OperandMatchResultTy parseVReg32OrOff(OperandVector &Operands);
963 
964   void cvtDSOffset01(MCInst &Inst, const OperandVector &Operands);
965   void cvtDS(MCInst &Inst, const OperandVector &Operands) { cvtDSImpl(Inst, Operands, false); }
966   void cvtDSGds(MCInst &Inst, const OperandVector &Operands) { cvtDSImpl(Inst, Operands, true); }
967   void cvtExp(MCInst &Inst, const OperandVector &Operands);
968 
969   bool parseCnt(int64_t &IntVal);
970   OperandMatchResultTy parseSWaitCntOps(OperandVector &Operands);
971   OperandMatchResultTy parseHwreg(OperandVector &Operands);
972 
973 private:
974   struct OperandInfoTy {
975     int64_t Id;
976     bool IsSymbolic;
977     OperandInfoTy(int64_t Id_) : Id(Id_), IsSymbolic(false) { }
978   };
979 
980   bool parseSendMsgConstruct(OperandInfoTy &Msg, OperandInfoTy &Operation, int64_t &StreamId);
981   bool parseHwregConstruct(OperandInfoTy &HwReg, int64_t &Offset, int64_t &Width);
982 
983   void errorExpTgt();
984   OperandMatchResultTy parseExpTgtImpl(StringRef Str, uint8_t &Val);
985 
986   bool validateOperandLimitations(const MCInst &Inst);
987   bool usesConstantBus(const MCInst &Inst, unsigned OpIdx);
988   bool isInlineConstant(const MCInst &Inst, unsigned OpIdx) const;
989   unsigned findImplicitSGPRReadInVOP(const MCInst &Inst) const;
990   bool isSGPR(unsigned Reg);
991 
992 public:
993   OperandMatchResultTy parseOptionalOperand(OperandVector &Operands);
994 
995   OperandMatchResultTy parseExpTgt(OperandVector &Operands);
996   OperandMatchResultTy parseSendMsgOp(OperandVector &Operands);
997   OperandMatchResultTy parseInterpSlot(OperandVector &Operands);
998   OperandMatchResultTy parseInterpAttr(OperandVector &Operands);
999   OperandMatchResultTy parseSOppBrTarget(OperandVector &Operands);
1000 
1001   void cvtMubuf(MCInst &Inst, const OperandVector &Operands) { cvtMubufImpl(Inst, Operands, false, false); }
1002   void cvtMubufAtomic(MCInst &Inst, const OperandVector &Operands) { cvtMubufImpl(Inst, Operands, true, false); }
1003   void cvtMubufAtomicReturn(MCInst &Inst, const OperandVector &Operands) { cvtMubufImpl(Inst, Operands, true, true); }
1004   AMDGPUOperand::Ptr defaultGLC() const;
1005   AMDGPUOperand::Ptr defaultSLC() const;
1006   AMDGPUOperand::Ptr defaultTFE() const;
1007 
1008   AMDGPUOperand::Ptr defaultDMask() const;
1009   AMDGPUOperand::Ptr defaultUNorm() const;
1010   AMDGPUOperand::Ptr defaultDA() const;
1011   AMDGPUOperand::Ptr defaultR128() const;
1012   AMDGPUOperand::Ptr defaultLWE() const;
1013   AMDGPUOperand::Ptr defaultSMRDOffset8() const;
1014   AMDGPUOperand::Ptr defaultSMRDOffset20() const;
1015   AMDGPUOperand::Ptr defaultSMRDLiteralOffset() const;
1016 
1017   OperandMatchResultTy parseOModOperand(OperandVector &Operands);
1018 
1019   void cvtId(MCInst &Inst, const OperandVector &Operands);
1020   void cvtVOP3_2_mod(MCInst &Inst, const OperandVector &Operands);
1021 
1022   void cvtVOP3Impl(MCInst &Inst,
1023                    const OperandVector &Operands,
1024                    OptionalImmIndexMap &OptionalIdx);
1025   void cvtVOP3(MCInst &Inst, const OperandVector &Operands);
1026   void cvtVOP3P(MCInst &Inst, const OperandVector &Operands);
1027 
1028   void cvtMIMG(MCInst &Inst, const OperandVector &Operands);
1029   void cvtMIMGAtomic(MCInst &Inst, const OperandVector &Operands);
1030 
1031   OperandMatchResultTy parseDPPCtrl(OperandVector &Operands);
1032   AMDGPUOperand::Ptr defaultRowMask() const;
1033   AMDGPUOperand::Ptr defaultBankMask() const;
1034   AMDGPUOperand::Ptr defaultBoundCtrl() const;
1035   void cvtDPP(MCInst &Inst, const OperandVector &Operands);
1036 
1037   OperandMatchResultTy parseSDWASel(OperandVector &Operands, StringRef Prefix,
1038                                     AMDGPUOperand::ImmTy Type);
1039   OperandMatchResultTy parseSDWADstUnused(OperandVector &Operands);
1040   void cvtSdwaVOP1(MCInst &Inst, const OperandVector &Operands);
1041   void cvtSdwaVOP2(MCInst &Inst, const OperandVector &Operands);
1042   void cvtSdwaVOPC(MCInst &Inst, const OperandVector &Operands);
1043   void cvtSDWA(MCInst &Inst, const OperandVector &Operands,
1044                uint64_t BasicInstType);
1045 };
1046 
1047 struct OptionalOperand {
1048   const char *Name;
1049   AMDGPUOperand::ImmTy Type;
1050   bool IsBit;
1051   bool (*ConvertResult)(int64_t&);
1052 };
1053 
1054 } // end anonymous namespace
1055 
1056 // May be called with integer type with equivalent bitwidth.
1057 static const fltSemantics *getFltSemantics(unsigned Size) {
1058   switch (Size) {
1059   case 4:
1060     return &APFloat::IEEEsingle();
1061   case 8:
1062     return &APFloat::IEEEdouble();
1063   case 2:
1064     return &APFloat::IEEEhalf();
1065   default:
1066     llvm_unreachable("unsupported fp type");
1067   }
1068 }
1069 
1070 static const fltSemantics *getFltSemantics(MVT VT) {
1071   return getFltSemantics(VT.getSizeInBits() / 8);
1072 }
1073 
1074 static const fltSemantics *getOpFltSemantics(uint8_t OperandType) {
1075   switch (OperandType) {
1076   case AMDGPU::OPERAND_REG_IMM_INT32:
1077   case AMDGPU::OPERAND_REG_IMM_FP32:
1078   case AMDGPU::OPERAND_REG_INLINE_C_INT32:
1079   case AMDGPU::OPERAND_REG_INLINE_C_FP32:
1080     return &APFloat::IEEEsingle();
1081   case AMDGPU::OPERAND_REG_IMM_INT64:
1082   case AMDGPU::OPERAND_REG_IMM_FP64:
1083   case AMDGPU::OPERAND_REG_INLINE_C_INT64:
1084   case AMDGPU::OPERAND_REG_INLINE_C_FP64:
1085     return &APFloat::IEEEdouble();
1086   case AMDGPU::OPERAND_REG_IMM_INT16:
1087   case AMDGPU::OPERAND_REG_IMM_FP16:
1088   case AMDGPU::OPERAND_REG_INLINE_C_INT16:
1089   case AMDGPU::OPERAND_REG_INLINE_C_FP16:
1090   case AMDGPU::OPERAND_REG_INLINE_C_V2INT16:
1091   case AMDGPU::OPERAND_REG_INLINE_C_V2FP16:
1092     return &APFloat::IEEEhalf();
1093   default:
1094     llvm_unreachable("unsupported fp type");
1095   }
1096 }
1097 
1098 //===----------------------------------------------------------------------===//
1099 // Operand
1100 //===----------------------------------------------------------------------===//
1101 
1102 static bool canLosslesslyConvertToFPType(APFloat &FPLiteral, MVT VT) {
1103   bool Lost;
1104 
1105   // Convert literal to single precision
1106   APFloat::opStatus Status = FPLiteral.convert(*getFltSemantics(VT),
1107                                                APFloat::rmNearestTiesToEven,
1108                                                &Lost);
1109   // We allow precision lost but not overflow or underflow
1110   if (Status != APFloat::opOK &&
1111       Lost &&
1112       ((Status & APFloat::opOverflow)  != 0 ||
1113        (Status & APFloat::opUnderflow) != 0)) {
1114     return false;
1115   }
1116 
1117   return true;
1118 }
1119 
1120 bool AMDGPUOperand::isInlinableImm(MVT type) const {
1121   if (!isImmTy(ImmTyNone)) {
1122     // Only plain immediates are inlinable (e.g. "clamp" attribute is not)
1123     return false;
1124   }
1125   // TODO: We should avoid using host float here. It would be better to
1126   // check the float bit values which is what a few other places do.
1127   // We've had bot failures before due to weird NaN support on mips hosts.
1128 
1129   APInt Literal(64, Imm.Val);
1130 
1131   if (Imm.IsFPImm) { // We got fp literal token
1132     if (type == MVT::f64 || type == MVT::i64) { // Expected 64-bit operand
1133       return AMDGPU::isInlinableLiteral64(Imm.Val,
1134                                           AsmParser->hasInv2PiInlineImm());
1135     }
1136 
1137     APFloat FPLiteral(APFloat::IEEEdouble(), APInt(64, Imm.Val));
1138     if (!canLosslesslyConvertToFPType(FPLiteral, type))
1139       return false;
1140 
1141     if (type.getScalarSizeInBits() == 16) {
1142       return AMDGPU::isInlinableLiteral16(
1143         static_cast<int16_t>(FPLiteral.bitcastToAPInt().getZExtValue()),
1144         AsmParser->hasInv2PiInlineImm());
1145     }
1146 
1147     // Check if single precision literal is inlinable
1148     return AMDGPU::isInlinableLiteral32(
1149       static_cast<int32_t>(FPLiteral.bitcastToAPInt().getZExtValue()),
1150       AsmParser->hasInv2PiInlineImm());
1151   }
1152 
1153   // We got int literal token.
1154   if (type == MVT::f64 || type == MVT::i64) { // Expected 64-bit operand
1155     return AMDGPU::isInlinableLiteral64(Imm.Val,
1156                                         AsmParser->hasInv2PiInlineImm());
1157   }
1158 
1159   if (type.getScalarSizeInBits() == 16) {
1160     return AMDGPU::isInlinableLiteral16(
1161       static_cast<int16_t>(Literal.getLoBits(16).getSExtValue()),
1162       AsmParser->hasInv2PiInlineImm());
1163   }
1164 
1165   return AMDGPU::isInlinableLiteral32(
1166     static_cast<int32_t>(Literal.getLoBits(32).getZExtValue()),
1167     AsmParser->hasInv2PiInlineImm());
1168 }
1169 
1170 bool AMDGPUOperand::isLiteralImm(MVT type) const {
1171   // Check that this imediate can be added as literal
1172   if (!isImmTy(ImmTyNone)) {
1173     return false;
1174   }
1175 
1176   if (!Imm.IsFPImm) {
1177     // We got int literal token.
1178 
1179     if (type == MVT::f64 && hasFPModifiers()) {
1180       // Cannot apply fp modifiers to int literals preserving the same semantics
1181       // for VOP1/2/C and VOP3 because of integer truncation. To avoid ambiguity,
1182       // disable these cases.
1183       return false;
1184     }
1185 
1186     unsigned Size = type.getSizeInBits();
1187     if (Size == 64)
1188       Size = 32;
1189 
1190     // FIXME: 64-bit operands can zero extend, sign extend, or pad zeroes for FP
1191     // types.
1192     return isUIntN(Size, Imm.Val) || isIntN(Size, Imm.Val);
1193   }
1194 
1195   // We got fp literal token
1196   if (type == MVT::f64) { // Expected 64-bit fp operand
1197     // We would set low 64-bits of literal to zeroes but we accept this literals
1198     return true;
1199   }
1200 
1201   if (type == MVT::i64) { // Expected 64-bit int operand
1202     // We don't allow fp literals in 64-bit integer instructions. It is
1203     // unclear how we should encode them.
1204     return false;
1205   }
1206 
1207   APFloat FPLiteral(APFloat::IEEEdouble(), APInt(64, Imm.Val));
1208   return canLosslesslyConvertToFPType(FPLiteral, type);
1209 }
1210 
1211 bool AMDGPUOperand::isRegClass(unsigned RCID) const {
1212   return isRegKind() && AsmParser->getMRI()->getRegClass(RCID).contains(getReg());
1213 }
1214 
1215 uint64_t AMDGPUOperand::applyInputFPModifiers(uint64_t Val, unsigned Size) const
1216 {
1217   assert(isImmTy(ImmTyNone) && Imm.Mods.hasFPModifiers());
1218   assert(Size == 2 || Size == 4 || Size == 8);
1219 
1220   const uint64_t FpSignMask = (1ULL << (Size * 8 - 1));
1221 
1222   if (Imm.Mods.Abs) {
1223     Val &= ~FpSignMask;
1224   }
1225   if (Imm.Mods.Neg) {
1226     Val ^= FpSignMask;
1227   }
1228 
1229   return Val;
1230 }
1231 
1232 void AMDGPUOperand::addImmOperands(MCInst &Inst, unsigned N, bool ApplyModifiers) const {
1233 
1234   if (AMDGPU::isSISrcOperand(AsmParser->getMII()->get(Inst.getOpcode()),
1235                              Inst.getNumOperands())) {
1236     addLiteralImmOperand(Inst, Imm.Val,
1237                          ApplyModifiers &
1238                          isImmTy(ImmTyNone) && Imm.Mods.hasFPModifiers());
1239   } else {
1240     assert(!isImmTy(ImmTyNone) || !hasModifiers());
1241     Inst.addOperand(MCOperand::createImm(Imm.Val));
1242   }
1243 }
1244 
1245 void AMDGPUOperand::addLiteralImmOperand(MCInst &Inst, int64_t Val, bool ApplyModifiers) const {
1246   const auto& InstDesc = AsmParser->getMII()->get(Inst.getOpcode());
1247   auto OpNum = Inst.getNumOperands();
1248   // Check that this operand accepts literals
1249   assert(AMDGPU::isSISrcOperand(InstDesc, OpNum));
1250 
1251   if (ApplyModifiers) {
1252     assert(AMDGPU::isSISrcFPOperand(InstDesc, OpNum));
1253     const unsigned Size = Imm.IsFPImm ? sizeof(double) : getOperandSize(InstDesc, OpNum);
1254     Val = applyInputFPModifiers(Val, Size);
1255   }
1256 
1257   APInt Literal(64, Val);
1258   uint8_t OpTy = InstDesc.OpInfo[OpNum].OperandType;
1259 
1260   if (Imm.IsFPImm) { // We got fp literal token
1261     switch (OpTy) {
1262     case AMDGPU::OPERAND_REG_IMM_INT64:
1263     case AMDGPU::OPERAND_REG_IMM_FP64:
1264     case AMDGPU::OPERAND_REG_INLINE_C_INT64:
1265     case AMDGPU::OPERAND_REG_INLINE_C_FP64: {
1266       if (AMDGPU::isInlinableLiteral64(Literal.getZExtValue(),
1267                                        AsmParser->hasInv2PiInlineImm())) {
1268         Inst.addOperand(MCOperand::createImm(Literal.getZExtValue()));
1269         return;
1270       }
1271 
1272       // Non-inlineable
1273       if (AMDGPU::isSISrcFPOperand(InstDesc, OpNum)) { // Expected 64-bit fp operand
1274         // For fp operands we check if low 32 bits are zeros
1275         if (Literal.getLoBits(32) != 0) {
1276           const_cast<AMDGPUAsmParser *>(AsmParser)->Warning(Inst.getLoc(),
1277           "Can't encode literal as exact 64-bit floating-point operand. "
1278           "Low 32-bits will be set to zero");
1279         }
1280 
1281         Inst.addOperand(MCOperand::createImm(Literal.lshr(32).getZExtValue()));
1282         return;
1283       }
1284 
1285       // We don't allow fp literals in 64-bit integer instructions. It is
1286       // unclear how we should encode them. This case should be checked earlier
1287       // in predicate methods (isLiteralImm())
1288       llvm_unreachable("fp literal in 64-bit integer instruction.");
1289     }
1290     case AMDGPU::OPERAND_REG_IMM_INT32:
1291     case AMDGPU::OPERAND_REG_IMM_FP32:
1292     case AMDGPU::OPERAND_REG_INLINE_C_INT32:
1293     case AMDGPU::OPERAND_REG_INLINE_C_FP32:
1294     case AMDGPU::OPERAND_REG_IMM_INT16:
1295     case AMDGPU::OPERAND_REG_IMM_FP16:
1296     case AMDGPU::OPERAND_REG_INLINE_C_INT16:
1297     case AMDGPU::OPERAND_REG_INLINE_C_FP16:
1298     case AMDGPU::OPERAND_REG_INLINE_C_V2INT16:
1299     case AMDGPU::OPERAND_REG_INLINE_C_V2FP16: {
1300       bool lost;
1301       APFloat FPLiteral(APFloat::IEEEdouble(), Literal);
1302       // Convert literal to single precision
1303       FPLiteral.convert(*getOpFltSemantics(OpTy),
1304                         APFloat::rmNearestTiesToEven, &lost);
1305       // We allow precision lost but not overflow or underflow. This should be
1306       // checked earlier in isLiteralImm()
1307 
1308       uint64_t ImmVal = FPLiteral.bitcastToAPInt().getZExtValue();
1309       if (OpTy == AMDGPU::OPERAND_REG_INLINE_C_V2INT16 ||
1310           OpTy == AMDGPU::OPERAND_REG_INLINE_C_V2FP16) {
1311         ImmVal |= (ImmVal << 16);
1312       }
1313 
1314       Inst.addOperand(MCOperand::createImm(ImmVal));
1315       return;
1316     }
1317     default:
1318       llvm_unreachable("invalid operand size");
1319     }
1320 
1321     return;
1322   }
1323 
1324    // We got int literal token.
1325   // Only sign extend inline immediates.
1326   // FIXME: No errors on truncation
1327   switch (OpTy) {
1328   case AMDGPU::OPERAND_REG_IMM_INT32:
1329   case AMDGPU::OPERAND_REG_IMM_FP32:
1330   case AMDGPU::OPERAND_REG_INLINE_C_INT32:
1331   case AMDGPU::OPERAND_REG_INLINE_C_FP32: {
1332     if (isInt<32>(Val) &&
1333         AMDGPU::isInlinableLiteral32(static_cast<int32_t>(Val),
1334                                      AsmParser->hasInv2PiInlineImm())) {
1335       Inst.addOperand(MCOperand::createImm(Val));
1336       return;
1337     }
1338 
1339     Inst.addOperand(MCOperand::createImm(Val & 0xffffffff));
1340     return;
1341   }
1342   case AMDGPU::OPERAND_REG_IMM_INT64:
1343   case AMDGPU::OPERAND_REG_IMM_FP64:
1344   case AMDGPU::OPERAND_REG_INLINE_C_INT64:
1345   case AMDGPU::OPERAND_REG_INLINE_C_FP64: {
1346     if (AMDGPU::isInlinableLiteral64(Val, AsmParser->hasInv2PiInlineImm())) {
1347       Inst.addOperand(MCOperand::createImm(Val));
1348       return;
1349     }
1350 
1351     Inst.addOperand(MCOperand::createImm(Lo_32(Val)));
1352     return;
1353   }
1354   case AMDGPU::OPERAND_REG_IMM_INT16:
1355   case AMDGPU::OPERAND_REG_IMM_FP16:
1356   case AMDGPU::OPERAND_REG_INLINE_C_INT16:
1357   case AMDGPU::OPERAND_REG_INLINE_C_FP16: {
1358     if (isInt<16>(Val) &&
1359         AMDGPU::isInlinableLiteral16(static_cast<int16_t>(Val),
1360                                      AsmParser->hasInv2PiInlineImm())) {
1361       Inst.addOperand(MCOperand::createImm(Val));
1362       return;
1363     }
1364 
1365     Inst.addOperand(MCOperand::createImm(Val & 0xffff));
1366     return;
1367   }
1368   case AMDGPU::OPERAND_REG_INLINE_C_V2INT16:
1369   case AMDGPU::OPERAND_REG_INLINE_C_V2FP16: {
1370     auto LiteralVal = static_cast<uint16_t>(Literal.getLoBits(16).getZExtValue());
1371     assert(AMDGPU::isInlinableLiteral16(LiteralVal,
1372                                         AsmParser->hasInv2PiInlineImm()));
1373 
1374     uint32_t ImmVal = static_cast<uint32_t>(LiteralVal) << 16 |
1375                       static_cast<uint32_t>(LiteralVal);
1376     Inst.addOperand(MCOperand::createImm(ImmVal));
1377     return;
1378   }
1379   default:
1380     llvm_unreachable("invalid operand size");
1381   }
1382 }
1383 
1384 template <unsigned Bitwidth>
1385 void AMDGPUOperand::addKImmFPOperands(MCInst &Inst, unsigned N) const {
1386   APInt Literal(64, Imm.Val);
1387 
1388   if (!Imm.IsFPImm) {
1389     // We got int literal token.
1390     Inst.addOperand(MCOperand::createImm(Literal.getLoBits(Bitwidth).getZExtValue()));
1391     return;
1392   }
1393 
1394   bool Lost;
1395   APFloat FPLiteral(APFloat::IEEEdouble(), Literal);
1396   FPLiteral.convert(*getFltSemantics(Bitwidth / 8),
1397                     APFloat::rmNearestTiesToEven, &Lost);
1398   Inst.addOperand(MCOperand::createImm(FPLiteral.bitcastToAPInt().getZExtValue()));
1399 }
1400 
1401 void AMDGPUOperand::addRegOperands(MCInst &Inst, unsigned N) const {
1402   Inst.addOperand(MCOperand::createReg(AMDGPU::getMCReg(getReg(), AsmParser->getSTI())));
1403 }
1404 
1405 //===----------------------------------------------------------------------===//
1406 // AsmParser
1407 //===----------------------------------------------------------------------===//
1408 
1409 static int getRegClass(RegisterKind Is, unsigned RegWidth) {
1410   if (Is == IS_VGPR) {
1411     switch (RegWidth) {
1412       default: return -1;
1413       case 1: return AMDGPU::VGPR_32RegClassID;
1414       case 2: return AMDGPU::VReg_64RegClassID;
1415       case 3: return AMDGPU::VReg_96RegClassID;
1416       case 4: return AMDGPU::VReg_128RegClassID;
1417       case 8: return AMDGPU::VReg_256RegClassID;
1418       case 16: return AMDGPU::VReg_512RegClassID;
1419     }
1420   } else if (Is == IS_TTMP) {
1421     switch (RegWidth) {
1422       default: return -1;
1423       case 1: return AMDGPU::TTMP_32RegClassID;
1424       case 2: return AMDGPU::TTMP_64RegClassID;
1425       case 4: return AMDGPU::TTMP_128RegClassID;
1426     }
1427   } else if (Is == IS_SGPR) {
1428     switch (RegWidth) {
1429       default: return -1;
1430       case 1: return AMDGPU::SGPR_32RegClassID;
1431       case 2: return AMDGPU::SGPR_64RegClassID;
1432       case 4: return AMDGPU::SGPR_128RegClassID;
1433       case 8: return AMDGPU::SReg_256RegClassID;
1434       case 16: return AMDGPU::SReg_512RegClassID;
1435     }
1436   }
1437   return -1;
1438 }
1439 
1440 static unsigned getSpecialRegForName(StringRef RegName) {
1441   return StringSwitch<unsigned>(RegName)
1442     .Case("exec", AMDGPU::EXEC)
1443     .Case("vcc", AMDGPU::VCC)
1444     .Case("flat_scratch", AMDGPU::FLAT_SCR)
1445     .Case("m0", AMDGPU::M0)
1446     .Case("scc", AMDGPU::SCC)
1447     .Case("tba", AMDGPU::TBA)
1448     .Case("tma", AMDGPU::TMA)
1449     .Case("flat_scratch_lo", AMDGPU::FLAT_SCR_LO)
1450     .Case("flat_scratch_hi", AMDGPU::FLAT_SCR_HI)
1451     .Case("vcc_lo", AMDGPU::VCC_LO)
1452     .Case("vcc_hi", AMDGPU::VCC_HI)
1453     .Case("exec_lo", AMDGPU::EXEC_LO)
1454     .Case("exec_hi", AMDGPU::EXEC_HI)
1455     .Case("tma_lo", AMDGPU::TMA_LO)
1456     .Case("tma_hi", AMDGPU::TMA_HI)
1457     .Case("tba_lo", AMDGPU::TBA_LO)
1458     .Case("tba_hi", AMDGPU::TBA_HI)
1459     .Default(0);
1460 }
1461 
1462 bool AMDGPUAsmParser::ParseRegister(unsigned &RegNo, SMLoc &StartLoc,
1463                                     SMLoc &EndLoc) {
1464   auto R = parseRegister();
1465   if (!R) return true;
1466   assert(R->isReg());
1467   RegNo = R->getReg();
1468   StartLoc = R->getStartLoc();
1469   EndLoc = R->getEndLoc();
1470   return false;
1471 }
1472 
1473 bool AMDGPUAsmParser::AddNextRegisterToList(unsigned &Reg, unsigned &RegWidth,
1474                                             RegisterKind RegKind, unsigned Reg1,
1475                                             unsigned RegNum) {
1476   switch (RegKind) {
1477   case IS_SPECIAL:
1478     if (Reg == AMDGPU::EXEC_LO && Reg1 == AMDGPU::EXEC_HI) {
1479       Reg = AMDGPU::EXEC;
1480       RegWidth = 2;
1481       return true;
1482     }
1483     if (Reg == AMDGPU::FLAT_SCR_LO && Reg1 == AMDGPU::FLAT_SCR_HI) {
1484       Reg = AMDGPU::FLAT_SCR;
1485       RegWidth = 2;
1486       return true;
1487     }
1488     if (Reg == AMDGPU::VCC_LO && Reg1 == AMDGPU::VCC_HI) {
1489       Reg = AMDGPU::VCC;
1490       RegWidth = 2;
1491       return true;
1492     }
1493     if (Reg == AMDGPU::TBA_LO && Reg1 == AMDGPU::TBA_HI) {
1494       Reg = AMDGPU::TBA;
1495       RegWidth = 2;
1496       return true;
1497     }
1498     if (Reg == AMDGPU::TMA_LO && Reg1 == AMDGPU::TMA_HI) {
1499       Reg = AMDGPU::TMA;
1500       RegWidth = 2;
1501       return true;
1502     }
1503     return false;
1504   case IS_VGPR:
1505   case IS_SGPR:
1506   case IS_TTMP:
1507     if (Reg1 != Reg + RegWidth) {
1508       return false;
1509     }
1510     RegWidth++;
1511     return true;
1512   default:
1513     llvm_unreachable("unexpected register kind");
1514   }
1515 }
1516 
1517 bool AMDGPUAsmParser::ParseAMDGPURegister(RegisterKind &RegKind, unsigned &Reg,
1518                                           unsigned &RegNum, unsigned &RegWidth,
1519                                           unsigned *DwordRegIndex) {
1520   if (DwordRegIndex) { *DwordRegIndex = 0; }
1521   const MCRegisterInfo *TRI = getContext().getRegisterInfo();
1522   if (getLexer().is(AsmToken::Identifier)) {
1523     StringRef RegName = Parser.getTok().getString();
1524     if ((Reg = getSpecialRegForName(RegName))) {
1525       Parser.Lex();
1526       RegKind = IS_SPECIAL;
1527     } else {
1528       unsigned RegNumIndex = 0;
1529       if (RegName[0] == 'v') {
1530         RegNumIndex = 1;
1531         RegKind = IS_VGPR;
1532       } else if (RegName[0] == 's') {
1533         RegNumIndex = 1;
1534         RegKind = IS_SGPR;
1535       } else if (RegName.startswith("ttmp")) {
1536         RegNumIndex = strlen("ttmp");
1537         RegKind = IS_TTMP;
1538       } else {
1539         return false;
1540       }
1541       if (RegName.size() > RegNumIndex) {
1542         // Single 32-bit register: vXX.
1543         if (RegName.substr(RegNumIndex).getAsInteger(10, RegNum))
1544           return false;
1545         Parser.Lex();
1546         RegWidth = 1;
1547       } else {
1548         // Range of registers: v[XX:YY]. ":YY" is optional.
1549         Parser.Lex();
1550         int64_t RegLo, RegHi;
1551         if (getLexer().isNot(AsmToken::LBrac))
1552           return false;
1553         Parser.Lex();
1554 
1555         if (getParser().parseAbsoluteExpression(RegLo))
1556           return false;
1557 
1558         const bool isRBrace = getLexer().is(AsmToken::RBrac);
1559         if (!isRBrace && getLexer().isNot(AsmToken::Colon))
1560           return false;
1561         Parser.Lex();
1562 
1563         if (isRBrace) {
1564           RegHi = RegLo;
1565         } else {
1566           if (getParser().parseAbsoluteExpression(RegHi))
1567             return false;
1568 
1569           if (getLexer().isNot(AsmToken::RBrac))
1570             return false;
1571           Parser.Lex();
1572         }
1573         RegNum = (unsigned) RegLo;
1574         RegWidth = (RegHi - RegLo) + 1;
1575       }
1576     }
1577   } else if (getLexer().is(AsmToken::LBrac)) {
1578     // List of consecutive registers: [s0,s1,s2,s3]
1579     Parser.Lex();
1580     if (!ParseAMDGPURegister(RegKind, Reg, RegNum, RegWidth, nullptr))
1581       return false;
1582     if (RegWidth != 1)
1583       return false;
1584     RegisterKind RegKind1;
1585     unsigned Reg1, RegNum1, RegWidth1;
1586     do {
1587       if (getLexer().is(AsmToken::Comma)) {
1588         Parser.Lex();
1589       } else if (getLexer().is(AsmToken::RBrac)) {
1590         Parser.Lex();
1591         break;
1592       } else if (ParseAMDGPURegister(RegKind1, Reg1, RegNum1, RegWidth1, nullptr)) {
1593         if (RegWidth1 != 1) {
1594           return false;
1595         }
1596         if (RegKind1 != RegKind) {
1597           return false;
1598         }
1599         if (!AddNextRegisterToList(Reg, RegWidth, RegKind1, Reg1, RegNum1)) {
1600           return false;
1601         }
1602       } else {
1603         return false;
1604       }
1605     } while (true);
1606   } else {
1607     return false;
1608   }
1609   switch (RegKind) {
1610   case IS_SPECIAL:
1611     RegNum = 0;
1612     RegWidth = 1;
1613     break;
1614   case IS_VGPR:
1615   case IS_SGPR:
1616   case IS_TTMP:
1617   {
1618     unsigned Size = 1;
1619     if (RegKind == IS_SGPR || RegKind == IS_TTMP) {
1620       // SGPR and TTMP registers must be aligned. Max required alignment is 4 dwords.
1621       Size = std::min(RegWidth, 4u);
1622     }
1623     if (RegNum % Size != 0)
1624       return false;
1625     if (DwordRegIndex) { *DwordRegIndex = RegNum; }
1626     RegNum = RegNum / Size;
1627     int RCID = getRegClass(RegKind, RegWidth);
1628     if (RCID == -1)
1629       return false;
1630     const MCRegisterClass RC = TRI->getRegClass(RCID);
1631     if (RegNum >= RC.getNumRegs())
1632       return false;
1633     Reg = RC.getRegister(RegNum);
1634     break;
1635   }
1636 
1637   default:
1638     llvm_unreachable("unexpected register kind");
1639   }
1640 
1641   if (!subtargetHasRegister(*TRI, Reg))
1642     return false;
1643   return true;
1644 }
1645 
1646 std::unique_ptr<AMDGPUOperand> AMDGPUAsmParser::parseRegister() {
1647   const auto &Tok = Parser.getTok();
1648   SMLoc StartLoc = Tok.getLoc();
1649   SMLoc EndLoc = Tok.getEndLoc();
1650   RegisterKind RegKind;
1651   unsigned Reg, RegNum, RegWidth, DwordRegIndex;
1652 
1653   if (!ParseAMDGPURegister(RegKind, Reg, RegNum, RegWidth, &DwordRegIndex)) {
1654     return nullptr;
1655   }
1656   KernelScope.usesRegister(RegKind, DwordRegIndex, RegWidth);
1657   return AMDGPUOperand::CreateReg(this, Reg, StartLoc, EndLoc, false);
1658 }
1659 
1660 bool
1661 AMDGPUAsmParser::parseAbsoluteExpr(int64_t &Val, bool AbsMod) {
1662   if (AbsMod && getLexer().peekTok().is(AsmToken::Pipe) &&
1663       (getLexer().getKind() == AsmToken::Integer ||
1664        getLexer().getKind() == AsmToken::Real)) {
1665 
1666     // This is a workaround for handling operands like these:
1667     //     |1.0|
1668     //     |-1|
1669     // This syntax is not compatible with syntax of standard
1670     // MC expressions (due to the trailing '|').
1671 
1672     SMLoc EndLoc;
1673     const MCExpr *Expr;
1674 
1675     if (getParser().parsePrimaryExpr(Expr, EndLoc)) {
1676       return true;
1677     }
1678 
1679     return !Expr->evaluateAsAbsolute(Val);
1680   }
1681 
1682   return getParser().parseAbsoluteExpression(Val);
1683 }
1684 
1685 OperandMatchResultTy
1686 AMDGPUAsmParser::parseImm(OperandVector &Operands, bool AbsMod) {
1687   // TODO: add syntactic sugar for 1/(2*PI)
1688   bool Minus = false;
1689   if (getLexer().getKind() == AsmToken::Minus) {
1690     Minus = true;
1691     Parser.Lex();
1692   }
1693 
1694   SMLoc S = Parser.getTok().getLoc();
1695   switch(getLexer().getKind()) {
1696   case AsmToken::Integer: {
1697     int64_t IntVal;
1698     if (parseAbsoluteExpr(IntVal, AbsMod))
1699       return MatchOperand_ParseFail;
1700     if (Minus)
1701       IntVal *= -1;
1702     Operands.push_back(AMDGPUOperand::CreateImm(this, IntVal, S));
1703     return MatchOperand_Success;
1704   }
1705   case AsmToken::Real: {
1706     int64_t IntVal;
1707     if (parseAbsoluteExpr(IntVal, AbsMod))
1708       return MatchOperand_ParseFail;
1709 
1710     APFloat F(BitsToDouble(IntVal));
1711     if (Minus)
1712       F.changeSign();
1713     Operands.push_back(
1714         AMDGPUOperand::CreateImm(this, F.bitcastToAPInt().getZExtValue(), S,
1715                                  AMDGPUOperand::ImmTyNone, true));
1716     return MatchOperand_Success;
1717   }
1718   default:
1719     return Minus ? MatchOperand_ParseFail : MatchOperand_NoMatch;
1720   }
1721 }
1722 
1723 OperandMatchResultTy
1724 AMDGPUAsmParser::parseReg(OperandVector &Operands) {
1725   if (auto R = parseRegister()) {
1726     assert(R->isReg());
1727     R->Reg.IsForcedVOP3 = isForcedVOP3();
1728     Operands.push_back(std::move(R));
1729     return MatchOperand_Success;
1730   }
1731   return MatchOperand_NoMatch;
1732 }
1733 
1734 OperandMatchResultTy
1735 AMDGPUAsmParser::parseRegOrImm(OperandVector &Operands, bool AbsMod) {
1736   auto res = parseImm(Operands, AbsMod);
1737   if (res != MatchOperand_NoMatch) {
1738     return res;
1739   }
1740 
1741   return parseReg(Operands);
1742 }
1743 
1744 OperandMatchResultTy
1745 AMDGPUAsmParser::parseRegOrImmWithFPInputMods(OperandVector &Operands,
1746                                               bool AllowImm) {
1747   bool Negate = false, Negate2 = false, Abs = false, Abs2 = false;
1748 
1749   if (getLexer().getKind()== AsmToken::Minus) {
1750     const AsmToken NextToken = getLexer().peekTok();
1751 
1752     // Disable ambiguous constructs like '--1' etc. Should use neg(-1) instead.
1753     if (NextToken.is(AsmToken::Minus)) {
1754       Error(Parser.getTok().getLoc(), "invalid syntax, expected 'neg' modifier");
1755       return MatchOperand_ParseFail;
1756     }
1757 
1758     // '-' followed by an integer literal N should be interpreted as integer
1759     // negation rather than a floating-point NEG modifier applied to N.
1760     // Beside being contr-intuitive, such use of floating-point NEG modifier
1761     // results in different meaning of integer literals used with VOP1/2/C
1762     // and VOP3, for example:
1763     //    v_exp_f32_e32 v5, -1 // VOP1: src0 = 0xFFFFFFFF
1764     //    v_exp_f32_e64 v5, -1 // VOP3: src0 = 0x80000001
1765     // Negative fp literals should be handled likewise for unifomtity
1766     if (!NextToken.is(AsmToken::Integer) && !NextToken.is(AsmToken::Real)) {
1767       Parser.Lex();
1768       Negate = true;
1769     }
1770   }
1771 
1772   if (getLexer().getKind() == AsmToken::Identifier &&
1773       Parser.getTok().getString() == "neg") {
1774     if (Negate) {
1775       Error(Parser.getTok().getLoc(), "expected register or immediate");
1776       return MatchOperand_ParseFail;
1777     }
1778     Parser.Lex();
1779     Negate2 = true;
1780     if (getLexer().isNot(AsmToken::LParen)) {
1781       Error(Parser.getTok().getLoc(), "expected left paren after neg");
1782       return MatchOperand_ParseFail;
1783     }
1784     Parser.Lex();
1785   }
1786 
1787   if (getLexer().getKind() == AsmToken::Identifier &&
1788       Parser.getTok().getString() == "abs") {
1789     Parser.Lex();
1790     Abs2 = true;
1791     if (getLexer().isNot(AsmToken::LParen)) {
1792       Error(Parser.getTok().getLoc(), "expected left paren after abs");
1793       return MatchOperand_ParseFail;
1794     }
1795     Parser.Lex();
1796   }
1797 
1798   if (getLexer().getKind() == AsmToken::Pipe) {
1799     if (Abs2) {
1800       Error(Parser.getTok().getLoc(), "expected register or immediate");
1801       return MatchOperand_ParseFail;
1802     }
1803     Parser.Lex();
1804     Abs = true;
1805   }
1806 
1807   OperandMatchResultTy Res;
1808   if (AllowImm) {
1809     Res = parseRegOrImm(Operands, Abs);
1810   } else {
1811     Res = parseReg(Operands);
1812   }
1813   if (Res != MatchOperand_Success) {
1814     return Res;
1815   }
1816 
1817   AMDGPUOperand::Modifiers Mods;
1818   if (Abs) {
1819     if (getLexer().getKind() != AsmToken::Pipe) {
1820       Error(Parser.getTok().getLoc(), "expected vertical bar");
1821       return MatchOperand_ParseFail;
1822     }
1823     Parser.Lex();
1824     Mods.Abs = true;
1825   }
1826   if (Abs2) {
1827     if (getLexer().isNot(AsmToken::RParen)) {
1828       Error(Parser.getTok().getLoc(), "expected closing parentheses");
1829       return MatchOperand_ParseFail;
1830     }
1831     Parser.Lex();
1832     Mods.Abs = true;
1833   }
1834 
1835   if (Negate) {
1836     Mods.Neg = true;
1837   } else if (Negate2) {
1838     if (getLexer().isNot(AsmToken::RParen)) {
1839       Error(Parser.getTok().getLoc(), "expected closing parentheses");
1840       return MatchOperand_ParseFail;
1841     }
1842     Parser.Lex();
1843     Mods.Neg = true;
1844   }
1845 
1846   if (Mods.hasFPModifiers()) {
1847     AMDGPUOperand &Op = static_cast<AMDGPUOperand &>(*Operands.back());
1848     Op.setModifiers(Mods);
1849   }
1850   return MatchOperand_Success;
1851 }
1852 
1853 OperandMatchResultTy
1854 AMDGPUAsmParser::parseRegOrImmWithIntInputMods(OperandVector &Operands,
1855                                                bool AllowImm) {
1856   bool Sext = false;
1857 
1858   if (getLexer().getKind() == AsmToken::Identifier &&
1859       Parser.getTok().getString() == "sext") {
1860     Parser.Lex();
1861     Sext = true;
1862     if (getLexer().isNot(AsmToken::LParen)) {
1863       Error(Parser.getTok().getLoc(), "expected left paren after sext");
1864       return MatchOperand_ParseFail;
1865     }
1866     Parser.Lex();
1867   }
1868 
1869   OperandMatchResultTy Res;
1870   if (AllowImm) {
1871     Res = parseRegOrImm(Operands);
1872   } else {
1873     Res = parseReg(Operands);
1874   }
1875   if (Res != MatchOperand_Success) {
1876     return Res;
1877   }
1878 
1879   AMDGPUOperand::Modifiers Mods;
1880   if (Sext) {
1881     if (getLexer().isNot(AsmToken::RParen)) {
1882       Error(Parser.getTok().getLoc(), "expected closing parentheses");
1883       return MatchOperand_ParseFail;
1884     }
1885     Parser.Lex();
1886     Mods.Sext = true;
1887   }
1888 
1889   if (Mods.hasIntModifiers()) {
1890     AMDGPUOperand &Op = static_cast<AMDGPUOperand &>(*Operands.back());
1891     Op.setModifiers(Mods);
1892   }
1893 
1894   return MatchOperand_Success;
1895 }
1896 
1897 OperandMatchResultTy
1898 AMDGPUAsmParser::parseRegWithFPInputMods(OperandVector &Operands) {
1899   return parseRegOrImmWithFPInputMods(Operands, false);
1900 }
1901 
1902 OperandMatchResultTy
1903 AMDGPUAsmParser::parseRegWithIntInputMods(OperandVector &Operands) {
1904   return parseRegOrImmWithIntInputMods(Operands, false);
1905 }
1906 
1907 OperandMatchResultTy AMDGPUAsmParser::parseVReg32OrOff(OperandVector &Operands) {
1908   std::unique_ptr<AMDGPUOperand> Reg = parseRegister();
1909   if (Reg) {
1910     Operands.push_back(std::move(Reg));
1911     return MatchOperand_Success;
1912   }
1913 
1914   const AsmToken &Tok = Parser.getTok();
1915   if (Tok.getString() == "off") {
1916     Operands.push_back(AMDGPUOperand::CreateImm(this, 0, Tok.getLoc(),
1917                                                 AMDGPUOperand::ImmTyOff, false));
1918     Parser.Lex();
1919     return MatchOperand_Success;
1920   }
1921 
1922   return MatchOperand_NoMatch;
1923 }
1924 
1925 unsigned AMDGPUAsmParser::checkTargetMatchPredicate(MCInst &Inst) {
1926   uint64_t TSFlags = MII.get(Inst.getOpcode()).TSFlags;
1927 
1928   if ((getForcedEncodingSize() == 32 && (TSFlags & SIInstrFlags::VOP3)) ||
1929       (getForcedEncodingSize() == 64 && !(TSFlags & SIInstrFlags::VOP3)) ||
1930       (isForcedDPP() && !(TSFlags & SIInstrFlags::DPP)) ||
1931       (isForcedSDWA() && !(TSFlags & SIInstrFlags::SDWA)) )
1932     return Match_InvalidOperand;
1933 
1934   if ((TSFlags & SIInstrFlags::VOP3) &&
1935       (TSFlags & SIInstrFlags::VOPAsmPrefer32Bit) &&
1936       getForcedEncodingSize() != 64)
1937     return Match_PreferE32;
1938 
1939   if (Inst.getOpcode() == AMDGPU::V_MAC_F32_sdwa_vi ||
1940       Inst.getOpcode() == AMDGPU::V_MAC_F16_sdwa_vi) {
1941     // v_mac_f32/16 allow only dst_sel == DWORD;
1942     auto OpNum =
1943         AMDGPU::getNamedOperandIdx(Inst.getOpcode(), AMDGPU::OpName::dst_sel);
1944     const auto &Op = Inst.getOperand(OpNum);
1945     if (!Op.isImm() || Op.getImm() != AMDGPU::SDWA::SdwaSel::DWORD) {
1946       return Match_InvalidOperand;
1947     }
1948   }
1949 
1950   return Match_Success;
1951 }
1952 
1953 // What asm variants we should check
1954 ArrayRef<unsigned> AMDGPUAsmParser::getMatchedVariants() const {
1955   if (getForcedEncodingSize() == 32) {
1956     static const unsigned Variants[] = {AMDGPUAsmVariants::DEFAULT};
1957     return makeArrayRef(Variants);
1958   }
1959 
1960   if (isForcedVOP3()) {
1961     static const unsigned Variants[] = {AMDGPUAsmVariants::VOP3};
1962     return makeArrayRef(Variants);
1963   }
1964 
1965   if (isForcedSDWA()) {
1966     static const unsigned Variants[] = {AMDGPUAsmVariants::SDWA};
1967     return makeArrayRef(Variants);
1968   }
1969 
1970   if (isForcedDPP()) {
1971     static const unsigned Variants[] = {AMDGPUAsmVariants::DPP};
1972     return makeArrayRef(Variants);
1973   }
1974 
1975   static const unsigned Variants[] = {
1976     AMDGPUAsmVariants::DEFAULT, AMDGPUAsmVariants::VOP3,
1977     AMDGPUAsmVariants::SDWA, AMDGPUAsmVariants::DPP
1978   };
1979 
1980   return makeArrayRef(Variants);
1981 }
1982 
1983 unsigned AMDGPUAsmParser::findImplicitSGPRReadInVOP(const MCInst &Inst) const {
1984   const MCInstrDesc &Desc = MII.get(Inst.getOpcode());
1985   const unsigned Num = Desc.getNumImplicitUses();
1986   for (unsigned i = 0; i < Num; ++i) {
1987     unsigned Reg = Desc.ImplicitUses[i];
1988     switch (Reg) {
1989     case AMDGPU::FLAT_SCR:
1990     case AMDGPU::VCC:
1991     case AMDGPU::M0:
1992       return Reg;
1993     default:
1994       break;
1995     }
1996   }
1997   return AMDGPU::NoRegister;
1998 }
1999 
2000 bool AMDGPUAsmParser::isSGPR(unsigned Reg) {
2001   const MCRegisterInfo *TRI = getContext().getRegisterInfo();
2002   const MCRegisterClass SGPRClass = TRI->getRegClass(AMDGPU::SReg_32RegClassID);
2003   const unsigned FirstSubReg = TRI->getSubReg(Reg, 1);
2004   return SGPRClass.contains(FirstSubReg != 0 ? FirstSubReg : Reg) ||
2005          Reg == AMDGPU::SCC;
2006 }
2007 
2008 // NB: This code is correct only when used to check constant
2009 // bus limitations because GFX7 support no f16 inline constants.
2010 // Note that there are no cases when a GFX7 opcode violates
2011 // constant bus limitations due to the use of an f16 constant.
2012 bool AMDGPUAsmParser::isInlineConstant(const MCInst &Inst,
2013                                        unsigned OpIdx) const {
2014   const MCInstrDesc &Desc = MII.get(Inst.getOpcode());
2015 
2016   if (!AMDGPU::isSISrcOperand(Desc, OpIdx)) {
2017     return false;
2018   }
2019 
2020   const MCOperand &MO = Inst.getOperand(OpIdx);
2021 
2022   int64_t Val = MO.getImm();
2023   auto OpSize = AMDGPU::getOperandSize(Desc, OpIdx);
2024 
2025   switch (OpSize) { // expected operand size
2026   case 8:
2027     return AMDGPU::isInlinableLiteral64(Val, hasInv2PiInlineImm());
2028   case 4:
2029     return AMDGPU::isInlinableLiteral32(Val, hasInv2PiInlineImm());
2030   case 2: {
2031     const unsigned OperandType = Desc.OpInfo[OpIdx].OperandType;
2032     if (OperandType == AMDGPU::OPERAND_REG_INLINE_C_V2INT16 ||
2033         OperandType == AMDGPU::OPERAND_REG_INLINE_C_V2FP16) {
2034       return AMDGPU::isInlinableLiteralV216(Val, hasInv2PiInlineImm());
2035     } else {
2036       return AMDGPU::isInlinableLiteral16(Val, hasInv2PiInlineImm());
2037     }
2038   }
2039   default:
2040     llvm_unreachable("invalid operand size");
2041   }
2042 }
2043 
2044 bool AMDGPUAsmParser::usesConstantBus(const MCInst &Inst, unsigned OpIdx) {
2045   const MCOperand &MO = Inst.getOperand(OpIdx);
2046   if (MO.isImm()) {
2047     return !isInlineConstant(Inst, OpIdx);
2048   }
2049   return !MO.isReg() || isSGPR(mc2PseudoReg(MO.getReg()));
2050 }
2051 
2052 bool AMDGPUAsmParser::validateOperandLimitations(const MCInst &Inst) {
2053   const unsigned Opcode = Inst.getOpcode();
2054   const MCInstrDesc &Desc = MII.get(Opcode);
2055   unsigned ConstantBusUseCount = 0;
2056 
2057   if (Desc.TSFlags &
2058       (SIInstrFlags::VOPC |
2059        SIInstrFlags::VOP1 | SIInstrFlags::VOP2 |
2060        SIInstrFlags::VOP3 | SIInstrFlags::VOP3P)) {
2061 
2062     // Check special imm operands (used by madmk, etc)
2063     if (AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::imm) != -1) {
2064       ++ConstantBusUseCount;
2065     }
2066 
2067     unsigned SGPRUsed = findImplicitSGPRReadInVOP(Inst);
2068     if (SGPRUsed != AMDGPU::NoRegister) {
2069       ++ConstantBusUseCount;
2070     }
2071 
2072     const int Src0Idx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::src0);
2073     const int Src1Idx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::src1);
2074     const int Src2Idx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::src2);
2075 
2076     const int OpIndices[] = { Src0Idx, Src1Idx, Src2Idx };
2077 
2078     for (int OpIdx : OpIndices) {
2079       if (OpIdx == -1) break;
2080 
2081       const MCOperand &MO = Inst.getOperand(OpIdx);
2082       if (usesConstantBus(Inst, OpIdx)) {
2083         if (MO.isReg()) {
2084           const unsigned Reg = mc2PseudoReg(MO.getReg());
2085           // Pairs of registers with a partial intersections like these
2086           //   s0, s[0:1]
2087           //   flat_scratch_lo, flat_scratch
2088           //   flat_scratch_lo, flat_scratch_hi
2089           // are theoretically valid but they are disabled anyway.
2090           // Note that this code mimics SIInstrInfo::verifyInstruction
2091           if (Reg != SGPRUsed) {
2092             ++ConstantBusUseCount;
2093           }
2094           SGPRUsed = Reg;
2095         } else { // Expression or a literal
2096           ++ConstantBusUseCount;
2097         }
2098       }
2099     }
2100   }
2101 
2102   return ConstantBusUseCount <= 1;
2103 }
2104 
2105 bool AMDGPUAsmParser::MatchAndEmitInstruction(SMLoc IDLoc, unsigned &Opcode,
2106                                               OperandVector &Operands,
2107                                               MCStreamer &Out,
2108                                               uint64_t &ErrorInfo,
2109                                               bool MatchingInlineAsm) {
2110   MCInst Inst;
2111   unsigned Result = Match_Success;
2112   for (auto Variant : getMatchedVariants()) {
2113     uint64_t EI;
2114     auto R = MatchInstructionImpl(Operands, Inst, EI, MatchingInlineAsm,
2115                                   Variant);
2116     // We order match statuses from least to most specific. We use most specific
2117     // status as resulting
2118     // Match_MnemonicFail < Match_InvalidOperand < Match_MissingFeature < Match_PreferE32
2119     if ((R == Match_Success) ||
2120         (R == Match_PreferE32) ||
2121         (R == Match_MissingFeature && Result != Match_PreferE32) ||
2122         (R == Match_InvalidOperand && Result != Match_MissingFeature
2123                                    && Result != Match_PreferE32) ||
2124         (R == Match_MnemonicFail   && Result != Match_InvalidOperand
2125                                    && Result != Match_MissingFeature
2126                                    && Result != Match_PreferE32)) {
2127       Result = R;
2128       ErrorInfo = EI;
2129     }
2130     if (R == Match_Success)
2131       break;
2132   }
2133 
2134   switch (Result) {
2135   default: break;
2136   case Match_Success:
2137     if (!validateOperandLimitations(Inst)) {
2138       return Error(IDLoc,
2139                    "invalid operand (violates constant bus restrictions)");
2140     }
2141     Inst.setLoc(IDLoc);
2142     Out.EmitInstruction(Inst, getSTI());
2143     return false;
2144 
2145   case Match_MissingFeature:
2146     return Error(IDLoc, "instruction not supported on this GPU");
2147 
2148   case Match_MnemonicFail:
2149     return Error(IDLoc, "unrecognized instruction mnemonic");
2150 
2151   case Match_InvalidOperand: {
2152     SMLoc ErrorLoc = IDLoc;
2153     if (ErrorInfo != ~0ULL) {
2154       if (ErrorInfo >= Operands.size()) {
2155         return Error(IDLoc, "too few operands for instruction");
2156       }
2157       ErrorLoc = ((AMDGPUOperand &)*Operands[ErrorInfo]).getStartLoc();
2158       if (ErrorLoc == SMLoc())
2159         ErrorLoc = IDLoc;
2160     }
2161     return Error(ErrorLoc, "invalid operand for instruction");
2162   }
2163 
2164   case Match_PreferE32:
2165     return Error(IDLoc, "internal error: instruction without _e64 suffix "
2166                         "should be encoded as e32");
2167   }
2168   llvm_unreachable("Implement any new match types added!");
2169 }
2170 
2171 bool AMDGPUAsmParser::ParseAsAbsoluteExpression(uint32_t &Ret) {
2172   int64_t Tmp = -1;
2173   if (getLexer().isNot(AsmToken::Integer) && getLexer().isNot(AsmToken::Identifier)) {
2174     return true;
2175   }
2176   if (getParser().parseAbsoluteExpression(Tmp)) {
2177     return true;
2178   }
2179   Ret = static_cast<uint32_t>(Tmp);
2180   return false;
2181 }
2182 
2183 bool AMDGPUAsmParser::ParseDirectiveMajorMinor(uint32_t &Major,
2184                                                uint32_t &Minor) {
2185   if (ParseAsAbsoluteExpression(Major))
2186     return TokError("invalid major version");
2187 
2188   if (getLexer().isNot(AsmToken::Comma))
2189     return TokError("minor version number required, comma expected");
2190   Lex();
2191 
2192   if (ParseAsAbsoluteExpression(Minor))
2193     return TokError("invalid minor version");
2194 
2195   return false;
2196 }
2197 
2198 bool AMDGPUAsmParser::ParseDirectiveHSACodeObjectVersion() {
2199   uint32_t Major;
2200   uint32_t Minor;
2201 
2202   if (ParseDirectiveMajorMinor(Major, Minor))
2203     return true;
2204 
2205   getTargetStreamer().EmitDirectiveHSACodeObjectVersion(Major, Minor);
2206   return false;
2207 }
2208 
2209 bool AMDGPUAsmParser::ParseDirectiveHSACodeObjectISA() {
2210   uint32_t Major;
2211   uint32_t Minor;
2212   uint32_t Stepping;
2213   StringRef VendorName;
2214   StringRef ArchName;
2215 
2216   // If this directive has no arguments, then use the ISA version for the
2217   // targeted GPU.
2218   if (getLexer().is(AsmToken::EndOfStatement)) {
2219     AMDGPU::IsaInfo::IsaVersion ISA =
2220         AMDGPU::IsaInfo::getIsaVersion(getFeatureBits());
2221     getTargetStreamer().EmitDirectiveHSACodeObjectISA(ISA.Major, ISA.Minor,
2222                                                       ISA.Stepping,
2223                                                       "AMD", "AMDGPU");
2224     return false;
2225   }
2226 
2227   if (ParseDirectiveMajorMinor(Major, Minor))
2228     return true;
2229 
2230   if (getLexer().isNot(AsmToken::Comma))
2231     return TokError("stepping version number required, comma expected");
2232   Lex();
2233 
2234   if (ParseAsAbsoluteExpression(Stepping))
2235     return TokError("invalid stepping version");
2236 
2237   if (getLexer().isNot(AsmToken::Comma))
2238     return TokError("vendor name required, comma expected");
2239   Lex();
2240 
2241   if (getLexer().isNot(AsmToken::String))
2242     return TokError("invalid vendor name");
2243 
2244   VendorName = getLexer().getTok().getStringContents();
2245   Lex();
2246 
2247   if (getLexer().isNot(AsmToken::Comma))
2248     return TokError("arch name required, comma expected");
2249   Lex();
2250 
2251   if (getLexer().isNot(AsmToken::String))
2252     return TokError("invalid arch name");
2253 
2254   ArchName = getLexer().getTok().getStringContents();
2255   Lex();
2256 
2257   getTargetStreamer().EmitDirectiveHSACodeObjectISA(Major, Minor, Stepping,
2258                                                     VendorName, ArchName);
2259   return false;
2260 }
2261 
2262 bool AMDGPUAsmParser::ParseDirectiveCodeObjectMetadata() {
2263   std::string YamlString;
2264   raw_string_ostream YamlStream(YamlString);
2265 
2266   getLexer().setSkipSpace(false);
2267 
2268   bool FoundEnd = false;
2269   while (!getLexer().is(AsmToken::Eof)) {
2270     while (getLexer().is(AsmToken::Space)) {
2271       YamlStream << getLexer().getTok().getString();
2272       Lex();
2273     }
2274 
2275     if (getLexer().is(AsmToken::Identifier)) {
2276       StringRef ID = getLexer().getTok().getIdentifier();
2277       if (ID == AMDGPU::CodeObject::MetadataAssemblerDirectiveEnd) {
2278         Lex();
2279         FoundEnd = true;
2280         break;
2281       }
2282     }
2283 
2284     YamlStream << Parser.parseStringToEndOfStatement()
2285                << getContext().getAsmInfo()->getSeparatorString();
2286 
2287     Parser.eatToEndOfStatement();
2288   }
2289 
2290   getLexer().setSkipSpace(true);
2291 
2292   if (getLexer().is(AsmToken::Eof) && !FoundEnd) {
2293     return TokError(
2294         "expected directive .end_amdgpu_code_object_metadata not found");
2295   }
2296 
2297   YamlStream.flush();
2298 
2299   if (!getTargetStreamer().EmitCodeObjectMetadata(YamlString))
2300     return Error(getParser().getTok().getLoc(), "invalid code object metadata");
2301 
2302   return false;
2303 }
2304 
2305 bool AMDGPUAsmParser::ParseAMDKernelCodeTValue(StringRef ID,
2306                                                amd_kernel_code_t &Header) {
2307   SmallString<40> ErrStr;
2308   raw_svector_ostream Err(ErrStr);
2309   if (!parseAmdKernelCodeField(ID, getParser(), Header, Err)) {
2310     return TokError(Err.str());
2311   }
2312   Lex();
2313   return false;
2314 }
2315 
2316 bool AMDGPUAsmParser::ParseDirectiveAMDKernelCodeT() {
2317   amd_kernel_code_t Header;
2318   AMDGPU::initDefaultAMDKernelCodeT(Header, getFeatureBits());
2319 
2320   while (true) {
2321     // Lex EndOfStatement.  This is in a while loop, because lexing a comment
2322     // will set the current token to EndOfStatement.
2323     while(getLexer().is(AsmToken::EndOfStatement))
2324       Lex();
2325 
2326     if (getLexer().isNot(AsmToken::Identifier))
2327       return TokError("expected value identifier or .end_amd_kernel_code_t");
2328 
2329     StringRef ID = getLexer().getTok().getIdentifier();
2330     Lex();
2331 
2332     if (ID == ".end_amd_kernel_code_t")
2333       break;
2334 
2335     if (ParseAMDKernelCodeTValue(ID, Header))
2336       return true;
2337   }
2338 
2339   getTargetStreamer().EmitAMDKernelCodeT(Header);
2340 
2341   return false;
2342 }
2343 
2344 bool AMDGPUAsmParser::ParseSectionDirectiveHSAText() {
2345   getParser().getStreamer().SwitchSection(
2346       AMDGPU::getHSATextSection(getContext()));
2347   return false;
2348 }
2349 
2350 bool AMDGPUAsmParser::ParseDirectiveAMDGPUHsaKernel() {
2351   if (getLexer().isNot(AsmToken::Identifier))
2352     return TokError("expected symbol name");
2353 
2354   StringRef KernelName = Parser.getTok().getString();
2355 
2356   getTargetStreamer().EmitAMDGPUSymbolType(KernelName,
2357                                            ELF::STT_AMDGPU_HSA_KERNEL);
2358   Lex();
2359   KernelScope.initialize(getContext());
2360   return false;
2361 }
2362 
2363 bool AMDGPUAsmParser::ParseDirectiveAMDGPUHsaModuleGlobal() {
2364   if (getLexer().isNot(AsmToken::Identifier))
2365     return TokError("expected symbol name");
2366 
2367   StringRef GlobalName = Parser.getTok().getIdentifier();
2368 
2369   getTargetStreamer().EmitAMDGPUHsaModuleScopeGlobal(GlobalName);
2370   Lex();
2371   return false;
2372 }
2373 
2374 bool AMDGPUAsmParser::ParseDirectiveAMDGPUHsaProgramGlobal() {
2375   if (getLexer().isNot(AsmToken::Identifier))
2376     return TokError("expected symbol name");
2377 
2378   StringRef GlobalName = Parser.getTok().getIdentifier();
2379 
2380   getTargetStreamer().EmitAMDGPUHsaProgramScopeGlobal(GlobalName);
2381   Lex();
2382   return false;
2383 }
2384 
2385 bool AMDGPUAsmParser::ParseSectionDirectiveHSADataGlobalAgent() {
2386   getParser().getStreamer().SwitchSection(
2387       AMDGPU::getHSADataGlobalAgentSection(getContext()));
2388   return false;
2389 }
2390 
2391 bool AMDGPUAsmParser::ParseSectionDirectiveHSADataGlobalProgram() {
2392   getParser().getStreamer().SwitchSection(
2393       AMDGPU::getHSADataGlobalProgramSection(getContext()));
2394   return false;
2395 }
2396 
2397 bool AMDGPUAsmParser::ParseSectionDirectiveHSARodataReadonlyAgent() {
2398   getParser().getStreamer().SwitchSection(
2399       AMDGPU::getHSARodataReadonlyAgentSection(getContext()));
2400   return false;
2401 }
2402 
2403 bool AMDGPUAsmParser::ParseDirective(AsmToken DirectiveID) {
2404   StringRef IDVal = DirectiveID.getString();
2405 
2406   if (IDVal == ".hsa_code_object_version")
2407     return ParseDirectiveHSACodeObjectVersion();
2408 
2409   if (IDVal == ".hsa_code_object_isa")
2410     return ParseDirectiveHSACodeObjectISA();
2411 
2412   if (IDVal == AMDGPU::CodeObject::MetadataAssemblerDirectiveBegin)
2413     return ParseDirectiveCodeObjectMetadata();
2414 
2415   if (IDVal == ".amd_kernel_code_t")
2416     return ParseDirectiveAMDKernelCodeT();
2417 
2418   if (IDVal == ".hsatext")
2419     return ParseSectionDirectiveHSAText();
2420 
2421   if (IDVal == ".amdgpu_hsa_kernel")
2422     return ParseDirectiveAMDGPUHsaKernel();
2423 
2424   if (IDVal == ".amdgpu_hsa_module_global")
2425     return ParseDirectiveAMDGPUHsaModuleGlobal();
2426 
2427   if (IDVal == ".amdgpu_hsa_program_global")
2428     return ParseDirectiveAMDGPUHsaProgramGlobal();
2429 
2430   if (IDVal == ".hsadata_global_agent")
2431     return ParseSectionDirectiveHSADataGlobalAgent();
2432 
2433   if (IDVal == ".hsadata_global_program")
2434     return ParseSectionDirectiveHSADataGlobalProgram();
2435 
2436   if (IDVal == ".hsarodata_readonly_agent")
2437     return ParseSectionDirectiveHSARodataReadonlyAgent();
2438 
2439   return true;
2440 }
2441 
2442 bool AMDGPUAsmParser::subtargetHasRegister(const MCRegisterInfo &MRI,
2443                                            unsigned RegNo) const {
2444   if (isCI())
2445     return true;
2446 
2447   if (isSI()) {
2448     // No flat_scr
2449     switch (RegNo) {
2450     case AMDGPU::FLAT_SCR:
2451     case AMDGPU::FLAT_SCR_LO:
2452     case AMDGPU::FLAT_SCR_HI:
2453       return false;
2454     default:
2455       return true;
2456     }
2457   }
2458 
2459   // VI only has 102 SGPRs, so make sure we aren't trying to use the 2 more that
2460   // SI/CI have.
2461   for (MCRegAliasIterator R(AMDGPU::SGPR102_SGPR103, &MRI, true);
2462        R.isValid(); ++R) {
2463     if (*R == RegNo)
2464       return false;
2465   }
2466 
2467   return true;
2468 }
2469 
2470 OperandMatchResultTy
2471 AMDGPUAsmParser::parseOperand(OperandVector &Operands, StringRef Mnemonic) {
2472   // Try to parse with a custom parser
2473   OperandMatchResultTy ResTy = MatchOperandParserImpl(Operands, Mnemonic);
2474 
2475   // If we successfully parsed the operand or if there as an error parsing,
2476   // we are done.
2477   //
2478   // If we are parsing after we reach EndOfStatement then this means we
2479   // are appending default values to the Operands list.  This is only done
2480   // by custom parser, so we shouldn't continue on to the generic parsing.
2481   if (ResTy == MatchOperand_Success || ResTy == MatchOperand_ParseFail ||
2482       getLexer().is(AsmToken::EndOfStatement))
2483     return ResTy;
2484 
2485   ResTy = parseRegOrImm(Operands);
2486 
2487   if (ResTy == MatchOperand_Success)
2488     return ResTy;
2489 
2490   if (getLexer().getKind() == AsmToken::Identifier) {
2491     // If this identifier is a symbol, we want to create an expression for it.
2492     // It is a little difficult to distinguish between a symbol name, and
2493     // an instruction flag like 'gds'.  In order to do this, we parse
2494     // all tokens as expressions and then treate the symbol name as the token
2495     // string when we want to interpret the operand as a token.
2496     const auto &Tok = Parser.getTok();
2497     SMLoc S = Tok.getLoc();
2498     const MCExpr *Expr = nullptr;
2499     if (!Parser.parseExpression(Expr)) {
2500       Operands.push_back(AMDGPUOperand::CreateExpr(this, Expr, S));
2501       return MatchOperand_Success;
2502     }
2503 
2504     Operands.push_back(AMDGPUOperand::CreateToken(this, Tok.getString(), Tok.getLoc()));
2505     Parser.Lex();
2506     return MatchOperand_Success;
2507   }
2508   return MatchOperand_NoMatch;
2509 }
2510 
2511 StringRef AMDGPUAsmParser::parseMnemonicSuffix(StringRef Name) {
2512   // Clear any forced encodings from the previous instruction.
2513   setForcedEncodingSize(0);
2514   setForcedDPP(false);
2515   setForcedSDWA(false);
2516 
2517   if (Name.endswith("_e64")) {
2518     setForcedEncodingSize(64);
2519     return Name.substr(0, Name.size() - 4);
2520   } else if (Name.endswith("_e32")) {
2521     setForcedEncodingSize(32);
2522     return Name.substr(0, Name.size() - 4);
2523   } else if (Name.endswith("_dpp")) {
2524     setForcedDPP(true);
2525     return Name.substr(0, Name.size() - 4);
2526   } else if (Name.endswith("_sdwa")) {
2527     setForcedSDWA(true);
2528     return Name.substr(0, Name.size() - 5);
2529   }
2530   return Name;
2531 }
2532 
2533 bool AMDGPUAsmParser::ParseInstruction(ParseInstructionInfo &Info,
2534                                        StringRef Name,
2535                                        SMLoc NameLoc, OperandVector &Operands) {
2536   // Add the instruction mnemonic
2537   Name = parseMnemonicSuffix(Name);
2538   Operands.push_back(AMDGPUOperand::CreateToken(this, Name, NameLoc));
2539 
2540   while (!getLexer().is(AsmToken::EndOfStatement)) {
2541     OperandMatchResultTy Res = parseOperand(Operands, Name);
2542 
2543     // Eat the comma or space if there is one.
2544     if (getLexer().is(AsmToken::Comma))
2545       Parser.Lex();
2546 
2547     switch (Res) {
2548       case MatchOperand_Success: break;
2549       case MatchOperand_ParseFail:
2550         Error(getLexer().getLoc(), "failed parsing operand.");
2551         while (!getLexer().is(AsmToken::EndOfStatement)) {
2552           Parser.Lex();
2553         }
2554         return true;
2555       case MatchOperand_NoMatch:
2556         Error(getLexer().getLoc(), "not a valid operand.");
2557         while (!getLexer().is(AsmToken::EndOfStatement)) {
2558           Parser.Lex();
2559         }
2560         return true;
2561     }
2562   }
2563 
2564   return false;
2565 }
2566 
2567 //===----------------------------------------------------------------------===//
2568 // Utility functions
2569 //===----------------------------------------------------------------------===//
2570 
2571 OperandMatchResultTy
2572 AMDGPUAsmParser::parseIntWithPrefix(const char *Prefix, int64_t &Int) {
2573   switch(getLexer().getKind()) {
2574     default: return MatchOperand_NoMatch;
2575     case AsmToken::Identifier: {
2576       StringRef Name = Parser.getTok().getString();
2577       if (!Name.equals(Prefix)) {
2578         return MatchOperand_NoMatch;
2579       }
2580 
2581       Parser.Lex();
2582       if (getLexer().isNot(AsmToken::Colon))
2583         return MatchOperand_ParseFail;
2584 
2585       Parser.Lex();
2586       if (getLexer().isNot(AsmToken::Integer))
2587         return MatchOperand_ParseFail;
2588 
2589       if (getParser().parseAbsoluteExpression(Int))
2590         return MatchOperand_ParseFail;
2591       break;
2592     }
2593   }
2594   return MatchOperand_Success;
2595 }
2596 
2597 OperandMatchResultTy
2598 AMDGPUAsmParser::parseIntWithPrefix(const char *Prefix, OperandVector &Operands,
2599                                     AMDGPUOperand::ImmTy ImmTy,
2600                                     bool (*ConvertResult)(int64_t&)) {
2601   SMLoc S = Parser.getTok().getLoc();
2602   int64_t Value = 0;
2603 
2604   OperandMatchResultTy Res = parseIntWithPrefix(Prefix, Value);
2605   if (Res != MatchOperand_Success)
2606     return Res;
2607 
2608   if (ConvertResult && !ConvertResult(Value)) {
2609     return MatchOperand_ParseFail;
2610   }
2611 
2612   Operands.push_back(AMDGPUOperand::CreateImm(this, Value, S, ImmTy));
2613   return MatchOperand_Success;
2614 }
2615 
2616 OperandMatchResultTy AMDGPUAsmParser::parseOperandArrayWithPrefix(
2617   const char *Prefix,
2618   OperandVector &Operands,
2619   AMDGPUOperand::ImmTy ImmTy,
2620   bool (*ConvertResult)(int64_t&)) {
2621   StringRef Name = Parser.getTok().getString();
2622   if (!Name.equals(Prefix))
2623     return MatchOperand_NoMatch;
2624 
2625   Parser.Lex();
2626   if (getLexer().isNot(AsmToken::Colon))
2627     return MatchOperand_ParseFail;
2628 
2629   Parser.Lex();
2630   if (getLexer().isNot(AsmToken::LBrac))
2631     return MatchOperand_ParseFail;
2632   Parser.Lex();
2633 
2634   unsigned Val = 0;
2635   SMLoc S = Parser.getTok().getLoc();
2636 
2637   // FIXME: How to verify the number of elements matches the number of src
2638   // operands?
2639   for (int I = 0; I < 3; ++I) {
2640     if (I != 0) {
2641       if (getLexer().is(AsmToken::RBrac))
2642         break;
2643 
2644       if (getLexer().isNot(AsmToken::Comma))
2645         return MatchOperand_ParseFail;
2646       Parser.Lex();
2647     }
2648 
2649     if (getLexer().isNot(AsmToken::Integer))
2650       return MatchOperand_ParseFail;
2651 
2652     int64_t Op;
2653     if (getParser().parseAbsoluteExpression(Op))
2654       return MatchOperand_ParseFail;
2655 
2656     if (Op != 0 && Op != 1)
2657       return MatchOperand_ParseFail;
2658     Val |= (Op << I);
2659   }
2660 
2661   Parser.Lex();
2662   Operands.push_back(AMDGPUOperand::CreateImm(this, Val, S, ImmTy));
2663   return MatchOperand_Success;
2664 }
2665 
2666 OperandMatchResultTy
2667 AMDGPUAsmParser::parseNamedBit(const char *Name, OperandVector &Operands,
2668                                AMDGPUOperand::ImmTy ImmTy) {
2669   int64_t Bit = 0;
2670   SMLoc S = Parser.getTok().getLoc();
2671 
2672   // We are at the end of the statement, and this is a default argument, so
2673   // use a default value.
2674   if (getLexer().isNot(AsmToken::EndOfStatement)) {
2675     switch(getLexer().getKind()) {
2676       case AsmToken::Identifier: {
2677         StringRef Tok = Parser.getTok().getString();
2678         if (Tok == Name) {
2679           Bit = 1;
2680           Parser.Lex();
2681         } else if (Tok.startswith("no") && Tok.endswith(Name)) {
2682           Bit = 0;
2683           Parser.Lex();
2684         } else {
2685           return MatchOperand_NoMatch;
2686         }
2687         break;
2688       }
2689       default:
2690         return MatchOperand_NoMatch;
2691     }
2692   }
2693 
2694   Operands.push_back(AMDGPUOperand::CreateImm(this, Bit, S, ImmTy));
2695   return MatchOperand_Success;
2696 }
2697 
2698 static void addOptionalImmOperand(
2699   MCInst& Inst, const OperandVector& Operands,
2700   AMDGPUAsmParser::OptionalImmIndexMap& OptionalIdx,
2701   AMDGPUOperand::ImmTy ImmT,
2702   int64_t Default = 0) {
2703   auto i = OptionalIdx.find(ImmT);
2704   if (i != OptionalIdx.end()) {
2705     unsigned Idx = i->second;
2706     ((AMDGPUOperand &)*Operands[Idx]).addImmOperands(Inst, 1);
2707   } else {
2708     Inst.addOperand(MCOperand::createImm(Default));
2709   }
2710 }
2711 
2712 OperandMatchResultTy
2713 AMDGPUAsmParser::parseStringWithPrefix(StringRef Prefix, StringRef &Value) {
2714   if (getLexer().isNot(AsmToken::Identifier)) {
2715     return MatchOperand_NoMatch;
2716   }
2717   StringRef Tok = Parser.getTok().getString();
2718   if (Tok != Prefix) {
2719     return MatchOperand_NoMatch;
2720   }
2721 
2722   Parser.Lex();
2723   if (getLexer().isNot(AsmToken::Colon)) {
2724     return MatchOperand_ParseFail;
2725   }
2726 
2727   Parser.Lex();
2728   if (getLexer().isNot(AsmToken::Identifier)) {
2729     return MatchOperand_ParseFail;
2730   }
2731 
2732   Value = Parser.getTok().getString();
2733   return MatchOperand_Success;
2734 }
2735 
2736 //===----------------------------------------------------------------------===//
2737 // ds
2738 //===----------------------------------------------------------------------===//
2739 
2740 void AMDGPUAsmParser::cvtDSOffset01(MCInst &Inst,
2741                                     const OperandVector &Operands) {
2742   OptionalImmIndexMap OptionalIdx;
2743 
2744   for (unsigned i = 1, e = Operands.size(); i != e; ++i) {
2745     AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[i]);
2746 
2747     // Add the register arguments
2748     if (Op.isReg()) {
2749       Op.addRegOperands(Inst, 1);
2750       continue;
2751     }
2752 
2753     // Handle optional arguments
2754     OptionalIdx[Op.getImmTy()] = i;
2755   }
2756 
2757   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyOffset0);
2758   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyOffset1);
2759   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyGDS);
2760 
2761   Inst.addOperand(MCOperand::createReg(AMDGPU::M0)); // m0
2762 }
2763 
2764 void AMDGPUAsmParser::cvtDSImpl(MCInst &Inst, const OperandVector &Operands,
2765                                 bool IsGdsHardcoded) {
2766   OptionalImmIndexMap OptionalIdx;
2767 
2768   for (unsigned i = 1, e = Operands.size(); i != e; ++i) {
2769     AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[i]);
2770 
2771     // Add the register arguments
2772     if (Op.isReg()) {
2773       Op.addRegOperands(Inst, 1);
2774       continue;
2775     }
2776 
2777     if (Op.isToken() && Op.getToken() == "gds") {
2778       IsGdsHardcoded = true;
2779       continue;
2780     }
2781 
2782     // Handle optional arguments
2783     OptionalIdx[Op.getImmTy()] = i;
2784   }
2785 
2786   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyOffset);
2787   if (!IsGdsHardcoded) {
2788     addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyGDS);
2789   }
2790   Inst.addOperand(MCOperand::createReg(AMDGPU::M0)); // m0
2791 }
2792 
2793 void AMDGPUAsmParser::cvtExp(MCInst &Inst, const OperandVector &Operands) {
2794   OptionalImmIndexMap OptionalIdx;
2795 
2796   unsigned EnMask = 0;
2797   int SrcIdx = 0;
2798 
2799   for (unsigned i = 1, e = Operands.size(); i != e; ++i) {
2800     AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[i]);
2801 
2802     // Add the register arguments
2803     if (Op.isReg()) {
2804       EnMask |= (1 << SrcIdx);
2805       Op.addRegOperands(Inst, 1);
2806       ++SrcIdx;
2807       continue;
2808     }
2809 
2810     if (Op.isOff()) {
2811       ++SrcIdx;
2812       Inst.addOperand(MCOperand::createReg(AMDGPU::NoRegister));
2813       continue;
2814     }
2815 
2816     if (Op.isImm() && Op.getImmTy() == AMDGPUOperand::ImmTyExpTgt) {
2817       Op.addImmOperands(Inst, 1);
2818       continue;
2819     }
2820 
2821     if (Op.isToken() && Op.getToken() == "done")
2822       continue;
2823 
2824     // Handle optional arguments
2825     OptionalIdx[Op.getImmTy()] = i;
2826   }
2827 
2828   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyExpVM);
2829   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyExpCompr);
2830 
2831   Inst.addOperand(MCOperand::createImm(EnMask));
2832 }
2833 
2834 //===----------------------------------------------------------------------===//
2835 // s_waitcnt
2836 //===----------------------------------------------------------------------===//
2837 
2838 bool AMDGPUAsmParser::parseCnt(int64_t &IntVal) {
2839   StringRef CntName = Parser.getTok().getString();
2840   int64_t CntVal;
2841 
2842   Parser.Lex();
2843   if (getLexer().isNot(AsmToken::LParen))
2844     return true;
2845 
2846   Parser.Lex();
2847   if (getLexer().isNot(AsmToken::Integer))
2848     return true;
2849 
2850   if (getParser().parseAbsoluteExpression(CntVal))
2851     return true;
2852 
2853   if (getLexer().isNot(AsmToken::RParen))
2854     return true;
2855 
2856   Parser.Lex();
2857   if (getLexer().is(AsmToken::Amp) || getLexer().is(AsmToken::Comma))
2858     Parser.Lex();
2859 
2860   AMDGPU::IsaInfo::IsaVersion ISA =
2861       AMDGPU::IsaInfo::getIsaVersion(getFeatureBits());
2862   if (CntName == "vmcnt")
2863     IntVal = encodeVmcnt(ISA, IntVal, CntVal);
2864   else if (CntName == "expcnt")
2865     IntVal = encodeExpcnt(ISA, IntVal, CntVal);
2866   else if (CntName == "lgkmcnt")
2867     IntVal = encodeLgkmcnt(ISA, IntVal, CntVal);
2868   else
2869     return true;
2870 
2871   return false;
2872 }
2873 
2874 OperandMatchResultTy
2875 AMDGPUAsmParser::parseSWaitCntOps(OperandVector &Operands) {
2876   AMDGPU::IsaInfo::IsaVersion ISA =
2877       AMDGPU::IsaInfo::getIsaVersion(getFeatureBits());
2878   int64_t Waitcnt = getWaitcntBitMask(ISA);
2879   SMLoc S = Parser.getTok().getLoc();
2880 
2881   switch(getLexer().getKind()) {
2882     default: return MatchOperand_ParseFail;
2883     case AsmToken::Integer:
2884       // The operand can be an integer value.
2885       if (getParser().parseAbsoluteExpression(Waitcnt))
2886         return MatchOperand_ParseFail;
2887       break;
2888 
2889     case AsmToken::Identifier:
2890       do {
2891         if (parseCnt(Waitcnt))
2892           return MatchOperand_ParseFail;
2893       } while(getLexer().isNot(AsmToken::EndOfStatement));
2894       break;
2895   }
2896   Operands.push_back(AMDGPUOperand::CreateImm(this, Waitcnt, S));
2897   return MatchOperand_Success;
2898 }
2899 
2900 bool AMDGPUAsmParser::parseHwregConstruct(OperandInfoTy &HwReg, int64_t &Offset,
2901                                           int64_t &Width) {
2902   using namespace llvm::AMDGPU::Hwreg;
2903 
2904   if (Parser.getTok().getString() != "hwreg")
2905     return true;
2906   Parser.Lex();
2907 
2908   if (getLexer().isNot(AsmToken::LParen))
2909     return true;
2910   Parser.Lex();
2911 
2912   if (getLexer().is(AsmToken::Identifier)) {
2913     HwReg.IsSymbolic = true;
2914     HwReg.Id = ID_UNKNOWN_;
2915     const StringRef tok = Parser.getTok().getString();
2916     for (int i = ID_SYMBOLIC_FIRST_; i < ID_SYMBOLIC_LAST_; ++i) {
2917       if (tok == IdSymbolic[i]) {
2918         HwReg.Id = i;
2919         break;
2920       }
2921     }
2922     Parser.Lex();
2923   } else {
2924     HwReg.IsSymbolic = false;
2925     if (getLexer().isNot(AsmToken::Integer))
2926       return true;
2927     if (getParser().parseAbsoluteExpression(HwReg.Id))
2928       return true;
2929   }
2930 
2931   if (getLexer().is(AsmToken::RParen)) {
2932     Parser.Lex();
2933     return false;
2934   }
2935 
2936   // optional params
2937   if (getLexer().isNot(AsmToken::Comma))
2938     return true;
2939   Parser.Lex();
2940 
2941   if (getLexer().isNot(AsmToken::Integer))
2942     return true;
2943   if (getParser().parseAbsoluteExpression(Offset))
2944     return true;
2945 
2946   if (getLexer().isNot(AsmToken::Comma))
2947     return true;
2948   Parser.Lex();
2949 
2950   if (getLexer().isNot(AsmToken::Integer))
2951     return true;
2952   if (getParser().parseAbsoluteExpression(Width))
2953     return true;
2954 
2955   if (getLexer().isNot(AsmToken::RParen))
2956     return true;
2957   Parser.Lex();
2958 
2959   return false;
2960 }
2961 
2962 OperandMatchResultTy AMDGPUAsmParser::parseHwreg(OperandVector &Operands) {
2963   using namespace llvm::AMDGPU::Hwreg;
2964 
2965   int64_t Imm16Val = 0;
2966   SMLoc S = Parser.getTok().getLoc();
2967 
2968   switch(getLexer().getKind()) {
2969     default: return MatchOperand_NoMatch;
2970     case AsmToken::Integer:
2971       // The operand can be an integer value.
2972       if (getParser().parseAbsoluteExpression(Imm16Val))
2973         return MatchOperand_NoMatch;
2974       if (Imm16Val < 0 || !isUInt<16>(Imm16Val)) {
2975         Error(S, "invalid immediate: only 16-bit values are legal");
2976         // Do not return error code, but create an imm operand anyway and proceed
2977         // to the next operand, if any. That avoids unneccessary error messages.
2978       }
2979       break;
2980 
2981     case AsmToken::Identifier: {
2982         OperandInfoTy HwReg(ID_UNKNOWN_);
2983         int64_t Offset = OFFSET_DEFAULT_;
2984         int64_t Width = WIDTH_M1_DEFAULT_ + 1;
2985         if (parseHwregConstruct(HwReg, Offset, Width))
2986           return MatchOperand_ParseFail;
2987         if (HwReg.Id < 0 || !isUInt<ID_WIDTH_>(HwReg.Id)) {
2988           if (HwReg.IsSymbolic)
2989             Error(S, "invalid symbolic name of hardware register");
2990           else
2991             Error(S, "invalid code of hardware register: only 6-bit values are legal");
2992         }
2993         if (Offset < 0 || !isUInt<OFFSET_WIDTH_>(Offset))
2994           Error(S, "invalid bit offset: only 5-bit values are legal");
2995         if ((Width-1) < 0 || !isUInt<WIDTH_M1_WIDTH_>(Width-1))
2996           Error(S, "invalid bitfield width: only values from 1 to 32 are legal");
2997         Imm16Val = (HwReg.Id << ID_SHIFT_) | (Offset << OFFSET_SHIFT_) | ((Width-1) << WIDTH_M1_SHIFT_);
2998       }
2999       break;
3000   }
3001   Operands.push_back(AMDGPUOperand::CreateImm(this, Imm16Val, S, AMDGPUOperand::ImmTyHwreg));
3002   return MatchOperand_Success;
3003 }
3004 
3005 bool AMDGPUOperand::isSWaitCnt() const {
3006   return isImm();
3007 }
3008 
3009 bool AMDGPUOperand::isHwreg() const {
3010   return isImmTy(ImmTyHwreg);
3011 }
3012 
3013 bool AMDGPUAsmParser::parseSendMsgConstruct(OperandInfoTy &Msg, OperandInfoTy &Operation, int64_t &StreamId) {
3014   using namespace llvm::AMDGPU::SendMsg;
3015 
3016   if (Parser.getTok().getString() != "sendmsg")
3017     return true;
3018   Parser.Lex();
3019 
3020   if (getLexer().isNot(AsmToken::LParen))
3021     return true;
3022   Parser.Lex();
3023 
3024   if (getLexer().is(AsmToken::Identifier)) {
3025     Msg.IsSymbolic = true;
3026     Msg.Id = ID_UNKNOWN_;
3027     const std::string tok = Parser.getTok().getString();
3028     for (int i = ID_GAPS_FIRST_; i < ID_GAPS_LAST_; ++i) {
3029       switch(i) {
3030         default: continue; // Omit gaps.
3031         case ID_INTERRUPT: case ID_GS: case ID_GS_DONE:  case ID_SYSMSG: break;
3032       }
3033       if (tok == IdSymbolic[i]) {
3034         Msg.Id = i;
3035         break;
3036       }
3037     }
3038     Parser.Lex();
3039   } else {
3040     Msg.IsSymbolic = false;
3041     if (getLexer().isNot(AsmToken::Integer))
3042       return true;
3043     if (getParser().parseAbsoluteExpression(Msg.Id))
3044       return true;
3045     if (getLexer().is(AsmToken::Integer))
3046       if (getParser().parseAbsoluteExpression(Msg.Id))
3047         Msg.Id = ID_UNKNOWN_;
3048   }
3049   if (Msg.Id == ID_UNKNOWN_) // Don't know how to parse the rest.
3050     return false;
3051 
3052   if (!(Msg.Id == ID_GS || Msg.Id == ID_GS_DONE || Msg.Id == ID_SYSMSG)) {
3053     if (getLexer().isNot(AsmToken::RParen))
3054       return true;
3055     Parser.Lex();
3056     return false;
3057   }
3058 
3059   if (getLexer().isNot(AsmToken::Comma))
3060     return true;
3061   Parser.Lex();
3062 
3063   assert(Msg.Id == ID_GS || Msg.Id == ID_GS_DONE || Msg.Id == ID_SYSMSG);
3064   Operation.Id = ID_UNKNOWN_;
3065   if (getLexer().is(AsmToken::Identifier)) {
3066     Operation.IsSymbolic = true;
3067     const char* const *S = (Msg.Id == ID_SYSMSG) ? OpSysSymbolic : OpGsSymbolic;
3068     const int F = (Msg.Id == ID_SYSMSG) ? OP_SYS_FIRST_ : OP_GS_FIRST_;
3069     const int L = (Msg.Id == ID_SYSMSG) ? OP_SYS_LAST_ : OP_GS_LAST_;
3070     const StringRef Tok = Parser.getTok().getString();
3071     for (int i = F; i < L; ++i) {
3072       if (Tok == S[i]) {
3073         Operation.Id = i;
3074         break;
3075       }
3076     }
3077     Parser.Lex();
3078   } else {
3079     Operation.IsSymbolic = false;
3080     if (getLexer().isNot(AsmToken::Integer))
3081       return true;
3082     if (getParser().parseAbsoluteExpression(Operation.Id))
3083       return true;
3084   }
3085 
3086   if ((Msg.Id == ID_GS || Msg.Id == ID_GS_DONE) && Operation.Id != OP_GS_NOP) {
3087     // Stream id is optional.
3088     if (getLexer().is(AsmToken::RParen)) {
3089       Parser.Lex();
3090       return false;
3091     }
3092 
3093     if (getLexer().isNot(AsmToken::Comma))
3094       return true;
3095     Parser.Lex();
3096 
3097     if (getLexer().isNot(AsmToken::Integer))
3098       return true;
3099     if (getParser().parseAbsoluteExpression(StreamId))
3100       return true;
3101   }
3102 
3103   if (getLexer().isNot(AsmToken::RParen))
3104     return true;
3105   Parser.Lex();
3106   return false;
3107 }
3108 
3109 OperandMatchResultTy AMDGPUAsmParser::parseInterpSlot(OperandVector &Operands) {
3110   if (getLexer().getKind() != AsmToken::Identifier)
3111     return MatchOperand_NoMatch;
3112 
3113   StringRef Str = Parser.getTok().getString();
3114   int Slot = StringSwitch<int>(Str)
3115     .Case("p10", 0)
3116     .Case("p20", 1)
3117     .Case("p0", 2)
3118     .Default(-1);
3119 
3120   SMLoc S = Parser.getTok().getLoc();
3121   if (Slot == -1)
3122     return MatchOperand_ParseFail;
3123 
3124   Parser.Lex();
3125   Operands.push_back(AMDGPUOperand::CreateImm(this, Slot, S,
3126                                               AMDGPUOperand::ImmTyInterpSlot));
3127   return MatchOperand_Success;
3128 }
3129 
3130 OperandMatchResultTy AMDGPUAsmParser::parseInterpAttr(OperandVector &Operands) {
3131   if (getLexer().getKind() != AsmToken::Identifier)
3132     return MatchOperand_NoMatch;
3133 
3134   StringRef Str = Parser.getTok().getString();
3135   if (!Str.startswith("attr"))
3136     return MatchOperand_NoMatch;
3137 
3138   StringRef Chan = Str.take_back(2);
3139   int AttrChan = StringSwitch<int>(Chan)
3140     .Case(".x", 0)
3141     .Case(".y", 1)
3142     .Case(".z", 2)
3143     .Case(".w", 3)
3144     .Default(-1);
3145   if (AttrChan == -1)
3146     return MatchOperand_ParseFail;
3147 
3148   Str = Str.drop_back(2).drop_front(4);
3149 
3150   uint8_t Attr;
3151   if (Str.getAsInteger(10, Attr))
3152     return MatchOperand_ParseFail;
3153 
3154   SMLoc S = Parser.getTok().getLoc();
3155   Parser.Lex();
3156   if (Attr > 63) {
3157     Error(S, "out of bounds attr");
3158     return MatchOperand_Success;
3159   }
3160 
3161   SMLoc SChan = SMLoc::getFromPointer(Chan.data());
3162 
3163   Operands.push_back(AMDGPUOperand::CreateImm(this, Attr, S,
3164                                               AMDGPUOperand::ImmTyInterpAttr));
3165   Operands.push_back(AMDGPUOperand::CreateImm(this, AttrChan, SChan,
3166                                               AMDGPUOperand::ImmTyAttrChan));
3167   return MatchOperand_Success;
3168 }
3169 
3170 void AMDGPUAsmParser::errorExpTgt() {
3171   Error(Parser.getTok().getLoc(), "invalid exp target");
3172 }
3173 
3174 OperandMatchResultTy AMDGPUAsmParser::parseExpTgtImpl(StringRef Str,
3175                                                       uint8_t &Val) {
3176   if (Str == "null") {
3177     Val = 9;
3178     return MatchOperand_Success;
3179   }
3180 
3181   if (Str.startswith("mrt")) {
3182     Str = Str.drop_front(3);
3183     if (Str == "z") { // == mrtz
3184       Val = 8;
3185       return MatchOperand_Success;
3186     }
3187 
3188     if (Str.getAsInteger(10, Val))
3189       return MatchOperand_ParseFail;
3190 
3191     if (Val > 7)
3192       errorExpTgt();
3193 
3194     return MatchOperand_Success;
3195   }
3196 
3197   if (Str.startswith("pos")) {
3198     Str = Str.drop_front(3);
3199     if (Str.getAsInteger(10, Val))
3200       return MatchOperand_ParseFail;
3201 
3202     if (Val > 3)
3203       errorExpTgt();
3204 
3205     Val += 12;
3206     return MatchOperand_Success;
3207   }
3208 
3209   if (Str.startswith("param")) {
3210     Str = Str.drop_front(5);
3211     if (Str.getAsInteger(10, Val))
3212       return MatchOperand_ParseFail;
3213 
3214     if (Val >= 32)
3215       errorExpTgt();
3216 
3217     Val += 32;
3218     return MatchOperand_Success;
3219   }
3220 
3221   if (Str.startswith("invalid_target_")) {
3222     Str = Str.drop_front(15);
3223     if (Str.getAsInteger(10, Val))
3224       return MatchOperand_ParseFail;
3225 
3226     errorExpTgt();
3227     return MatchOperand_Success;
3228   }
3229 
3230   return MatchOperand_NoMatch;
3231 }
3232 
3233 OperandMatchResultTy AMDGPUAsmParser::parseExpTgt(OperandVector &Operands) {
3234   uint8_t Val;
3235   StringRef Str = Parser.getTok().getString();
3236 
3237   auto Res = parseExpTgtImpl(Str, Val);
3238   if (Res != MatchOperand_Success)
3239     return Res;
3240 
3241   SMLoc S = Parser.getTok().getLoc();
3242   Parser.Lex();
3243 
3244   Operands.push_back(AMDGPUOperand::CreateImm(this, Val, S,
3245                                               AMDGPUOperand::ImmTyExpTgt));
3246   return MatchOperand_Success;
3247 }
3248 
3249 OperandMatchResultTy
3250 AMDGPUAsmParser::parseSendMsgOp(OperandVector &Operands) {
3251   using namespace llvm::AMDGPU::SendMsg;
3252 
3253   int64_t Imm16Val = 0;
3254   SMLoc S = Parser.getTok().getLoc();
3255 
3256   switch(getLexer().getKind()) {
3257   default:
3258     return MatchOperand_NoMatch;
3259   case AsmToken::Integer:
3260     // The operand can be an integer value.
3261     if (getParser().parseAbsoluteExpression(Imm16Val))
3262       return MatchOperand_NoMatch;
3263     if (Imm16Val < 0 || !isUInt<16>(Imm16Val)) {
3264       Error(S, "invalid immediate: only 16-bit values are legal");
3265       // Do not return error code, but create an imm operand anyway and proceed
3266       // to the next operand, if any. That avoids unneccessary error messages.
3267     }
3268     break;
3269   case AsmToken::Identifier: {
3270       OperandInfoTy Msg(ID_UNKNOWN_);
3271       OperandInfoTy Operation(OP_UNKNOWN_);
3272       int64_t StreamId = STREAM_ID_DEFAULT_;
3273       if (parseSendMsgConstruct(Msg, Operation, StreamId))
3274         return MatchOperand_ParseFail;
3275       do {
3276         // Validate and encode message ID.
3277         if (! ((ID_INTERRUPT <= Msg.Id && Msg.Id <= ID_GS_DONE)
3278                 || Msg.Id == ID_SYSMSG)) {
3279           if (Msg.IsSymbolic)
3280             Error(S, "invalid/unsupported symbolic name of message");
3281           else
3282             Error(S, "invalid/unsupported code of message");
3283           break;
3284         }
3285         Imm16Val = (Msg.Id << ID_SHIFT_);
3286         // Validate and encode operation ID.
3287         if (Msg.Id == ID_GS || Msg.Id == ID_GS_DONE) {
3288           if (! (OP_GS_FIRST_ <= Operation.Id && Operation.Id < OP_GS_LAST_)) {
3289             if (Operation.IsSymbolic)
3290               Error(S, "invalid symbolic name of GS_OP");
3291             else
3292               Error(S, "invalid code of GS_OP: only 2-bit values are legal");
3293             break;
3294           }
3295           if (Operation.Id == OP_GS_NOP
3296               && Msg.Id != ID_GS_DONE) {
3297             Error(S, "invalid GS_OP: NOP is for GS_DONE only");
3298             break;
3299           }
3300           Imm16Val |= (Operation.Id << OP_SHIFT_);
3301         }
3302         if (Msg.Id == ID_SYSMSG) {
3303           if (! (OP_SYS_FIRST_ <= Operation.Id && Operation.Id < OP_SYS_LAST_)) {
3304             if (Operation.IsSymbolic)
3305               Error(S, "invalid/unsupported symbolic name of SYSMSG_OP");
3306             else
3307               Error(S, "invalid/unsupported code of SYSMSG_OP");
3308             break;
3309           }
3310           Imm16Val |= (Operation.Id << OP_SHIFT_);
3311         }
3312         // Validate and encode stream ID.
3313         if ((Msg.Id == ID_GS || Msg.Id == ID_GS_DONE) && Operation.Id != OP_GS_NOP) {
3314           if (! (STREAM_ID_FIRST_ <= StreamId && StreamId < STREAM_ID_LAST_)) {
3315             Error(S, "invalid stream id: only 2-bit values are legal");
3316             break;
3317           }
3318           Imm16Val |= (StreamId << STREAM_ID_SHIFT_);
3319         }
3320       } while (false);
3321     }
3322     break;
3323   }
3324   Operands.push_back(AMDGPUOperand::CreateImm(this, Imm16Val, S, AMDGPUOperand::ImmTySendMsg));
3325   return MatchOperand_Success;
3326 }
3327 
3328 bool AMDGPUOperand::isSendMsg() const {
3329   return isImmTy(ImmTySendMsg);
3330 }
3331 
3332 //===----------------------------------------------------------------------===//
3333 // sopp branch targets
3334 //===----------------------------------------------------------------------===//
3335 
3336 OperandMatchResultTy
3337 AMDGPUAsmParser::parseSOppBrTarget(OperandVector &Operands) {
3338   SMLoc S = Parser.getTok().getLoc();
3339 
3340   switch (getLexer().getKind()) {
3341     default: return MatchOperand_ParseFail;
3342     case AsmToken::Integer: {
3343       int64_t Imm;
3344       if (getParser().parseAbsoluteExpression(Imm))
3345         return MatchOperand_ParseFail;
3346       Operands.push_back(AMDGPUOperand::CreateImm(this, Imm, S));
3347       return MatchOperand_Success;
3348     }
3349 
3350     case AsmToken::Identifier:
3351       Operands.push_back(AMDGPUOperand::CreateExpr(this,
3352           MCSymbolRefExpr::create(getContext().getOrCreateSymbol(
3353                                   Parser.getTok().getString()), getContext()), S));
3354       Parser.Lex();
3355       return MatchOperand_Success;
3356   }
3357 }
3358 
3359 //===----------------------------------------------------------------------===//
3360 // mubuf
3361 //===----------------------------------------------------------------------===//
3362 
3363 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultGLC() const {
3364   return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTyGLC);
3365 }
3366 
3367 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultSLC() const {
3368   return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTySLC);
3369 }
3370 
3371 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultTFE() const {
3372   return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTyTFE);
3373 }
3374 
3375 void AMDGPUAsmParser::cvtMubufImpl(MCInst &Inst,
3376                                const OperandVector &Operands,
3377                                bool IsAtomic, bool IsAtomicReturn) {
3378   OptionalImmIndexMap OptionalIdx;
3379   assert(IsAtomicReturn ? IsAtomic : true);
3380 
3381   for (unsigned i = 1, e = Operands.size(); i != e; ++i) {
3382     AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[i]);
3383 
3384     // Add the register arguments
3385     if (Op.isReg()) {
3386       Op.addRegOperands(Inst, 1);
3387       continue;
3388     }
3389 
3390     // Handle the case where soffset is an immediate
3391     if (Op.isImm() && Op.getImmTy() == AMDGPUOperand::ImmTyNone) {
3392       Op.addImmOperands(Inst, 1);
3393       continue;
3394     }
3395 
3396     // Handle tokens like 'offen' which are sometimes hard-coded into the
3397     // asm string.  There are no MCInst operands for these.
3398     if (Op.isToken()) {
3399       continue;
3400     }
3401     assert(Op.isImm());
3402 
3403     // Handle optional arguments
3404     OptionalIdx[Op.getImmTy()] = i;
3405   }
3406 
3407   // Copy $vdata_in operand and insert as $vdata for MUBUF_Atomic RTN insns.
3408   if (IsAtomicReturn) {
3409     MCInst::iterator I = Inst.begin(); // $vdata_in is always at the beginning.
3410     Inst.insert(I, *I);
3411   }
3412 
3413   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyOffset);
3414   if (!IsAtomic) { // glc is hard-coded.
3415     addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyGLC);
3416   }
3417   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySLC);
3418   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyTFE);
3419 }
3420 
3421 //===----------------------------------------------------------------------===//
3422 // mimg
3423 //===----------------------------------------------------------------------===//
3424 
3425 void AMDGPUAsmParser::cvtMIMG(MCInst &Inst, const OperandVector &Operands) {
3426   unsigned I = 1;
3427   const MCInstrDesc &Desc = MII.get(Inst.getOpcode());
3428   for (unsigned J = 0; J < Desc.getNumDefs(); ++J) {
3429     ((AMDGPUOperand &)*Operands[I++]).addRegOperands(Inst, 1);
3430   }
3431 
3432   OptionalImmIndexMap OptionalIdx;
3433 
3434   for (unsigned E = Operands.size(); I != E; ++I) {
3435     AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[I]);
3436 
3437     // Add the register arguments
3438     if (Op.isRegOrImm()) {
3439       Op.addRegOrImmOperands(Inst, 1);
3440       continue;
3441     } else if (Op.isImmModifier()) {
3442       OptionalIdx[Op.getImmTy()] = I;
3443     } else {
3444       llvm_unreachable("unexpected operand type");
3445     }
3446   }
3447 
3448   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyDMask);
3449   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyUNorm);
3450   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyGLC);
3451   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyDA);
3452   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyR128);
3453   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyTFE);
3454   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyLWE);
3455   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySLC);
3456 }
3457 
3458 void AMDGPUAsmParser::cvtMIMGAtomic(MCInst &Inst, const OperandVector &Operands) {
3459   unsigned I = 1;
3460   const MCInstrDesc &Desc = MII.get(Inst.getOpcode());
3461   for (unsigned J = 0; J < Desc.getNumDefs(); ++J) {
3462     ((AMDGPUOperand &)*Operands[I++]).addRegOperands(Inst, 1);
3463   }
3464 
3465   // Add src, same as dst
3466   ((AMDGPUOperand &)*Operands[I]).addRegOperands(Inst, 1);
3467 
3468   OptionalImmIndexMap OptionalIdx;
3469 
3470   for (unsigned E = Operands.size(); I != E; ++I) {
3471     AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[I]);
3472 
3473     // Add the register arguments
3474     if (Op.isRegOrImm()) {
3475       Op.addRegOrImmOperands(Inst, 1);
3476       continue;
3477     } else if (Op.isImmModifier()) {
3478       OptionalIdx[Op.getImmTy()] = I;
3479     } else {
3480       llvm_unreachable("unexpected operand type");
3481     }
3482   }
3483 
3484   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyDMask);
3485   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyUNorm);
3486   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyGLC);
3487   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyDA);
3488   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyR128);
3489   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyTFE);
3490   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyLWE);
3491   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySLC);
3492 }
3493 
3494 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultDMask() const {
3495   return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTyDMask);
3496 }
3497 
3498 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultUNorm() const {
3499   return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTyUNorm);
3500 }
3501 
3502 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultDA() const {
3503   return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTyDA);
3504 }
3505 
3506 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultR128() const {
3507   return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTyR128);
3508 }
3509 
3510 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultLWE() const {
3511   return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTyLWE);
3512 }
3513 
3514 //===----------------------------------------------------------------------===//
3515 // smrd
3516 //===----------------------------------------------------------------------===//
3517 
3518 bool AMDGPUOperand::isSMRDOffset8() const {
3519   return isImm() && isUInt<8>(getImm());
3520 }
3521 
3522 bool AMDGPUOperand::isSMRDOffset20() const {
3523   return isImm() && isUInt<20>(getImm());
3524 }
3525 
3526 bool AMDGPUOperand::isSMRDLiteralOffset() const {
3527   // 32-bit literals are only supported on CI and we only want to use them
3528   // when the offset is > 8-bits.
3529   return isImm() && !isUInt<8>(getImm()) && isUInt<32>(getImm());
3530 }
3531 
3532 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultSMRDOffset8() const {
3533   return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTyOffset);
3534 }
3535 
3536 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultSMRDOffset20() const {
3537   return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTyOffset);
3538 }
3539 
3540 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultSMRDLiteralOffset() const {
3541   return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTyOffset);
3542 }
3543 
3544 //===----------------------------------------------------------------------===//
3545 // vop3
3546 //===----------------------------------------------------------------------===//
3547 
3548 static bool ConvertOmodMul(int64_t &Mul) {
3549   if (Mul != 1 && Mul != 2 && Mul != 4)
3550     return false;
3551 
3552   Mul >>= 1;
3553   return true;
3554 }
3555 
3556 static bool ConvertOmodDiv(int64_t &Div) {
3557   if (Div == 1) {
3558     Div = 0;
3559     return true;
3560   }
3561 
3562   if (Div == 2) {
3563     Div = 3;
3564     return true;
3565   }
3566 
3567   return false;
3568 }
3569 
3570 static bool ConvertBoundCtrl(int64_t &BoundCtrl) {
3571   if (BoundCtrl == 0) {
3572     BoundCtrl = 1;
3573     return true;
3574   }
3575 
3576   if (BoundCtrl == -1) {
3577     BoundCtrl = 0;
3578     return true;
3579   }
3580 
3581   return false;
3582 }
3583 
3584 // Note: the order in this table matches the order of operands in AsmString.
3585 static const OptionalOperand AMDGPUOptionalOperandTable[] = {
3586   {"offen",   AMDGPUOperand::ImmTyOffen, true, nullptr},
3587   {"idxen",   AMDGPUOperand::ImmTyIdxen, true, nullptr},
3588   {"addr64",  AMDGPUOperand::ImmTyAddr64, true, nullptr},
3589   {"offset0", AMDGPUOperand::ImmTyOffset0, false, nullptr},
3590   {"offset1", AMDGPUOperand::ImmTyOffset1, false, nullptr},
3591   {"gds",     AMDGPUOperand::ImmTyGDS, true, nullptr},
3592   {"offset",  AMDGPUOperand::ImmTyOffset, false, nullptr},
3593   {"glc",     AMDGPUOperand::ImmTyGLC, true, nullptr},
3594   {"slc",     AMDGPUOperand::ImmTySLC, true, nullptr},
3595   {"tfe",     AMDGPUOperand::ImmTyTFE, true, nullptr},
3596   {"clamp",   AMDGPUOperand::ImmTyClampSI, true, nullptr},
3597   {"omod",    AMDGPUOperand::ImmTyOModSI, false, ConvertOmodMul},
3598   {"unorm",   AMDGPUOperand::ImmTyUNorm, true, nullptr},
3599   {"da",      AMDGPUOperand::ImmTyDA,    true, nullptr},
3600   {"r128",    AMDGPUOperand::ImmTyR128,  true, nullptr},
3601   {"lwe",     AMDGPUOperand::ImmTyLWE,   true, nullptr},
3602   {"dmask",   AMDGPUOperand::ImmTyDMask, false, nullptr},
3603   {"row_mask",   AMDGPUOperand::ImmTyDppRowMask, false, nullptr},
3604   {"bank_mask",  AMDGPUOperand::ImmTyDppBankMask, false, nullptr},
3605   {"bound_ctrl", AMDGPUOperand::ImmTyDppBoundCtrl, false, ConvertBoundCtrl},
3606   {"dst_sel",    AMDGPUOperand::ImmTySdwaDstSel, false, nullptr},
3607   {"src0_sel",   AMDGPUOperand::ImmTySdwaSrc0Sel, false, nullptr},
3608   {"src1_sel",   AMDGPUOperand::ImmTySdwaSrc1Sel, false, nullptr},
3609   {"dst_unused", AMDGPUOperand::ImmTySdwaDstUnused, false, nullptr},
3610   {"vm", AMDGPUOperand::ImmTyExpVM, true, nullptr},
3611   {"op_sel", AMDGPUOperand::ImmTyOpSel, false, nullptr},
3612   {"op_sel_hi", AMDGPUOperand::ImmTyOpSelHi, false, nullptr},
3613   {"neg_lo", AMDGPUOperand::ImmTyNegLo, false, nullptr},
3614   {"neg_hi", AMDGPUOperand::ImmTyNegHi, false, nullptr}
3615 };
3616 
3617 OperandMatchResultTy AMDGPUAsmParser::parseOptionalOperand(OperandVector &Operands) {
3618   OperandMatchResultTy res;
3619   for (const OptionalOperand &Op : AMDGPUOptionalOperandTable) {
3620     // try to parse any optional operand here
3621     if (Op.IsBit) {
3622       res = parseNamedBit(Op.Name, Operands, Op.Type);
3623     } else if (Op.Type == AMDGPUOperand::ImmTyOModSI) {
3624       res = parseOModOperand(Operands);
3625     } else if (Op.Type == AMDGPUOperand::ImmTySdwaDstSel ||
3626                Op.Type == AMDGPUOperand::ImmTySdwaSrc0Sel ||
3627                Op.Type == AMDGPUOperand::ImmTySdwaSrc1Sel) {
3628       res = parseSDWASel(Operands, Op.Name, Op.Type);
3629     } else if (Op.Type == AMDGPUOperand::ImmTySdwaDstUnused) {
3630       res = parseSDWADstUnused(Operands);
3631     } else if (Op.Type == AMDGPUOperand::ImmTyOpSel ||
3632                Op.Type == AMDGPUOperand::ImmTyOpSelHi ||
3633                Op.Type == AMDGPUOperand::ImmTyNegLo ||
3634                Op.Type == AMDGPUOperand::ImmTyNegHi) {
3635       res = parseOperandArrayWithPrefix(Op.Name, Operands, Op.Type,
3636                                         Op.ConvertResult);
3637     } else {
3638       res = parseIntWithPrefix(Op.Name, Operands, Op.Type, Op.ConvertResult);
3639     }
3640     if (res != MatchOperand_NoMatch) {
3641       return res;
3642     }
3643   }
3644   return MatchOperand_NoMatch;
3645 }
3646 
3647 OperandMatchResultTy AMDGPUAsmParser::parseOModOperand(OperandVector &Operands) {
3648   StringRef Name = Parser.getTok().getString();
3649   if (Name == "mul") {
3650     return parseIntWithPrefix("mul", Operands,
3651                               AMDGPUOperand::ImmTyOModSI, ConvertOmodMul);
3652   }
3653 
3654   if (Name == "div") {
3655     return parseIntWithPrefix("div", Operands,
3656                               AMDGPUOperand::ImmTyOModSI, ConvertOmodDiv);
3657   }
3658 
3659   return MatchOperand_NoMatch;
3660 }
3661 
3662 void AMDGPUAsmParser::cvtId(MCInst &Inst, const OperandVector &Operands) {
3663   unsigned I = 1;
3664   const MCInstrDesc &Desc = MII.get(Inst.getOpcode());
3665   for (unsigned J = 0; J < Desc.getNumDefs(); ++J) {
3666     ((AMDGPUOperand &)*Operands[I++]).addRegOperands(Inst, 1);
3667   }
3668   for (unsigned E = Operands.size(); I != E; ++I)
3669     ((AMDGPUOperand &)*Operands[I]).addRegOrImmOperands(Inst, 1);
3670 }
3671 
3672 void AMDGPUAsmParser::cvtVOP3_2_mod(MCInst &Inst, const OperandVector &Operands) {
3673   uint64_t TSFlags = MII.get(Inst.getOpcode()).TSFlags;
3674   if (TSFlags & SIInstrFlags::VOP3) {
3675     cvtVOP3(Inst, Operands);
3676   } else {
3677     cvtId(Inst, Operands);
3678   }
3679 }
3680 
3681 static bool isRegOrImmWithInputMods(const MCInstrDesc &Desc, unsigned OpNum) {
3682       // 1. This operand is input modifiers
3683   return Desc.OpInfo[OpNum].OperandType == AMDGPU::OPERAND_INPUT_MODS
3684       // 2. This is not last operand
3685       && Desc.NumOperands > (OpNum + 1)
3686       // 3. Next operand is register class
3687       && Desc.OpInfo[OpNum + 1].RegClass != -1
3688       // 4. Next register is not tied to any other operand
3689       && Desc.getOperandConstraint(OpNum + 1, MCOI::OperandConstraint::TIED_TO) == -1;
3690 }
3691 
3692 void AMDGPUAsmParser::cvtVOP3Impl(MCInst &Inst, const OperandVector &Operands,
3693                                   OptionalImmIndexMap &OptionalIdx) {
3694   unsigned I = 1;
3695   const MCInstrDesc &Desc = MII.get(Inst.getOpcode());
3696   for (unsigned J = 0; J < Desc.getNumDefs(); ++J) {
3697     ((AMDGPUOperand &)*Operands[I++]).addRegOperands(Inst, 1);
3698   }
3699 
3700   for (unsigned E = Operands.size(); I != E; ++I) {
3701     AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[I]);
3702     if (isRegOrImmWithInputMods(Desc, Inst.getNumOperands())) {
3703       Op.addRegOrImmWithFPInputModsOperands(Inst, 2);
3704     } else if (Op.isImm()) {
3705       OptionalIdx[Op.getImmTy()] = I;
3706     } else {
3707       llvm_unreachable("unhandled operand type");
3708     }
3709   }
3710 }
3711 
3712 void AMDGPUAsmParser::cvtVOP3(MCInst &Inst, const OperandVector &Operands) {
3713   OptionalImmIndexMap OptionalIdx;
3714 
3715   cvtVOP3Impl(Inst, Operands, OptionalIdx);
3716 
3717   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyClampSI);
3718   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyOModSI);
3719 
3720   // special case v_mac_{f16, f32}:
3721   // it has src2 register operand that is tied to dst operand
3722   // we don't allow modifiers for this operand in assembler so src2_modifiers
3723   // should be 0
3724   if (Inst.getOpcode() == AMDGPU::V_MAC_F32_e64_si ||
3725       Inst.getOpcode() == AMDGPU::V_MAC_F32_e64_vi ||
3726       Inst.getOpcode() == AMDGPU::V_MAC_F16_e64_vi) {
3727     auto it = Inst.begin();
3728     std::advance(
3729       it,
3730       AMDGPU::getNamedOperandIdx(Inst.getOpcode() == AMDGPU::V_MAC_F16_e64_vi ?
3731                                      AMDGPU::V_MAC_F16_e64 :
3732                                      AMDGPU::V_MAC_F32_e64,
3733                                  AMDGPU::OpName::src2_modifiers));
3734     it = Inst.insert(it, MCOperand::createImm(0)); // no modifiers for src2
3735     ++it;
3736     Inst.insert(it, Inst.getOperand(0)); // src2 = dst
3737   }
3738 }
3739 
3740 void AMDGPUAsmParser::cvtVOP3P(MCInst &Inst, const OperandVector &Operands) {
3741   OptionalImmIndexMap OptIdx;
3742 
3743   cvtVOP3Impl(Inst, Operands, OptIdx);
3744 
3745   // FIXME: This is messy. Parse the modifiers as if it was a normal VOP3
3746   // instruction, and then figure out where to actually put the modifiers
3747   int Opc = Inst.getOpcode();
3748 
3749   if (AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::clamp) != -1) {
3750     addOptionalImmOperand(Inst, Operands, OptIdx, AMDGPUOperand::ImmTyClampSI);
3751   }
3752 
3753   addOptionalImmOperand(Inst, Operands, OptIdx, AMDGPUOperand::ImmTyOpSel);
3754   addOptionalImmOperand(Inst, Operands, OptIdx, AMDGPUOperand::ImmTyOpSelHi, -1);
3755 
3756   int NegLoIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::neg_lo);
3757   if (NegLoIdx != -1) {
3758     addOptionalImmOperand(Inst, Operands, OptIdx, AMDGPUOperand::ImmTyNegLo);
3759     addOptionalImmOperand(Inst, Operands, OptIdx, AMDGPUOperand::ImmTyNegHi);
3760   }
3761 
3762   const int Ops[] = { AMDGPU::OpName::src0,
3763                       AMDGPU::OpName::src1,
3764                       AMDGPU::OpName::src2 };
3765   const int ModOps[] = { AMDGPU::OpName::src0_modifiers,
3766                          AMDGPU::OpName::src1_modifiers,
3767                          AMDGPU::OpName::src2_modifiers };
3768 
3769   int OpSelIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::op_sel);
3770   int OpSelHiIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::op_sel_hi);
3771 
3772   unsigned OpSel = Inst.getOperand(OpSelIdx).getImm();
3773   unsigned OpSelHi = Inst.getOperand(OpSelHiIdx).getImm();
3774   unsigned NegLo = 0;
3775   unsigned NegHi = 0;
3776 
3777   if (NegLoIdx != -1) {
3778     int NegHiIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::neg_hi);
3779     NegLo = Inst.getOperand(NegLoIdx).getImm();
3780     NegHi = Inst.getOperand(NegHiIdx).getImm();
3781   }
3782 
3783   for (int J = 0; J < 3; ++J) {
3784     int OpIdx = AMDGPU::getNamedOperandIdx(Opc, Ops[J]);
3785     if (OpIdx == -1)
3786       break;
3787 
3788     uint32_t ModVal = 0;
3789 
3790     if ((OpSel & (1 << J)) != 0)
3791       ModVal |= SISrcMods::OP_SEL_0;
3792 
3793     if ((OpSelHi & (1 << J)) != 0)
3794       ModVal |= SISrcMods::OP_SEL_1;
3795 
3796     if ((NegLo & (1 << J)) != 0)
3797       ModVal |= SISrcMods::NEG;
3798 
3799     if ((NegHi & (1 << J)) != 0)
3800       ModVal |= SISrcMods::NEG_HI;
3801 
3802     int ModIdx = AMDGPU::getNamedOperandIdx(Opc, ModOps[J]);
3803 
3804     Inst.getOperand(ModIdx).setImm(ModVal);
3805   }
3806 }
3807 
3808 //===----------------------------------------------------------------------===//
3809 // dpp
3810 //===----------------------------------------------------------------------===//
3811 
3812 bool AMDGPUOperand::isDPPCtrl() const {
3813   bool result = isImm() && getImmTy() == ImmTyDppCtrl && isUInt<9>(getImm());
3814   if (result) {
3815     int64_t Imm = getImm();
3816     return ((Imm >= 0x000) && (Imm <= 0x0ff)) ||
3817            ((Imm >= 0x101) && (Imm <= 0x10f)) ||
3818            ((Imm >= 0x111) && (Imm <= 0x11f)) ||
3819            ((Imm >= 0x121) && (Imm <= 0x12f)) ||
3820            (Imm == 0x130) ||
3821            (Imm == 0x134) ||
3822            (Imm == 0x138) ||
3823            (Imm == 0x13c) ||
3824            (Imm == 0x140) ||
3825            (Imm == 0x141) ||
3826            (Imm == 0x142) ||
3827            (Imm == 0x143);
3828   }
3829   return false;
3830 }
3831 
3832 bool AMDGPUOperand::isGPRIdxMode() const {
3833   return isImm() && isUInt<4>(getImm());
3834 }
3835 
3836 OperandMatchResultTy
3837 AMDGPUAsmParser::parseDPPCtrl(OperandVector &Operands) {
3838   SMLoc S = Parser.getTok().getLoc();
3839   StringRef Prefix;
3840   int64_t Int;
3841 
3842   if (getLexer().getKind() == AsmToken::Identifier) {
3843     Prefix = Parser.getTok().getString();
3844   } else {
3845     return MatchOperand_NoMatch;
3846   }
3847 
3848   if (Prefix == "row_mirror") {
3849     Int = 0x140;
3850     Parser.Lex();
3851   } else if (Prefix == "row_half_mirror") {
3852     Int = 0x141;
3853     Parser.Lex();
3854   } else {
3855     // Check to prevent parseDPPCtrlOps from eating invalid tokens
3856     if (Prefix != "quad_perm"
3857         && Prefix != "row_shl"
3858         && Prefix != "row_shr"
3859         && Prefix != "row_ror"
3860         && Prefix != "wave_shl"
3861         && Prefix != "wave_rol"
3862         && Prefix != "wave_shr"
3863         && Prefix != "wave_ror"
3864         && Prefix != "row_bcast") {
3865       return MatchOperand_NoMatch;
3866     }
3867 
3868     Parser.Lex();
3869     if (getLexer().isNot(AsmToken::Colon))
3870       return MatchOperand_ParseFail;
3871 
3872     if (Prefix == "quad_perm") {
3873       // quad_perm:[%d,%d,%d,%d]
3874       Parser.Lex();
3875       if (getLexer().isNot(AsmToken::LBrac))
3876         return MatchOperand_ParseFail;
3877       Parser.Lex();
3878 
3879       if (getParser().parseAbsoluteExpression(Int) || !(0 <= Int && Int <=3))
3880         return MatchOperand_ParseFail;
3881 
3882       for (int i = 0; i < 3; ++i) {
3883         if (getLexer().isNot(AsmToken::Comma))
3884           return MatchOperand_ParseFail;
3885         Parser.Lex();
3886 
3887         int64_t Temp;
3888         if (getParser().parseAbsoluteExpression(Temp) || !(0 <= Temp && Temp <=3))
3889           return MatchOperand_ParseFail;
3890         const int shift = i*2 + 2;
3891         Int += (Temp << shift);
3892       }
3893 
3894       if (getLexer().isNot(AsmToken::RBrac))
3895         return MatchOperand_ParseFail;
3896       Parser.Lex();
3897 
3898     } else {
3899       // sel:%d
3900       Parser.Lex();
3901       if (getParser().parseAbsoluteExpression(Int))
3902         return MatchOperand_ParseFail;
3903 
3904       if (Prefix == "row_shl" && 1 <= Int && Int <= 15) {
3905         Int |= 0x100;
3906       } else if (Prefix == "row_shr" && 1 <= Int && Int <= 15) {
3907         Int |= 0x110;
3908       } else if (Prefix == "row_ror" && 1 <= Int && Int <= 15) {
3909         Int |= 0x120;
3910       } else if (Prefix == "wave_shl" && 1 == Int) {
3911         Int = 0x130;
3912       } else if (Prefix == "wave_rol" && 1 == Int) {
3913         Int = 0x134;
3914       } else if (Prefix == "wave_shr" && 1 == Int) {
3915         Int = 0x138;
3916       } else if (Prefix == "wave_ror" && 1 == Int) {
3917         Int = 0x13C;
3918       } else if (Prefix == "row_bcast") {
3919         if (Int == 15) {
3920           Int = 0x142;
3921         } else if (Int == 31) {
3922           Int = 0x143;
3923         } else {
3924           return MatchOperand_ParseFail;
3925         }
3926       } else {
3927         return MatchOperand_ParseFail;
3928       }
3929     }
3930   }
3931 
3932   Operands.push_back(AMDGPUOperand::CreateImm(this, Int, S, AMDGPUOperand::ImmTyDppCtrl));
3933   return MatchOperand_Success;
3934 }
3935 
3936 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultRowMask() const {
3937   return AMDGPUOperand::CreateImm(this, 0xf, SMLoc(), AMDGPUOperand::ImmTyDppRowMask);
3938 }
3939 
3940 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultBankMask() const {
3941   return AMDGPUOperand::CreateImm(this, 0xf, SMLoc(), AMDGPUOperand::ImmTyDppBankMask);
3942 }
3943 
3944 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultBoundCtrl() const {
3945   return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTyDppBoundCtrl);
3946 }
3947 
3948 void AMDGPUAsmParser::cvtDPP(MCInst &Inst, const OperandVector &Operands) {
3949   OptionalImmIndexMap OptionalIdx;
3950 
3951   unsigned I = 1;
3952   const MCInstrDesc &Desc = MII.get(Inst.getOpcode());
3953   for (unsigned J = 0; J < Desc.getNumDefs(); ++J) {
3954     ((AMDGPUOperand &)*Operands[I++]).addRegOperands(Inst, 1);
3955   }
3956 
3957   for (unsigned E = Operands.size(); I != E; ++I) {
3958     AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[I]);
3959     // Add the register arguments
3960     if (Op.isReg() && Op.Reg.RegNo == AMDGPU::VCC) {
3961       // VOP2b (v_add_u32, v_sub_u32 ...) dpp use "vcc" token.
3962       // Skip it.
3963       continue;
3964     } if (isRegOrImmWithInputMods(Desc, Inst.getNumOperands())) {
3965       Op.addRegWithFPInputModsOperands(Inst, 2);
3966     } else if (Op.isDPPCtrl()) {
3967       Op.addImmOperands(Inst, 1);
3968     } else if (Op.isImm()) {
3969       // Handle optional arguments
3970       OptionalIdx[Op.getImmTy()] = I;
3971     } else {
3972       llvm_unreachable("Invalid operand type");
3973     }
3974   }
3975 
3976   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyDppRowMask, 0xf);
3977   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyDppBankMask, 0xf);
3978   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyDppBoundCtrl);
3979 
3980   // special case v_mac_{f16, f32}:
3981   // it has src2 register operand that is tied to dst operand
3982   if (Inst.getOpcode() == AMDGPU::V_MAC_F32_dpp ||
3983       Inst.getOpcode() == AMDGPU::V_MAC_F16_dpp) {
3984     auto it = Inst.begin();
3985     std::advance(
3986         it, AMDGPU::getNamedOperandIdx(Inst.getOpcode(), AMDGPU::OpName::src2));
3987     Inst.insert(it, Inst.getOperand(0)); // src2 = dst
3988   }
3989 }
3990 
3991 //===----------------------------------------------------------------------===//
3992 // sdwa
3993 //===----------------------------------------------------------------------===//
3994 
3995 OperandMatchResultTy
3996 AMDGPUAsmParser::parseSDWASel(OperandVector &Operands, StringRef Prefix,
3997                               AMDGPUOperand::ImmTy Type) {
3998   using namespace llvm::AMDGPU::SDWA;
3999 
4000   SMLoc S = Parser.getTok().getLoc();
4001   StringRef Value;
4002   OperandMatchResultTy res;
4003 
4004   res = parseStringWithPrefix(Prefix, Value);
4005   if (res != MatchOperand_Success) {
4006     return res;
4007   }
4008 
4009   int64_t Int;
4010   Int = StringSwitch<int64_t>(Value)
4011         .Case("BYTE_0", SdwaSel::BYTE_0)
4012         .Case("BYTE_1", SdwaSel::BYTE_1)
4013         .Case("BYTE_2", SdwaSel::BYTE_2)
4014         .Case("BYTE_3", SdwaSel::BYTE_3)
4015         .Case("WORD_0", SdwaSel::WORD_0)
4016         .Case("WORD_1", SdwaSel::WORD_1)
4017         .Case("DWORD", SdwaSel::DWORD)
4018         .Default(0xffffffff);
4019   Parser.Lex(); // eat last token
4020 
4021   if (Int == 0xffffffff) {
4022     return MatchOperand_ParseFail;
4023   }
4024 
4025   Operands.push_back(AMDGPUOperand::CreateImm(this, Int, S, Type));
4026   return MatchOperand_Success;
4027 }
4028 
4029 OperandMatchResultTy
4030 AMDGPUAsmParser::parseSDWADstUnused(OperandVector &Operands) {
4031   using namespace llvm::AMDGPU::SDWA;
4032 
4033   SMLoc S = Parser.getTok().getLoc();
4034   StringRef Value;
4035   OperandMatchResultTy res;
4036 
4037   res = parseStringWithPrefix("dst_unused", Value);
4038   if (res != MatchOperand_Success) {
4039     return res;
4040   }
4041 
4042   int64_t Int;
4043   Int = StringSwitch<int64_t>(Value)
4044         .Case("UNUSED_PAD", DstUnused::UNUSED_PAD)
4045         .Case("UNUSED_SEXT", DstUnused::UNUSED_SEXT)
4046         .Case("UNUSED_PRESERVE", DstUnused::UNUSED_PRESERVE)
4047         .Default(0xffffffff);
4048   Parser.Lex(); // eat last token
4049 
4050   if (Int == 0xffffffff) {
4051     return MatchOperand_ParseFail;
4052   }
4053 
4054   Operands.push_back(AMDGPUOperand::CreateImm(this, Int, S, AMDGPUOperand::ImmTySdwaDstUnused));
4055   return MatchOperand_Success;
4056 }
4057 
4058 void AMDGPUAsmParser::cvtSdwaVOP1(MCInst &Inst, const OperandVector &Operands) {
4059   cvtSDWA(Inst, Operands, SIInstrFlags::VOP1);
4060 }
4061 
4062 void AMDGPUAsmParser::cvtSdwaVOP2(MCInst &Inst, const OperandVector &Operands) {
4063   cvtSDWA(Inst, Operands, SIInstrFlags::VOP2);
4064 }
4065 
4066 void AMDGPUAsmParser::cvtSdwaVOPC(MCInst &Inst, const OperandVector &Operands) {
4067   cvtSDWA(Inst, Operands, SIInstrFlags::VOPC);
4068 }
4069 
4070 void AMDGPUAsmParser::cvtSDWA(MCInst &Inst, const OperandVector &Operands,
4071                               uint64_t BasicInstType) {
4072   using namespace llvm::AMDGPU::SDWA;
4073   OptionalImmIndexMap OptionalIdx;
4074 
4075   unsigned I = 1;
4076   const MCInstrDesc &Desc = MII.get(Inst.getOpcode());
4077   for (unsigned J = 0; J < Desc.getNumDefs(); ++J) {
4078     ((AMDGPUOperand &)*Operands[I++]).addRegOperands(Inst, 1);
4079   }
4080 
4081   for (unsigned E = Operands.size(); I != E; ++I) {
4082     AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[I]);
4083     // Add the register arguments
4084     if ((BasicInstType == SIInstrFlags::VOPC ||
4085          BasicInstType == SIInstrFlags::VOP2)&&
4086         Op.isReg() &&
4087         Op.Reg.RegNo == AMDGPU::VCC) {
4088       // VOPC and VOP2b (v_add_u32, v_sub_u32 ...) sdwa use "vcc" token as dst.
4089       // Skip it.
4090       continue;
4091     } else if (isRegOrImmWithInputMods(Desc, Inst.getNumOperands())) {
4092       Op.addRegWithInputModsOperands(Inst, 2);
4093     } else if (Op.isImm()) {
4094       // Handle optional arguments
4095       OptionalIdx[Op.getImmTy()] = I;
4096     } else {
4097       llvm_unreachable("Invalid operand type");
4098     }
4099   }
4100 
4101   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyClampSI, 0);
4102 
4103   if (Inst.getOpcode() != AMDGPU::V_NOP_sdwa_vi) {
4104     // V_NOP_sdwa_vi has no optional sdwa arguments
4105     switch (BasicInstType) {
4106     case SIInstrFlags::VOP1:
4107       addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySdwaDstSel, SdwaSel::DWORD);
4108       addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySdwaDstUnused, DstUnused::UNUSED_PRESERVE);
4109       addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySdwaSrc0Sel, SdwaSel::DWORD);
4110       break;
4111 
4112     case SIInstrFlags::VOP2:
4113       addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySdwaDstSel, SdwaSel::DWORD);
4114       addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySdwaDstUnused, DstUnused::UNUSED_PRESERVE);
4115       addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySdwaSrc0Sel, SdwaSel::DWORD);
4116       addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySdwaSrc1Sel, SdwaSel::DWORD);
4117       break;
4118 
4119     case SIInstrFlags::VOPC:
4120       addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySdwaSrc0Sel, SdwaSel::DWORD);
4121       addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySdwaSrc1Sel, SdwaSel::DWORD);
4122       break;
4123 
4124     default:
4125       llvm_unreachable("Invalid instruction type. Only VOP1, VOP2 and VOPC allowed");
4126     }
4127   }
4128 
4129   // special case v_mac_{f16, f32}:
4130   // it has src2 register operand that is tied to dst operand
4131   if (Inst.getOpcode() == AMDGPU::V_MAC_F32_sdwa_vi ||
4132       Inst.getOpcode() == AMDGPU::V_MAC_F16_sdwa_vi)  {
4133     auto it = Inst.begin();
4134     std::advance(
4135         it, AMDGPU::getNamedOperandIdx(Inst.getOpcode(), AMDGPU::OpName::src2));
4136     Inst.insert(it, Inst.getOperand(0)); // src2 = dst
4137   }
4138 
4139 }
4140 
4141 /// Force static initialization.
4142 extern "C" void LLVMInitializeAMDGPUAsmParser() {
4143   RegisterMCAsmParser<AMDGPUAsmParser> A(getTheAMDGPUTarget());
4144   RegisterMCAsmParser<AMDGPUAsmParser> B(getTheGCNTarget());
4145 }
4146 
4147 #define GET_REGISTER_MATCHER
4148 #define GET_MATCHER_IMPLEMENTATION
4149 #include "AMDGPUGenAsmMatcher.inc"
4150 
4151 // This fuction should be defined after auto-generated include so that we have
4152 // MatchClassKind enum defined
4153 unsigned AMDGPUAsmParser::validateTargetOperandClass(MCParsedAsmOperand &Op,
4154                                                      unsigned Kind) {
4155   // Tokens like "glc" would be parsed as immediate operands in ParseOperand().
4156   // But MatchInstructionImpl() expects to meet token and fails to validate
4157   // operand. This method checks if we are given immediate operand but expect to
4158   // get corresponding token.
4159   AMDGPUOperand &Operand = (AMDGPUOperand&)Op;
4160   switch (Kind) {
4161   case MCK_addr64:
4162     return Operand.isAddr64() ? Match_Success : Match_InvalidOperand;
4163   case MCK_gds:
4164     return Operand.isGDS() ? Match_Success : Match_InvalidOperand;
4165   case MCK_glc:
4166     return Operand.isGLC() ? Match_Success : Match_InvalidOperand;
4167   case MCK_idxen:
4168     return Operand.isIdxen() ? Match_Success : Match_InvalidOperand;
4169   case MCK_offen:
4170     return Operand.isOffen() ? Match_Success : Match_InvalidOperand;
4171   case MCK_SSrcB32:
4172     // When operands have expression values, they will return true for isToken,
4173     // because it is not possible to distinguish between a token and an
4174     // expression at parse time. MatchInstructionImpl() will always try to
4175     // match an operand as a token, when isToken returns true, and when the
4176     // name of the expression is not a valid token, the match will fail,
4177     // so we need to handle it here.
4178     return Operand.isSSrcB32() ? Match_Success : Match_InvalidOperand;
4179   case MCK_SSrcF32:
4180     return Operand.isSSrcF32() ? Match_Success : Match_InvalidOperand;
4181   case MCK_SoppBrTarget:
4182     return Operand.isSoppBrTarget() ? Match_Success : Match_InvalidOperand;
4183   case MCK_VReg32OrOff:
4184     return Operand.isVReg32OrOff() ? Match_Success : Match_InvalidOperand;
4185   case MCK_InterpSlot:
4186     return Operand.isInterpSlot() ? Match_Success : Match_InvalidOperand;
4187   case MCK_Attr:
4188     return Operand.isInterpAttr() ? Match_Success : Match_InvalidOperand;
4189   case MCK_AttrChan:
4190     return Operand.isAttrChan() ? Match_Success : Match_InvalidOperand;
4191   default:
4192     return Match_InvalidOperand;
4193   }
4194 }
4195