1 //===- AMDGPUAsmParser.cpp - Parse SI asm to MCInst instructions ----------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 
9 #include "AMDKernelCodeT.h"
10 #include "MCTargetDesc/AMDGPUMCTargetDesc.h"
11 #include "MCTargetDesc/AMDGPUTargetStreamer.h"
12 #include "SIDefines.h"
13 #include "SIInstrInfo.h"
14 #include "SIRegisterInfo.h"
15 #include "TargetInfo/AMDGPUTargetInfo.h"
16 #include "Utils/AMDGPUAsmUtils.h"
17 #include "Utils/AMDGPUBaseInfo.h"
18 #include "Utils/AMDKernelCodeTUtils.h"
19 #include "llvm/ADT/APFloat.h"
20 #include "llvm/ADT/SmallBitVector.h"
21 #include "llvm/ADT/StringSet.h"
22 #include "llvm/ADT/Twine.h"
23 #include "llvm/BinaryFormat/ELF.h"
24 #include "llvm/MC/MCAsmInfo.h"
25 #include "llvm/MC/MCContext.h"
26 #include "llvm/MC/MCExpr.h"
27 #include "llvm/MC/MCInst.h"
28 #include "llvm/MC/MCInstrDesc.h"
29 #include "llvm/MC/MCParser/MCAsmLexer.h"
30 #include "llvm/MC/MCParser/MCAsmParser.h"
31 #include "llvm/MC/MCParser/MCParsedAsmOperand.h"
32 #include "llvm/MC/MCParser/MCTargetAsmParser.h"
33 #include "llvm/MC/MCSymbol.h"
34 #include "llvm/MC/TargetRegistry.h"
35 #include "llvm/Support/AMDGPUMetadata.h"
36 #include "llvm/Support/AMDHSAKernelDescriptor.h"
37 #include "llvm/Support/Casting.h"
38 #include "llvm/Support/MachineValueType.h"
39 #include "llvm/Support/MathExtras.h"
40 #include "llvm/Support/TargetParser.h"
41 
42 using namespace llvm;
43 using namespace llvm::AMDGPU;
44 using namespace llvm::amdhsa;
45 
46 namespace {
47 
48 class AMDGPUAsmParser;
49 
50 enum RegisterKind { IS_UNKNOWN, IS_VGPR, IS_SGPR, IS_AGPR, IS_TTMP, IS_SPECIAL };
51 
52 //===----------------------------------------------------------------------===//
53 // Operand
54 //===----------------------------------------------------------------------===//
55 
56 class AMDGPUOperand : public MCParsedAsmOperand {
57   enum KindTy {
58     Token,
59     Immediate,
60     Register,
61     Expression
62   } Kind;
63 
64   SMLoc StartLoc, EndLoc;
65   const AMDGPUAsmParser *AsmParser;
66 
67 public:
68   AMDGPUOperand(KindTy Kind_, const AMDGPUAsmParser *AsmParser_)
69       : Kind(Kind_), AsmParser(AsmParser_) {}
70 
71   using Ptr = std::unique_ptr<AMDGPUOperand>;
72 
73   struct Modifiers {
74     bool Abs = false;
75     bool Neg = false;
76     bool Sext = false;
77 
78     bool hasFPModifiers() const { return Abs || Neg; }
79     bool hasIntModifiers() const { return Sext; }
80     bool hasModifiers() const { return hasFPModifiers() || hasIntModifiers(); }
81 
82     int64_t getFPModifiersOperand() const {
83       int64_t Operand = 0;
84       Operand |= Abs ? SISrcMods::ABS : 0u;
85       Operand |= Neg ? SISrcMods::NEG : 0u;
86       return Operand;
87     }
88 
89     int64_t getIntModifiersOperand() const {
90       int64_t Operand = 0;
91       Operand |= Sext ? SISrcMods::SEXT : 0u;
92       return Operand;
93     }
94 
95     int64_t getModifiersOperand() const {
96       assert(!(hasFPModifiers() && hasIntModifiers())
97            && "fp and int modifiers should not be used simultaneously");
98       if (hasFPModifiers()) {
99         return getFPModifiersOperand();
100       } else if (hasIntModifiers()) {
101         return getIntModifiersOperand();
102       } else {
103         return 0;
104       }
105     }
106 
107     friend raw_ostream &operator <<(raw_ostream &OS, AMDGPUOperand::Modifiers Mods);
108   };
109 
110   enum ImmTy {
111     ImmTyNone,
112     ImmTyGDS,
113     ImmTyLDS,
114     ImmTyOffen,
115     ImmTyIdxen,
116     ImmTyAddr64,
117     ImmTyOffset,
118     ImmTyInstOffset,
119     ImmTyOffset0,
120     ImmTyOffset1,
121     ImmTyCPol,
122     ImmTySWZ,
123     ImmTyTFE,
124     ImmTyD16,
125     ImmTyClampSI,
126     ImmTyOModSI,
127     ImmTySdwaDstSel,
128     ImmTySdwaSrc0Sel,
129     ImmTySdwaSrc1Sel,
130     ImmTySdwaDstUnused,
131     ImmTyDMask,
132     ImmTyDim,
133     ImmTyUNorm,
134     ImmTyDA,
135     ImmTyR128A16,
136     ImmTyA16,
137     ImmTyLWE,
138     ImmTyExpTgt,
139     ImmTyExpCompr,
140     ImmTyExpVM,
141     ImmTyFORMAT,
142     ImmTyHwreg,
143     ImmTyOff,
144     ImmTySendMsg,
145     ImmTyInterpSlot,
146     ImmTyInterpAttr,
147     ImmTyAttrChan,
148     ImmTyOpSel,
149     ImmTyOpSelHi,
150     ImmTyNegLo,
151     ImmTyNegHi,
152     ImmTyDPP8,
153     ImmTyDppCtrl,
154     ImmTyDppRowMask,
155     ImmTyDppBankMask,
156     ImmTyDppBoundCtrl,
157     ImmTyDppFi,
158     ImmTySwizzle,
159     ImmTyGprIdxMode,
160     ImmTyHigh,
161     ImmTyBLGP,
162     ImmTyCBSZ,
163     ImmTyABID,
164     ImmTyEndpgm,
165     ImmTyWaitVDST,
166     ImmTyWaitEXP,
167   };
168 
169   enum ImmKindTy {
170     ImmKindTyNone,
171     ImmKindTyLiteral,
172     ImmKindTyConst,
173   };
174 
175 private:
176   struct TokOp {
177     const char *Data;
178     unsigned Length;
179   };
180 
181   struct ImmOp {
182     int64_t Val;
183     ImmTy Type;
184     bool IsFPImm;
185     mutable ImmKindTy Kind;
186     Modifiers Mods;
187   };
188 
189   struct RegOp {
190     unsigned RegNo;
191     Modifiers Mods;
192   };
193 
194   union {
195     TokOp Tok;
196     ImmOp Imm;
197     RegOp Reg;
198     const MCExpr *Expr;
199   };
200 
201 public:
202   bool isToken() const override {
203     if (Kind == Token)
204       return true;
205 
206     // When parsing operands, we can't always tell if something was meant to be
207     // a token, like 'gds', or an expression that references a global variable.
208     // In this case, we assume the string is an expression, and if we need to
209     // interpret is a token, then we treat the symbol name as the token.
210     return isSymbolRefExpr();
211   }
212 
213   bool isSymbolRefExpr() const {
214     return isExpr() && Expr && isa<MCSymbolRefExpr>(Expr);
215   }
216 
217   bool isImm() const override {
218     return Kind == Immediate;
219   }
220 
221   void setImmKindNone() const {
222     assert(isImm());
223     Imm.Kind = ImmKindTyNone;
224   }
225 
226   void setImmKindLiteral() const {
227     assert(isImm());
228     Imm.Kind = ImmKindTyLiteral;
229   }
230 
231   void setImmKindConst() const {
232     assert(isImm());
233     Imm.Kind = ImmKindTyConst;
234   }
235 
236   bool IsImmKindLiteral() const {
237     return isImm() && Imm.Kind == ImmKindTyLiteral;
238   }
239 
240   bool isImmKindConst() const {
241     return isImm() && Imm.Kind == ImmKindTyConst;
242   }
243 
244   bool isInlinableImm(MVT type) const;
245   bool isLiteralImm(MVT type) const;
246 
247   bool isRegKind() const {
248     return Kind == Register;
249   }
250 
251   bool isReg() const override {
252     return isRegKind() && !hasModifiers();
253   }
254 
255   bool isRegOrInline(unsigned RCID, MVT type) const {
256     return isRegClass(RCID) || isInlinableImm(type);
257   }
258 
259   bool isRegOrImmWithInputMods(unsigned RCID, MVT type) const {
260     return isRegOrInline(RCID, type) || isLiteralImm(type);
261   }
262 
263   bool isRegOrImmWithInt16InputMods() const {
264     return isRegOrImmWithInputMods(AMDGPU::VS_32RegClassID, MVT::i16);
265   }
266 
267   bool isRegOrImmWithInt32InputMods() const {
268     return isRegOrImmWithInputMods(AMDGPU::VS_32RegClassID, MVT::i32);
269   }
270 
271   bool isRegOrInlineImmWithInt16InputMods() const {
272     return isRegOrInline(AMDGPU::VS_32RegClassID, MVT::i16);
273   }
274 
275   bool isRegOrInlineImmWithInt32InputMods() const {
276     return isRegOrInline(AMDGPU::VS_32RegClassID, MVT::i32);
277   }
278 
279   bool isRegOrImmWithInt64InputMods() const {
280     return isRegOrImmWithInputMods(AMDGPU::VS_64RegClassID, MVT::i64);
281   }
282 
283   bool isRegOrImmWithFP16InputMods() const {
284     return isRegOrImmWithInputMods(AMDGPU::VS_32RegClassID, MVT::f16);
285   }
286 
287   bool isRegOrImmWithFP32InputMods() const {
288     return isRegOrImmWithInputMods(AMDGPU::VS_32RegClassID, MVT::f32);
289   }
290 
291   bool isRegOrImmWithFP64InputMods() const {
292     return isRegOrImmWithInputMods(AMDGPU::VS_64RegClassID, MVT::f64);
293   }
294 
295   bool isRegOrInlineImmWithFP16InputMods() const {
296     return isRegOrInline(AMDGPU::VS_32RegClassID, MVT::f16);
297   }
298 
299   bool isRegOrInlineImmWithFP32InputMods() const {
300     return isRegOrInline(AMDGPU::VS_32RegClassID, MVT::f32);
301   }
302 
303 
304   bool isVReg() const {
305     return isRegClass(AMDGPU::VGPR_32RegClassID) ||
306            isRegClass(AMDGPU::VReg_64RegClassID) ||
307            isRegClass(AMDGPU::VReg_96RegClassID) ||
308            isRegClass(AMDGPU::VReg_128RegClassID) ||
309            isRegClass(AMDGPU::VReg_160RegClassID) ||
310            isRegClass(AMDGPU::VReg_192RegClassID) ||
311            isRegClass(AMDGPU::VReg_256RegClassID) ||
312            isRegClass(AMDGPU::VReg_512RegClassID) ||
313            isRegClass(AMDGPU::VReg_1024RegClassID);
314   }
315 
316   bool isVReg32() const {
317     return isRegClass(AMDGPU::VGPR_32RegClassID);
318   }
319 
320   bool isVReg32OrOff() const {
321     return isOff() || isVReg32();
322   }
323 
324   bool isNull() const {
325     return isRegKind() && getReg() == AMDGPU::SGPR_NULL;
326   }
327 
328   bool isVRegWithInputMods() const;
329 
330   bool isSDWAOperand(MVT type) const;
331   bool isSDWAFP16Operand() const;
332   bool isSDWAFP32Operand() const;
333   bool isSDWAInt16Operand() const;
334   bool isSDWAInt32Operand() const;
335 
336   bool isImmTy(ImmTy ImmT) const {
337     return isImm() && Imm.Type == ImmT;
338   }
339 
340   bool isImmModifier() const {
341     return isImm() && Imm.Type != ImmTyNone;
342   }
343 
344   bool isClampSI() const { return isImmTy(ImmTyClampSI); }
345   bool isOModSI() const { return isImmTy(ImmTyOModSI); }
346   bool isDMask() const { return isImmTy(ImmTyDMask); }
347   bool isDim() const { return isImmTy(ImmTyDim); }
348   bool isUNorm() const { return isImmTy(ImmTyUNorm); }
349   bool isDA() const { return isImmTy(ImmTyDA); }
350   bool isR128A16() const { return isImmTy(ImmTyR128A16); }
351   bool isGFX10A16() const { return isImmTy(ImmTyA16); }
352   bool isLWE() const { return isImmTy(ImmTyLWE); }
353   bool isOff() const { return isImmTy(ImmTyOff); }
354   bool isExpTgt() const { return isImmTy(ImmTyExpTgt); }
355   bool isExpVM() const { return isImmTy(ImmTyExpVM); }
356   bool isExpCompr() const { return isImmTy(ImmTyExpCompr); }
357   bool isOffen() const { return isImmTy(ImmTyOffen); }
358   bool isIdxen() const { return isImmTy(ImmTyIdxen); }
359   bool isAddr64() const { return isImmTy(ImmTyAddr64); }
360   bool isOffset() const { return isImmTy(ImmTyOffset) && isUInt<16>(getImm()); }
361   bool isOffset0() const { return isImmTy(ImmTyOffset0) && isUInt<8>(getImm()); }
362   bool isOffset1() const { return isImmTy(ImmTyOffset1) && isUInt<8>(getImm()); }
363 
364   bool isFlatOffset() const { return isImmTy(ImmTyOffset) || isImmTy(ImmTyInstOffset); }
365   bool isGDS() const { return isImmTy(ImmTyGDS); }
366   bool isLDS() const { return isImmTy(ImmTyLDS); }
367   bool isCPol() const { return isImmTy(ImmTyCPol); }
368   bool isSWZ() const { return isImmTy(ImmTySWZ); }
369   bool isTFE() const { return isImmTy(ImmTyTFE); }
370   bool isD16() const { return isImmTy(ImmTyD16); }
371   bool isFORMAT() const { return isImmTy(ImmTyFORMAT) && isUInt<7>(getImm()); }
372   bool isBankMask() const { return isImmTy(ImmTyDppBankMask); }
373   bool isRowMask() const { return isImmTy(ImmTyDppRowMask); }
374   bool isBoundCtrl() const { return isImmTy(ImmTyDppBoundCtrl); }
375   bool isFI() const { return isImmTy(ImmTyDppFi); }
376   bool isSDWADstSel() const { return isImmTy(ImmTySdwaDstSel); }
377   bool isSDWASrc0Sel() const { return isImmTy(ImmTySdwaSrc0Sel); }
378   bool isSDWASrc1Sel() const { return isImmTy(ImmTySdwaSrc1Sel); }
379   bool isSDWADstUnused() const { return isImmTy(ImmTySdwaDstUnused); }
380   bool isInterpSlot() const { return isImmTy(ImmTyInterpSlot); }
381   bool isInterpAttr() const { return isImmTy(ImmTyInterpAttr); }
382   bool isAttrChan() const { return isImmTy(ImmTyAttrChan); }
383   bool isOpSel() const { return isImmTy(ImmTyOpSel); }
384   bool isOpSelHi() const { return isImmTy(ImmTyOpSelHi); }
385   bool isNegLo() const { return isImmTy(ImmTyNegLo); }
386   bool isNegHi() const { return isImmTy(ImmTyNegHi); }
387   bool isHigh() const { return isImmTy(ImmTyHigh); }
388 
389   bool isMod() const {
390     return isClampSI() || isOModSI();
391   }
392 
393   bool isRegOrImm() const {
394     return isReg() || isImm();
395   }
396 
397   bool isRegClass(unsigned RCID) const;
398 
399   bool isInlineValue() const;
400 
401   bool isRegOrInlineNoMods(unsigned RCID, MVT type) const {
402     return isRegOrInline(RCID, type) && !hasModifiers();
403   }
404 
405   bool isSCSrcB16() const {
406     return isRegOrInlineNoMods(AMDGPU::SReg_32RegClassID, MVT::i16);
407   }
408 
409   bool isSCSrcV2B16() const {
410     return isSCSrcB16();
411   }
412 
413   bool isSCSrcB32() const {
414     return isRegOrInlineNoMods(AMDGPU::SReg_32RegClassID, MVT::i32);
415   }
416 
417   bool isSCSrcB64() const {
418     return isRegOrInlineNoMods(AMDGPU::SReg_64RegClassID, MVT::i64);
419   }
420 
421   bool isBoolReg() const;
422 
423   bool isSCSrcF16() const {
424     return isRegOrInlineNoMods(AMDGPU::SReg_32RegClassID, MVT::f16);
425   }
426 
427   bool isSCSrcV2F16() const {
428     return isSCSrcF16();
429   }
430 
431   bool isSCSrcF32() const {
432     return isRegOrInlineNoMods(AMDGPU::SReg_32RegClassID, MVT::f32);
433   }
434 
435   bool isSCSrcF64() const {
436     return isRegOrInlineNoMods(AMDGPU::SReg_64RegClassID, MVT::f64);
437   }
438 
439   bool isSSrcB32() const {
440     return isSCSrcB32() || isLiteralImm(MVT::i32) || isExpr();
441   }
442 
443   bool isSSrcB16() const {
444     return isSCSrcB16() || isLiteralImm(MVT::i16);
445   }
446 
447   bool isSSrcV2B16() const {
448     llvm_unreachable("cannot happen");
449     return isSSrcB16();
450   }
451 
452   bool isSSrcB64() const {
453     // TODO: Find out how SALU supports extension of 32-bit literals to 64 bits.
454     // See isVSrc64().
455     return isSCSrcB64() || isLiteralImm(MVT::i64);
456   }
457 
458   bool isSSrcF32() const {
459     return isSCSrcB32() || isLiteralImm(MVT::f32) || isExpr();
460   }
461 
462   bool isSSrcF64() const {
463     return isSCSrcB64() || isLiteralImm(MVT::f64);
464   }
465 
466   bool isSSrcF16() const {
467     return isSCSrcB16() || isLiteralImm(MVT::f16);
468   }
469 
470   bool isSSrcV2F16() const {
471     llvm_unreachable("cannot happen");
472     return isSSrcF16();
473   }
474 
475   bool isSSrcV2FP32() const {
476     llvm_unreachable("cannot happen");
477     return isSSrcF32();
478   }
479 
480   bool isSCSrcV2FP32() const {
481     llvm_unreachable("cannot happen");
482     return isSCSrcF32();
483   }
484 
485   bool isSSrcV2INT32() const {
486     llvm_unreachable("cannot happen");
487     return isSSrcB32();
488   }
489 
490   bool isSCSrcV2INT32() const {
491     llvm_unreachable("cannot happen");
492     return isSCSrcB32();
493   }
494 
495   bool isSSrcOrLdsB32() const {
496     return isRegOrInlineNoMods(AMDGPU::SRegOrLds_32RegClassID, MVT::i32) ||
497            isLiteralImm(MVT::i32) || isExpr();
498   }
499 
500   bool isVCSrcB32() const {
501     return isRegOrInlineNoMods(AMDGPU::VS_32RegClassID, MVT::i32);
502   }
503 
504   bool isVCSrcB64() const {
505     return isRegOrInlineNoMods(AMDGPU::VS_64RegClassID, MVT::i64);
506   }
507 
508   bool isVCSrcB16() const {
509     return isRegOrInlineNoMods(AMDGPU::VS_32RegClassID, MVT::i16);
510   }
511 
512   bool isVCSrcV2B16() const {
513     return isVCSrcB16();
514   }
515 
516   bool isVCSrcF32() const {
517     return isRegOrInlineNoMods(AMDGPU::VS_32RegClassID, MVT::f32);
518   }
519 
520   bool isVCSrcF64() const {
521     return isRegOrInlineNoMods(AMDGPU::VS_64RegClassID, MVT::f64);
522   }
523 
524   bool isVCSrcF16() const {
525     return isRegOrInlineNoMods(AMDGPU::VS_32RegClassID, MVT::f16);
526   }
527 
528   bool isVCSrcV2F16() const {
529     return isVCSrcF16();
530   }
531 
532   bool isVSrcB32() const {
533     return isVCSrcF32() || isLiteralImm(MVT::i32) || isExpr();
534   }
535 
536   bool isVSrcB64() const {
537     return isVCSrcF64() || isLiteralImm(MVT::i64);
538   }
539 
540   bool isVSrcB16() const {
541     return isVCSrcB16() || isLiteralImm(MVT::i16);
542   }
543 
544   bool isVSrcV2B16() const {
545     return isVSrcB16() || isLiteralImm(MVT::v2i16);
546   }
547 
548   bool isVCSrcV2FP32() const {
549     return isVCSrcF64();
550   }
551 
552   bool isVSrcV2FP32() const {
553     return isVSrcF64() || isLiteralImm(MVT::v2f32);
554   }
555 
556   bool isVCSrcV2INT32() const {
557     return isVCSrcB64();
558   }
559 
560   bool isVSrcV2INT32() const {
561     return isVSrcB64() || isLiteralImm(MVT::v2i32);
562   }
563 
564   bool isVSrcF32() const {
565     return isVCSrcF32() || isLiteralImm(MVT::f32) || isExpr();
566   }
567 
568   bool isVSrcF64() const {
569     return isVCSrcF64() || isLiteralImm(MVT::f64);
570   }
571 
572   bool isVSrcF16() const {
573     return isVCSrcF16() || isLiteralImm(MVT::f16);
574   }
575 
576   bool isVSrcV2F16() const {
577     return isVSrcF16() || isLiteralImm(MVT::v2f16);
578   }
579 
580   bool isVISrcB32() const {
581     return isRegOrInlineNoMods(AMDGPU::VGPR_32RegClassID, MVT::i32);
582   }
583 
584   bool isVISrcB16() const {
585     return isRegOrInlineNoMods(AMDGPU::VGPR_32RegClassID, MVT::i16);
586   }
587 
588   bool isVISrcV2B16() const {
589     return isVISrcB16();
590   }
591 
592   bool isVISrcF32() const {
593     return isRegOrInlineNoMods(AMDGPU::VGPR_32RegClassID, MVT::f32);
594   }
595 
596   bool isVISrcF16() const {
597     return isRegOrInlineNoMods(AMDGPU::VGPR_32RegClassID, MVT::f16);
598   }
599 
600   bool isVISrcV2F16() const {
601     return isVISrcF16() || isVISrcB32();
602   }
603 
604   bool isVISrc_64B64() const {
605     return isRegOrInlineNoMods(AMDGPU::VReg_64RegClassID, MVT::i64);
606   }
607 
608   bool isVISrc_64F64() const {
609     return isRegOrInlineNoMods(AMDGPU::VReg_64RegClassID, MVT::f64);
610   }
611 
612   bool isVISrc_64V2FP32() const {
613     return isRegOrInlineNoMods(AMDGPU::VReg_64RegClassID, MVT::f32);
614   }
615 
616   bool isVISrc_64V2INT32() const {
617     return isRegOrInlineNoMods(AMDGPU::VReg_64RegClassID, MVT::i32);
618   }
619 
620   bool isVISrc_256B64() const {
621     return isRegOrInlineNoMods(AMDGPU::VReg_256RegClassID, MVT::i64);
622   }
623 
624   bool isVISrc_256F64() const {
625     return isRegOrInlineNoMods(AMDGPU::VReg_256RegClassID, MVT::f64);
626   }
627 
628   bool isVISrc_128B16() const {
629     return isRegOrInlineNoMods(AMDGPU::VReg_128RegClassID, MVT::i16);
630   }
631 
632   bool isVISrc_128V2B16() const {
633     return isVISrc_128B16();
634   }
635 
636   bool isVISrc_128B32() const {
637     return isRegOrInlineNoMods(AMDGPU::VReg_128RegClassID, MVT::i32);
638   }
639 
640   bool isVISrc_128F32() const {
641     return isRegOrInlineNoMods(AMDGPU::VReg_128RegClassID, MVT::f32);
642   }
643 
644   bool isVISrc_256V2FP32() const {
645     return isRegOrInlineNoMods(AMDGPU::VReg_256RegClassID, MVT::f32);
646   }
647 
648   bool isVISrc_256V2INT32() const {
649     return isRegOrInlineNoMods(AMDGPU::VReg_256RegClassID, MVT::i32);
650   }
651 
652   bool isVISrc_512B32() const {
653     return isRegOrInlineNoMods(AMDGPU::VReg_512RegClassID, MVT::i32);
654   }
655 
656   bool isVISrc_512B16() const {
657     return isRegOrInlineNoMods(AMDGPU::VReg_512RegClassID, MVT::i16);
658   }
659 
660   bool isVISrc_512V2B16() const {
661     return isVISrc_512B16();
662   }
663 
664   bool isVISrc_512F32() const {
665     return isRegOrInlineNoMods(AMDGPU::VReg_512RegClassID, MVT::f32);
666   }
667 
668   bool isVISrc_512F16() const {
669     return isRegOrInlineNoMods(AMDGPU::VReg_512RegClassID, MVT::f16);
670   }
671 
672   bool isVISrc_512V2F16() const {
673     return isVISrc_512F16() || isVISrc_512B32();
674   }
675 
676   bool isVISrc_1024B32() const {
677     return isRegOrInlineNoMods(AMDGPU::VReg_1024RegClassID, MVT::i32);
678   }
679 
680   bool isVISrc_1024B16() const {
681     return isRegOrInlineNoMods(AMDGPU::VReg_1024RegClassID, MVT::i16);
682   }
683 
684   bool isVISrc_1024V2B16() const {
685     return isVISrc_1024B16();
686   }
687 
688   bool isVISrc_1024F32() const {
689     return isRegOrInlineNoMods(AMDGPU::VReg_1024RegClassID, MVT::f32);
690   }
691 
692   bool isVISrc_1024F16() const {
693     return isRegOrInlineNoMods(AMDGPU::VReg_1024RegClassID, MVT::f16);
694   }
695 
696   bool isVISrc_1024V2F16() const {
697     return isVISrc_1024F16() || isVISrc_1024B32();
698   }
699 
700   bool isAISrcB32() const {
701     return isRegOrInlineNoMods(AMDGPU::AGPR_32RegClassID, MVT::i32);
702   }
703 
704   bool isAISrcB16() const {
705     return isRegOrInlineNoMods(AMDGPU::AGPR_32RegClassID, MVT::i16);
706   }
707 
708   bool isAISrcV2B16() const {
709     return isAISrcB16();
710   }
711 
712   bool isAISrcF32() const {
713     return isRegOrInlineNoMods(AMDGPU::AGPR_32RegClassID, MVT::f32);
714   }
715 
716   bool isAISrcF16() const {
717     return isRegOrInlineNoMods(AMDGPU::AGPR_32RegClassID, MVT::f16);
718   }
719 
720   bool isAISrcV2F16() const {
721     return isAISrcF16() || isAISrcB32();
722   }
723 
724   bool isAISrc_64B64() const {
725     return isRegOrInlineNoMods(AMDGPU::AReg_64RegClassID, MVT::i64);
726   }
727 
728   bool isAISrc_64F64() const {
729     return isRegOrInlineNoMods(AMDGPU::AReg_64RegClassID, MVT::f64);
730   }
731 
732   bool isAISrc_128B32() const {
733     return isRegOrInlineNoMods(AMDGPU::AReg_128RegClassID, MVT::i32);
734   }
735 
736   bool isAISrc_128B16() const {
737     return isRegOrInlineNoMods(AMDGPU::AReg_128RegClassID, MVT::i16);
738   }
739 
740   bool isAISrc_128V2B16() const {
741     return isAISrc_128B16();
742   }
743 
744   bool isAISrc_128F32() const {
745     return isRegOrInlineNoMods(AMDGPU::AReg_128RegClassID, MVT::f32);
746   }
747 
748   bool isAISrc_128F16() const {
749     return isRegOrInlineNoMods(AMDGPU::AReg_128RegClassID, MVT::f16);
750   }
751 
752   bool isAISrc_128V2F16() const {
753     return isAISrc_128F16() || isAISrc_128B32();
754   }
755 
756   bool isVISrc_128F16() const {
757     return isRegOrInlineNoMods(AMDGPU::VReg_128RegClassID, MVT::f16);
758   }
759 
760   bool isVISrc_128V2F16() const {
761     return isVISrc_128F16() || isVISrc_128B32();
762   }
763 
764   bool isAISrc_256B64() const {
765     return isRegOrInlineNoMods(AMDGPU::AReg_256RegClassID, MVT::i64);
766   }
767 
768   bool isAISrc_256F64() const {
769     return isRegOrInlineNoMods(AMDGPU::AReg_256RegClassID, MVT::f64);
770   }
771 
772   bool isAISrc_512B32() const {
773     return isRegOrInlineNoMods(AMDGPU::AReg_512RegClassID, MVT::i32);
774   }
775 
776   bool isAISrc_512B16() const {
777     return isRegOrInlineNoMods(AMDGPU::AReg_512RegClassID, MVT::i16);
778   }
779 
780   bool isAISrc_512V2B16() const {
781     return isAISrc_512B16();
782   }
783 
784   bool isAISrc_512F32() const {
785     return isRegOrInlineNoMods(AMDGPU::AReg_512RegClassID, MVT::f32);
786   }
787 
788   bool isAISrc_512F16() const {
789     return isRegOrInlineNoMods(AMDGPU::AReg_512RegClassID, MVT::f16);
790   }
791 
792   bool isAISrc_512V2F16() const {
793     return isAISrc_512F16() || isAISrc_512B32();
794   }
795 
796   bool isAISrc_1024B32() const {
797     return isRegOrInlineNoMods(AMDGPU::AReg_1024RegClassID, MVT::i32);
798   }
799 
800   bool isAISrc_1024B16() const {
801     return isRegOrInlineNoMods(AMDGPU::AReg_1024RegClassID, MVT::i16);
802   }
803 
804   bool isAISrc_1024V2B16() const {
805     return isAISrc_1024B16();
806   }
807 
808   bool isAISrc_1024F32() const {
809     return isRegOrInlineNoMods(AMDGPU::AReg_1024RegClassID, MVT::f32);
810   }
811 
812   bool isAISrc_1024F16() const {
813     return isRegOrInlineNoMods(AMDGPU::AReg_1024RegClassID, MVT::f16);
814   }
815 
816   bool isAISrc_1024V2F16() const {
817     return isAISrc_1024F16() || isAISrc_1024B32();
818   }
819 
820   bool isKImmFP32() const {
821     return isLiteralImm(MVT::f32);
822   }
823 
824   bool isKImmFP16() const {
825     return isLiteralImm(MVT::f16);
826   }
827 
828   bool isMem() const override {
829     return false;
830   }
831 
832   bool isExpr() const {
833     return Kind == Expression;
834   }
835 
836   bool isSoppBrTarget() const {
837     return isExpr() || isImm();
838   }
839 
840   bool isSWaitCnt() const;
841   bool isDepCtr() const;
842   bool isSDelayAlu() const;
843   bool isHwreg() const;
844   bool isSendMsg() const;
845   bool isSwizzle() const;
846   bool isSMRDOffset8() const;
847   bool isSMEMOffset() const;
848   bool isSMRDLiteralOffset() const;
849   bool isDPP8() const;
850   bool isDPPCtrl() const;
851   bool isBLGP() const;
852   bool isCBSZ() const;
853   bool isABID() const;
854   bool isGPRIdxMode() const;
855   bool isS16Imm() const;
856   bool isU16Imm() const;
857   bool isEndpgm() const;
858   bool isWaitVDST() const;
859   bool isWaitEXP() const;
860 
861   StringRef getExpressionAsToken() const {
862     assert(isExpr());
863     const MCSymbolRefExpr *S = cast<MCSymbolRefExpr>(Expr);
864     return S->getSymbol().getName();
865   }
866 
867   StringRef getToken() const {
868     assert(isToken());
869 
870     if (Kind == Expression)
871       return getExpressionAsToken();
872 
873     return StringRef(Tok.Data, Tok.Length);
874   }
875 
876   int64_t getImm() const {
877     assert(isImm());
878     return Imm.Val;
879   }
880 
881   void setImm(int64_t Val) {
882     assert(isImm());
883     Imm.Val = Val;
884   }
885 
886   ImmTy getImmTy() const {
887     assert(isImm());
888     return Imm.Type;
889   }
890 
891   unsigned getReg() const override {
892     assert(isRegKind());
893     return Reg.RegNo;
894   }
895 
896   SMLoc getStartLoc() const override {
897     return StartLoc;
898   }
899 
900   SMLoc getEndLoc() const override {
901     return EndLoc;
902   }
903 
904   SMRange getLocRange() const {
905     return SMRange(StartLoc, EndLoc);
906   }
907 
908   Modifiers getModifiers() const {
909     assert(isRegKind() || isImmTy(ImmTyNone));
910     return isRegKind() ? Reg.Mods : Imm.Mods;
911   }
912 
913   void setModifiers(Modifiers Mods) {
914     assert(isRegKind() || isImmTy(ImmTyNone));
915     if (isRegKind())
916       Reg.Mods = Mods;
917     else
918       Imm.Mods = Mods;
919   }
920 
921   bool hasModifiers() const {
922     return getModifiers().hasModifiers();
923   }
924 
925   bool hasFPModifiers() const {
926     return getModifiers().hasFPModifiers();
927   }
928 
929   bool hasIntModifiers() const {
930     return getModifiers().hasIntModifiers();
931   }
932 
933   uint64_t applyInputFPModifiers(uint64_t Val, unsigned Size) const;
934 
935   void addImmOperands(MCInst &Inst, unsigned N, bool ApplyModifiers = true) const;
936 
937   void addLiteralImmOperand(MCInst &Inst, int64_t Val, bool ApplyModifiers) const;
938 
939   template <unsigned Bitwidth>
940   void addKImmFPOperands(MCInst &Inst, unsigned N) const;
941 
942   void addKImmFP16Operands(MCInst &Inst, unsigned N) const {
943     addKImmFPOperands<16>(Inst, N);
944   }
945 
946   void addKImmFP32Operands(MCInst &Inst, unsigned N) const {
947     addKImmFPOperands<32>(Inst, N);
948   }
949 
950   void addRegOperands(MCInst &Inst, unsigned N) const;
951 
952   void addBoolRegOperands(MCInst &Inst, unsigned N) const {
953     addRegOperands(Inst, N);
954   }
955 
956   void addRegOrImmOperands(MCInst &Inst, unsigned N) const {
957     if (isRegKind())
958       addRegOperands(Inst, N);
959     else if (isExpr())
960       Inst.addOperand(MCOperand::createExpr(Expr));
961     else
962       addImmOperands(Inst, N);
963   }
964 
965   void addRegOrImmWithInputModsOperands(MCInst &Inst, unsigned N) const {
966     Modifiers Mods = getModifiers();
967     Inst.addOperand(MCOperand::createImm(Mods.getModifiersOperand()));
968     if (isRegKind()) {
969       addRegOperands(Inst, N);
970     } else {
971       addImmOperands(Inst, N, false);
972     }
973   }
974 
975   void addRegOrImmWithFPInputModsOperands(MCInst &Inst, unsigned N) const {
976     assert(!hasIntModifiers());
977     addRegOrImmWithInputModsOperands(Inst, N);
978   }
979 
980   void addRegOrImmWithIntInputModsOperands(MCInst &Inst, unsigned N) const {
981     assert(!hasFPModifiers());
982     addRegOrImmWithInputModsOperands(Inst, N);
983   }
984 
985   void addRegWithInputModsOperands(MCInst &Inst, unsigned N) const {
986     Modifiers Mods = getModifiers();
987     Inst.addOperand(MCOperand::createImm(Mods.getModifiersOperand()));
988     assert(isRegKind());
989     addRegOperands(Inst, N);
990   }
991 
992   void addRegWithFPInputModsOperands(MCInst &Inst, unsigned N) const {
993     assert(!hasIntModifiers());
994     addRegWithInputModsOperands(Inst, N);
995   }
996 
997   void addRegWithIntInputModsOperands(MCInst &Inst, unsigned N) const {
998     assert(!hasFPModifiers());
999     addRegWithInputModsOperands(Inst, N);
1000   }
1001 
1002   void addSoppBrTargetOperands(MCInst &Inst, unsigned N) const {
1003     if (isImm())
1004       addImmOperands(Inst, N);
1005     else {
1006       assert(isExpr());
1007       Inst.addOperand(MCOperand::createExpr(Expr));
1008     }
1009   }
1010 
1011   static void printImmTy(raw_ostream& OS, ImmTy Type) {
1012     switch (Type) {
1013     case ImmTyNone: OS << "None"; break;
1014     case ImmTyGDS: OS << "GDS"; break;
1015     case ImmTyLDS: OS << "LDS"; break;
1016     case ImmTyOffen: OS << "Offen"; break;
1017     case ImmTyIdxen: OS << "Idxen"; break;
1018     case ImmTyAddr64: OS << "Addr64"; break;
1019     case ImmTyOffset: OS << "Offset"; break;
1020     case ImmTyInstOffset: OS << "InstOffset"; break;
1021     case ImmTyOffset0: OS << "Offset0"; break;
1022     case ImmTyOffset1: OS << "Offset1"; break;
1023     case ImmTyCPol: OS << "CPol"; break;
1024     case ImmTySWZ: OS << "SWZ"; break;
1025     case ImmTyTFE: OS << "TFE"; break;
1026     case ImmTyD16: OS << "D16"; break;
1027     case ImmTyFORMAT: OS << "FORMAT"; break;
1028     case ImmTyClampSI: OS << "ClampSI"; break;
1029     case ImmTyOModSI: OS << "OModSI"; break;
1030     case ImmTyDPP8: OS << "DPP8"; break;
1031     case ImmTyDppCtrl: OS << "DppCtrl"; break;
1032     case ImmTyDppRowMask: OS << "DppRowMask"; break;
1033     case ImmTyDppBankMask: OS << "DppBankMask"; break;
1034     case ImmTyDppBoundCtrl: OS << "DppBoundCtrl"; break;
1035     case ImmTyDppFi: OS << "FI"; break;
1036     case ImmTySdwaDstSel: OS << "SdwaDstSel"; break;
1037     case ImmTySdwaSrc0Sel: OS << "SdwaSrc0Sel"; break;
1038     case ImmTySdwaSrc1Sel: OS << "SdwaSrc1Sel"; break;
1039     case ImmTySdwaDstUnused: OS << "SdwaDstUnused"; break;
1040     case ImmTyDMask: OS << "DMask"; break;
1041     case ImmTyDim: OS << "Dim"; break;
1042     case ImmTyUNorm: OS << "UNorm"; break;
1043     case ImmTyDA: OS << "DA"; break;
1044     case ImmTyR128A16: OS << "R128A16"; break;
1045     case ImmTyA16: OS << "A16"; break;
1046     case ImmTyLWE: OS << "LWE"; break;
1047     case ImmTyOff: OS << "Off"; break;
1048     case ImmTyExpTgt: OS << "ExpTgt"; break;
1049     case ImmTyExpCompr: OS << "ExpCompr"; break;
1050     case ImmTyExpVM: OS << "ExpVM"; break;
1051     case ImmTyHwreg: OS << "Hwreg"; break;
1052     case ImmTySendMsg: OS << "SendMsg"; break;
1053     case ImmTyInterpSlot: OS << "InterpSlot"; break;
1054     case ImmTyInterpAttr: OS << "InterpAttr"; break;
1055     case ImmTyAttrChan: OS << "AttrChan"; break;
1056     case ImmTyOpSel: OS << "OpSel"; break;
1057     case ImmTyOpSelHi: OS << "OpSelHi"; break;
1058     case ImmTyNegLo: OS << "NegLo"; break;
1059     case ImmTyNegHi: OS << "NegHi"; break;
1060     case ImmTySwizzle: OS << "Swizzle"; break;
1061     case ImmTyGprIdxMode: OS << "GprIdxMode"; break;
1062     case ImmTyHigh: OS << "High"; break;
1063     case ImmTyBLGP: OS << "BLGP"; break;
1064     case ImmTyCBSZ: OS << "CBSZ"; break;
1065     case ImmTyABID: OS << "ABID"; break;
1066     case ImmTyEndpgm: OS << "Endpgm"; break;
1067     case ImmTyWaitVDST: OS << "WaitVDST"; break;
1068     case ImmTyWaitEXP: OS << "WaitEXP"; break;
1069     }
1070   }
1071 
1072   void print(raw_ostream &OS) const override {
1073     switch (Kind) {
1074     case Register:
1075       OS << "<register " << getReg() << " mods: " << Reg.Mods << '>';
1076       break;
1077     case Immediate:
1078       OS << '<' << getImm();
1079       if (getImmTy() != ImmTyNone) {
1080         OS << " type: "; printImmTy(OS, getImmTy());
1081       }
1082       OS << " mods: " << Imm.Mods << '>';
1083       break;
1084     case Token:
1085       OS << '\'' << getToken() << '\'';
1086       break;
1087     case Expression:
1088       OS << "<expr " << *Expr << '>';
1089       break;
1090     }
1091   }
1092 
1093   static AMDGPUOperand::Ptr CreateImm(const AMDGPUAsmParser *AsmParser,
1094                                       int64_t Val, SMLoc Loc,
1095                                       ImmTy Type = ImmTyNone,
1096                                       bool IsFPImm = false) {
1097     auto Op = std::make_unique<AMDGPUOperand>(Immediate, AsmParser);
1098     Op->Imm.Val = Val;
1099     Op->Imm.IsFPImm = IsFPImm;
1100     Op->Imm.Kind = ImmKindTyNone;
1101     Op->Imm.Type = Type;
1102     Op->Imm.Mods = Modifiers();
1103     Op->StartLoc = Loc;
1104     Op->EndLoc = Loc;
1105     return Op;
1106   }
1107 
1108   static AMDGPUOperand::Ptr CreateToken(const AMDGPUAsmParser *AsmParser,
1109                                         StringRef Str, SMLoc Loc,
1110                                         bool HasExplicitEncodingSize = true) {
1111     auto Res = std::make_unique<AMDGPUOperand>(Token, AsmParser);
1112     Res->Tok.Data = Str.data();
1113     Res->Tok.Length = Str.size();
1114     Res->StartLoc = Loc;
1115     Res->EndLoc = Loc;
1116     return Res;
1117   }
1118 
1119   static AMDGPUOperand::Ptr CreateReg(const AMDGPUAsmParser *AsmParser,
1120                                       unsigned RegNo, SMLoc S,
1121                                       SMLoc E) {
1122     auto Op = std::make_unique<AMDGPUOperand>(Register, AsmParser);
1123     Op->Reg.RegNo = RegNo;
1124     Op->Reg.Mods = Modifiers();
1125     Op->StartLoc = S;
1126     Op->EndLoc = E;
1127     return Op;
1128   }
1129 
1130   static AMDGPUOperand::Ptr CreateExpr(const AMDGPUAsmParser *AsmParser,
1131                                        const class MCExpr *Expr, SMLoc S) {
1132     auto Op = std::make_unique<AMDGPUOperand>(Expression, AsmParser);
1133     Op->Expr = Expr;
1134     Op->StartLoc = S;
1135     Op->EndLoc = S;
1136     return Op;
1137   }
1138 };
1139 
1140 raw_ostream &operator <<(raw_ostream &OS, AMDGPUOperand::Modifiers Mods) {
1141   OS << "abs:" << Mods.Abs << " neg: " << Mods.Neg << " sext:" << Mods.Sext;
1142   return OS;
1143 }
1144 
1145 //===----------------------------------------------------------------------===//
1146 // AsmParser
1147 //===----------------------------------------------------------------------===//
1148 
1149 // Holds info related to the current kernel, e.g. count of SGPRs used.
1150 // Kernel scope begins at .amdgpu_hsa_kernel directive, ends at next
1151 // .amdgpu_hsa_kernel or at EOF.
1152 class KernelScopeInfo {
1153   int SgprIndexUnusedMin = -1;
1154   int VgprIndexUnusedMin = -1;
1155   int AgprIndexUnusedMin = -1;
1156   MCContext *Ctx = nullptr;
1157   MCSubtargetInfo const *MSTI = nullptr;
1158 
1159   void usesSgprAt(int i) {
1160     if (i >= SgprIndexUnusedMin) {
1161       SgprIndexUnusedMin = ++i;
1162       if (Ctx) {
1163         MCSymbol* const Sym =
1164           Ctx->getOrCreateSymbol(Twine(".kernel.sgpr_count"));
1165         Sym->setVariableValue(MCConstantExpr::create(SgprIndexUnusedMin, *Ctx));
1166       }
1167     }
1168   }
1169 
1170   void usesVgprAt(int i) {
1171     if (i >= VgprIndexUnusedMin) {
1172       VgprIndexUnusedMin = ++i;
1173       if (Ctx) {
1174         MCSymbol* const Sym =
1175           Ctx->getOrCreateSymbol(Twine(".kernel.vgpr_count"));
1176         int totalVGPR = getTotalNumVGPRs(isGFX90A(*MSTI), AgprIndexUnusedMin,
1177                                          VgprIndexUnusedMin);
1178         Sym->setVariableValue(MCConstantExpr::create(totalVGPR, *Ctx));
1179       }
1180     }
1181   }
1182 
1183   void usesAgprAt(int i) {
1184     // Instruction will error in AMDGPUAsmParser::MatchAndEmitInstruction
1185     if (!hasMAIInsts(*MSTI))
1186       return;
1187 
1188     if (i >= AgprIndexUnusedMin) {
1189       AgprIndexUnusedMin = ++i;
1190       if (Ctx) {
1191         MCSymbol* const Sym =
1192           Ctx->getOrCreateSymbol(Twine(".kernel.agpr_count"));
1193         Sym->setVariableValue(MCConstantExpr::create(AgprIndexUnusedMin, *Ctx));
1194 
1195         // Also update vgpr_count (dependent on agpr_count for gfx908/gfx90a)
1196         MCSymbol* const vSym =
1197           Ctx->getOrCreateSymbol(Twine(".kernel.vgpr_count"));
1198         int totalVGPR = getTotalNumVGPRs(isGFX90A(*MSTI), AgprIndexUnusedMin,
1199                                          VgprIndexUnusedMin);
1200         vSym->setVariableValue(MCConstantExpr::create(totalVGPR, *Ctx));
1201       }
1202     }
1203   }
1204 
1205 public:
1206   KernelScopeInfo() = default;
1207 
1208   void initialize(MCContext &Context) {
1209     Ctx = &Context;
1210     MSTI = Ctx->getSubtargetInfo();
1211 
1212     usesSgprAt(SgprIndexUnusedMin = -1);
1213     usesVgprAt(VgprIndexUnusedMin = -1);
1214     if (hasMAIInsts(*MSTI)) {
1215       usesAgprAt(AgprIndexUnusedMin = -1);
1216     }
1217   }
1218 
1219   void usesRegister(RegisterKind RegKind, unsigned DwordRegIndex,
1220                     unsigned RegWidth) {
1221     switch (RegKind) {
1222     case IS_SGPR:
1223       usesSgprAt(DwordRegIndex + divideCeil(RegWidth, 32) - 1);
1224       break;
1225     case IS_AGPR:
1226       usesAgprAt(DwordRegIndex + divideCeil(RegWidth, 32) - 1);
1227       break;
1228     case IS_VGPR:
1229       usesVgprAt(DwordRegIndex + divideCeil(RegWidth, 32) - 1);
1230       break;
1231     default:
1232       break;
1233     }
1234   }
1235 };
1236 
1237 class AMDGPUAsmParser : public MCTargetAsmParser {
1238   MCAsmParser &Parser;
1239 
1240   // Number of extra operands parsed after the first optional operand.
1241   // This may be necessary to skip hardcoded mandatory operands.
1242   static const unsigned MAX_OPR_LOOKAHEAD = 8;
1243 
1244   unsigned ForcedEncodingSize = 0;
1245   bool ForcedDPP = false;
1246   bool ForcedSDWA = false;
1247   KernelScopeInfo KernelScope;
1248   unsigned CPolSeen;
1249 
1250   /// @name Auto-generated Match Functions
1251   /// {
1252 
1253 #define GET_ASSEMBLER_HEADER
1254 #include "AMDGPUGenAsmMatcher.inc"
1255 
1256   /// }
1257 
1258 private:
1259   bool ParseAsAbsoluteExpression(uint32_t &Ret);
1260   bool OutOfRangeError(SMRange Range);
1261   /// Calculate VGPR/SGPR blocks required for given target, reserved
1262   /// registers, and user-specified NextFreeXGPR values.
1263   ///
1264   /// \param Features [in] Target features, used for bug corrections.
1265   /// \param VCCUsed [in] Whether VCC special SGPR is reserved.
1266   /// \param FlatScrUsed [in] Whether FLAT_SCRATCH special SGPR is reserved.
1267   /// \param XNACKUsed [in] Whether XNACK_MASK special SGPR is reserved.
1268   /// \param EnableWavefrontSize32 [in] Value of ENABLE_WAVEFRONT_SIZE32 kernel
1269   /// descriptor field, if valid.
1270   /// \param NextFreeVGPR [in] Max VGPR number referenced, plus one.
1271   /// \param VGPRRange [in] Token range, used for VGPR diagnostics.
1272   /// \param NextFreeSGPR [in] Max SGPR number referenced, plus one.
1273   /// \param SGPRRange [in] Token range, used for SGPR diagnostics.
1274   /// \param VGPRBlocks [out] Result VGPR block count.
1275   /// \param SGPRBlocks [out] Result SGPR block count.
1276   bool calculateGPRBlocks(const FeatureBitset &Features, bool VCCUsed,
1277                           bool FlatScrUsed, bool XNACKUsed,
1278                           Optional<bool> EnableWavefrontSize32, unsigned NextFreeVGPR,
1279                           SMRange VGPRRange, unsigned NextFreeSGPR,
1280                           SMRange SGPRRange, unsigned &VGPRBlocks,
1281                           unsigned &SGPRBlocks);
1282   bool ParseDirectiveAMDGCNTarget();
1283   bool ParseDirectiveAMDHSAKernel();
1284   bool ParseDirectiveMajorMinor(uint32_t &Major, uint32_t &Minor);
1285   bool ParseDirectiveHSACodeObjectVersion();
1286   bool ParseDirectiveHSACodeObjectISA();
1287   bool ParseAMDKernelCodeTValue(StringRef ID, amd_kernel_code_t &Header);
1288   bool ParseDirectiveAMDKernelCodeT();
1289   // TODO: Possibly make subtargetHasRegister const.
1290   bool subtargetHasRegister(const MCRegisterInfo &MRI, unsigned RegNo);
1291   bool ParseDirectiveAMDGPUHsaKernel();
1292 
1293   bool ParseDirectiveISAVersion();
1294   bool ParseDirectiveHSAMetadata();
1295   bool ParseDirectivePALMetadataBegin();
1296   bool ParseDirectivePALMetadata();
1297   bool ParseDirectiveAMDGPULDS();
1298 
1299   /// Common code to parse out a block of text (typically YAML) between start and
1300   /// end directives.
1301   bool ParseToEndDirective(const char *AssemblerDirectiveBegin,
1302                            const char *AssemblerDirectiveEnd,
1303                            std::string &CollectString);
1304 
1305   bool AddNextRegisterToList(unsigned& Reg, unsigned& RegWidth,
1306                              RegisterKind RegKind, unsigned Reg1, SMLoc Loc);
1307   bool ParseAMDGPURegister(RegisterKind &RegKind, unsigned &Reg,
1308                            unsigned &RegNum, unsigned &RegWidth,
1309                            bool RestoreOnFailure = false);
1310   bool ParseAMDGPURegister(RegisterKind &RegKind, unsigned &Reg,
1311                            unsigned &RegNum, unsigned &RegWidth,
1312                            SmallVectorImpl<AsmToken> &Tokens);
1313   unsigned ParseRegularReg(RegisterKind &RegKind, unsigned &RegNum,
1314                            unsigned &RegWidth,
1315                            SmallVectorImpl<AsmToken> &Tokens);
1316   unsigned ParseSpecialReg(RegisterKind &RegKind, unsigned &RegNum,
1317                            unsigned &RegWidth,
1318                            SmallVectorImpl<AsmToken> &Tokens);
1319   unsigned ParseRegList(RegisterKind &RegKind, unsigned &RegNum,
1320                         unsigned &RegWidth, SmallVectorImpl<AsmToken> &Tokens);
1321   bool ParseRegRange(unsigned& Num, unsigned& Width);
1322   unsigned getRegularReg(RegisterKind RegKind,
1323                          unsigned RegNum,
1324                          unsigned RegWidth,
1325                          SMLoc Loc);
1326 
1327   bool isRegister();
1328   bool isRegister(const AsmToken &Token, const AsmToken &NextToken) const;
1329   Optional<StringRef> getGprCountSymbolName(RegisterKind RegKind);
1330   void initializeGprCountSymbol(RegisterKind RegKind);
1331   bool updateGprCountSymbols(RegisterKind RegKind, unsigned DwordRegIndex,
1332                              unsigned RegWidth);
1333   void cvtMubufImpl(MCInst &Inst, const OperandVector &Operands,
1334                     bool IsAtomic, bool IsLds = false);
1335   void cvtDSImpl(MCInst &Inst, const OperandVector &Operands,
1336                  bool IsGdsHardcoded);
1337 
1338 public:
1339   enum AMDGPUMatchResultTy {
1340     Match_PreferE32 = FIRST_TARGET_MATCH_RESULT_TY
1341   };
1342   enum OperandMode {
1343     OperandMode_Default,
1344     OperandMode_NSA,
1345   };
1346 
1347   using OptionalImmIndexMap = std::map<AMDGPUOperand::ImmTy, unsigned>;
1348 
1349   AMDGPUAsmParser(const MCSubtargetInfo &STI, MCAsmParser &_Parser,
1350                const MCInstrInfo &MII,
1351                const MCTargetOptions &Options)
1352       : MCTargetAsmParser(Options, STI, MII), Parser(_Parser) {
1353     MCAsmParserExtension::Initialize(Parser);
1354 
1355     if (getFeatureBits().none()) {
1356       // Set default features.
1357       copySTI().ToggleFeature("southern-islands");
1358     }
1359 
1360     setAvailableFeatures(ComputeAvailableFeatures(getFeatureBits()));
1361 
1362     {
1363       // TODO: make those pre-defined variables read-only.
1364       // Currently there is none suitable machinery in the core llvm-mc for this.
1365       // MCSymbol::isRedefinable is intended for another purpose, and
1366       // AsmParser::parseDirectiveSet() cannot be specialized for specific target.
1367       AMDGPU::IsaVersion ISA = AMDGPU::getIsaVersion(getSTI().getCPU());
1368       MCContext &Ctx = getContext();
1369       if (ISA.Major >= 6 && isHsaAbiVersion3AndAbove(&getSTI())) {
1370         MCSymbol *Sym =
1371             Ctx.getOrCreateSymbol(Twine(".amdgcn.gfx_generation_number"));
1372         Sym->setVariableValue(MCConstantExpr::create(ISA.Major, Ctx));
1373         Sym = Ctx.getOrCreateSymbol(Twine(".amdgcn.gfx_generation_minor"));
1374         Sym->setVariableValue(MCConstantExpr::create(ISA.Minor, Ctx));
1375         Sym = Ctx.getOrCreateSymbol(Twine(".amdgcn.gfx_generation_stepping"));
1376         Sym->setVariableValue(MCConstantExpr::create(ISA.Stepping, Ctx));
1377       } else {
1378         MCSymbol *Sym =
1379             Ctx.getOrCreateSymbol(Twine(".option.machine_version_major"));
1380         Sym->setVariableValue(MCConstantExpr::create(ISA.Major, Ctx));
1381         Sym = Ctx.getOrCreateSymbol(Twine(".option.machine_version_minor"));
1382         Sym->setVariableValue(MCConstantExpr::create(ISA.Minor, Ctx));
1383         Sym = Ctx.getOrCreateSymbol(Twine(".option.machine_version_stepping"));
1384         Sym->setVariableValue(MCConstantExpr::create(ISA.Stepping, Ctx));
1385       }
1386       if (ISA.Major >= 6 && isHsaAbiVersion3AndAbove(&getSTI())) {
1387         initializeGprCountSymbol(IS_VGPR);
1388         initializeGprCountSymbol(IS_SGPR);
1389       } else
1390         KernelScope.initialize(getContext());
1391     }
1392   }
1393 
1394   bool hasMIMG_R128() const {
1395     return AMDGPU::hasMIMG_R128(getSTI());
1396   }
1397 
1398   bool hasPackedD16() const {
1399     return AMDGPU::hasPackedD16(getSTI());
1400   }
1401 
1402   bool hasGFX10A16() const {
1403     return AMDGPU::hasGFX10A16(getSTI());
1404   }
1405 
1406   bool hasG16() const { return AMDGPU::hasG16(getSTI()); }
1407 
1408   bool isSI() const {
1409     return AMDGPU::isSI(getSTI());
1410   }
1411 
1412   bool isCI() const {
1413     return AMDGPU::isCI(getSTI());
1414   }
1415 
1416   bool isVI() const {
1417     return AMDGPU::isVI(getSTI());
1418   }
1419 
1420   bool isGFX9() const {
1421     return AMDGPU::isGFX9(getSTI());
1422   }
1423 
1424   // TODO: isGFX90A is also true for GFX940. We need to clean it.
1425   bool isGFX90A() const {
1426     return AMDGPU::isGFX90A(getSTI());
1427   }
1428 
1429   bool isGFX940() const {
1430     return AMDGPU::isGFX940(getSTI());
1431   }
1432 
1433   bool isGFX9Plus() const {
1434     return AMDGPU::isGFX9Plus(getSTI());
1435   }
1436 
1437   bool isGFX10() const {
1438     return AMDGPU::isGFX10(getSTI());
1439   }
1440 
1441   bool isGFX10Plus() const { return AMDGPU::isGFX10Plus(getSTI()); }
1442 
1443   bool isGFX11() const {
1444     return AMDGPU::isGFX11(getSTI());
1445   }
1446 
1447   bool isGFX11Plus() const {
1448     return AMDGPU::isGFX11Plus(getSTI());
1449   }
1450 
1451   bool isGFX10_BEncoding() const {
1452     return AMDGPU::isGFX10_BEncoding(getSTI());
1453   }
1454 
1455   bool hasInv2PiInlineImm() const {
1456     return getFeatureBits()[AMDGPU::FeatureInv2PiInlineImm];
1457   }
1458 
1459   bool hasFlatOffsets() const {
1460     return getFeatureBits()[AMDGPU::FeatureFlatInstOffsets];
1461   }
1462 
1463   bool hasArchitectedFlatScratch() const {
1464     return getFeatureBits()[AMDGPU::FeatureArchitectedFlatScratch];
1465   }
1466 
1467   bool hasSGPR102_SGPR103() const {
1468     return !isVI() && !isGFX9();
1469   }
1470 
1471   bool hasSGPR104_SGPR105() const { return isGFX10Plus(); }
1472 
1473   bool hasIntClamp() const {
1474     return getFeatureBits()[AMDGPU::FeatureIntClamp];
1475   }
1476 
1477   AMDGPUTargetStreamer &getTargetStreamer() {
1478     MCTargetStreamer &TS = *getParser().getStreamer().getTargetStreamer();
1479     return static_cast<AMDGPUTargetStreamer &>(TS);
1480   }
1481 
1482   const MCRegisterInfo *getMRI() const {
1483     // We need this const_cast because for some reason getContext() is not const
1484     // in MCAsmParser.
1485     return const_cast<AMDGPUAsmParser*>(this)->getContext().getRegisterInfo();
1486   }
1487 
1488   const MCInstrInfo *getMII() const {
1489     return &MII;
1490   }
1491 
1492   const FeatureBitset &getFeatureBits() const {
1493     return getSTI().getFeatureBits();
1494   }
1495 
1496   void setForcedEncodingSize(unsigned Size) { ForcedEncodingSize = Size; }
1497   void setForcedDPP(bool ForceDPP_) { ForcedDPP = ForceDPP_; }
1498   void setForcedSDWA(bool ForceSDWA_) { ForcedSDWA = ForceSDWA_; }
1499 
1500   unsigned getForcedEncodingSize() const { return ForcedEncodingSize; }
1501   bool isForcedVOP3() const { return ForcedEncodingSize == 64; }
1502   bool isForcedDPP() const { return ForcedDPP; }
1503   bool isForcedSDWA() const { return ForcedSDWA; }
1504   ArrayRef<unsigned> getMatchedVariants() const;
1505   StringRef getMatchedVariantName() const;
1506 
1507   std::unique_ptr<AMDGPUOperand> parseRegister(bool RestoreOnFailure = false);
1508   bool ParseRegister(unsigned &RegNo, SMLoc &StartLoc, SMLoc &EndLoc,
1509                      bool RestoreOnFailure);
1510   bool ParseRegister(unsigned &RegNo, SMLoc &StartLoc, SMLoc &EndLoc) override;
1511   OperandMatchResultTy tryParseRegister(unsigned &RegNo, SMLoc &StartLoc,
1512                                         SMLoc &EndLoc) override;
1513   unsigned checkTargetMatchPredicate(MCInst &Inst) override;
1514   unsigned validateTargetOperandClass(MCParsedAsmOperand &Op,
1515                                       unsigned Kind) override;
1516   bool MatchAndEmitInstruction(SMLoc IDLoc, unsigned &Opcode,
1517                                OperandVector &Operands, MCStreamer &Out,
1518                                uint64_t &ErrorInfo,
1519                                bool MatchingInlineAsm) override;
1520   bool ParseDirective(AsmToken DirectiveID) override;
1521   OperandMatchResultTy parseOperand(OperandVector &Operands, StringRef Mnemonic,
1522                                     OperandMode Mode = OperandMode_Default);
1523   StringRef parseMnemonicSuffix(StringRef Name);
1524   bool ParseInstruction(ParseInstructionInfo &Info, StringRef Name,
1525                         SMLoc NameLoc, OperandVector &Operands) override;
1526   //bool ProcessInstruction(MCInst &Inst);
1527 
1528   OperandMatchResultTy parseIntWithPrefix(const char *Prefix, int64_t &Int);
1529 
1530   OperandMatchResultTy
1531   parseIntWithPrefix(const char *Prefix, OperandVector &Operands,
1532                      AMDGPUOperand::ImmTy ImmTy = AMDGPUOperand::ImmTyNone,
1533                      bool (*ConvertResult)(int64_t &) = nullptr);
1534 
1535   OperandMatchResultTy
1536   parseOperandArrayWithPrefix(const char *Prefix,
1537                               OperandVector &Operands,
1538                               AMDGPUOperand::ImmTy ImmTy = AMDGPUOperand::ImmTyNone,
1539                               bool (*ConvertResult)(int64_t&) = nullptr);
1540 
1541   OperandMatchResultTy
1542   parseNamedBit(StringRef Name, OperandVector &Operands,
1543                 AMDGPUOperand::ImmTy ImmTy = AMDGPUOperand::ImmTyNone);
1544   OperandMatchResultTy parseCPol(OperandVector &Operands);
1545   OperandMatchResultTy parseStringWithPrefix(StringRef Prefix,
1546                                              StringRef &Value,
1547                                              SMLoc &StringLoc);
1548 
1549   bool isModifier();
1550   bool isOperandModifier(const AsmToken &Token, const AsmToken &NextToken) const;
1551   bool isRegOrOperandModifier(const AsmToken &Token, const AsmToken &NextToken) const;
1552   bool isNamedOperandModifier(const AsmToken &Token, const AsmToken &NextToken) const;
1553   bool isOpcodeModifierWithVal(const AsmToken &Token, const AsmToken &NextToken) const;
1554   bool parseSP3NegModifier();
1555   OperandMatchResultTy parseImm(OperandVector &Operands, bool HasSP3AbsModifier = false);
1556   OperandMatchResultTy parseReg(OperandVector &Operands);
1557   OperandMatchResultTy parseRegOrImm(OperandVector &Operands, bool HasSP3AbsMod = false);
1558   OperandMatchResultTy parseRegOrImmWithFPInputMods(OperandVector &Operands, bool AllowImm = true);
1559   OperandMatchResultTy parseRegOrImmWithIntInputMods(OperandVector &Operands, bool AllowImm = true);
1560   OperandMatchResultTy parseRegWithFPInputMods(OperandVector &Operands);
1561   OperandMatchResultTy parseRegWithIntInputMods(OperandVector &Operands);
1562   OperandMatchResultTy parseVReg32OrOff(OperandVector &Operands);
1563   OperandMatchResultTy parseDfmtNfmt(int64_t &Format);
1564   OperandMatchResultTy parseUfmt(int64_t &Format);
1565   OperandMatchResultTy parseSymbolicSplitFormat(StringRef FormatStr, SMLoc Loc, int64_t &Format);
1566   OperandMatchResultTy parseSymbolicUnifiedFormat(StringRef FormatStr, SMLoc Loc, int64_t &Format);
1567   OperandMatchResultTy parseFORMAT(OperandVector &Operands);
1568   OperandMatchResultTy parseSymbolicOrNumericFormat(int64_t &Format);
1569   OperandMatchResultTy parseNumericFormat(int64_t &Format);
1570   bool tryParseFmt(const char *Pref, int64_t MaxVal, int64_t &Val);
1571   bool matchDfmtNfmt(int64_t &Dfmt, int64_t &Nfmt, StringRef FormatStr, SMLoc Loc);
1572 
1573   void cvtDSOffset01(MCInst &Inst, const OperandVector &Operands);
1574   void cvtDS(MCInst &Inst, const OperandVector &Operands) { cvtDSImpl(Inst, Operands, false); }
1575   void cvtDSGds(MCInst &Inst, const OperandVector &Operands) { cvtDSImpl(Inst, Operands, true); }
1576   void cvtExp(MCInst &Inst, const OperandVector &Operands);
1577 
1578   bool parseCnt(int64_t &IntVal);
1579   OperandMatchResultTy parseSWaitCntOps(OperandVector &Operands);
1580 
1581   bool parseDepCtr(int64_t &IntVal, unsigned &Mask);
1582   void depCtrError(SMLoc Loc, int ErrorId, StringRef DepCtrName);
1583   OperandMatchResultTy parseDepCtrOps(OperandVector &Operands);
1584 
1585   bool parseDelay(int64_t &Delay);
1586   OperandMatchResultTy parseSDelayAluOps(OperandVector &Operands);
1587 
1588   OperandMatchResultTy parseHwreg(OperandVector &Operands);
1589 
1590 private:
1591   struct OperandInfoTy {
1592     SMLoc Loc;
1593     int64_t Id;
1594     bool IsSymbolic = false;
1595     bool IsDefined = false;
1596 
1597     OperandInfoTy(int64_t Id_) : Id(Id_) {}
1598   };
1599 
1600   bool parseSendMsgBody(OperandInfoTy &Msg, OperandInfoTy &Op, OperandInfoTy &Stream);
1601   bool validateSendMsg(const OperandInfoTy &Msg,
1602                        const OperandInfoTy &Op,
1603                        const OperandInfoTy &Stream);
1604 
1605   bool parseHwregBody(OperandInfoTy &HwReg,
1606                       OperandInfoTy &Offset,
1607                       OperandInfoTy &Width);
1608   bool validateHwreg(const OperandInfoTy &HwReg,
1609                      const OperandInfoTy &Offset,
1610                      const OperandInfoTy &Width);
1611 
1612   SMLoc getFlatOffsetLoc(const OperandVector &Operands) const;
1613   SMLoc getSMEMOffsetLoc(const OperandVector &Operands) const;
1614   SMLoc getBLGPLoc(const OperandVector &Operands) const;
1615 
1616   SMLoc getOperandLoc(std::function<bool(const AMDGPUOperand&)> Test,
1617                       const OperandVector &Operands) const;
1618   SMLoc getImmLoc(AMDGPUOperand::ImmTy Type, const OperandVector &Operands) const;
1619   SMLoc getRegLoc(unsigned Reg, const OperandVector &Operands) const;
1620   SMLoc getLitLoc(const OperandVector &Operands) const;
1621   SMLoc getConstLoc(const OperandVector &Operands) const;
1622 
1623   bool validateInstruction(const MCInst &Inst, const SMLoc &IDLoc, const OperandVector &Operands);
1624   bool validateFlatOffset(const MCInst &Inst, const OperandVector &Operands);
1625   bool validateSMEMOffset(const MCInst &Inst, const OperandVector &Operands);
1626   bool validateSOPLiteral(const MCInst &Inst) const;
1627   bool validateConstantBusLimitations(const MCInst &Inst, const OperandVector &Operands);
1628   bool validateEarlyClobberLimitations(const MCInst &Inst, const OperandVector &Operands);
1629   bool validateIntClampSupported(const MCInst &Inst);
1630   bool validateMIMGAtomicDMask(const MCInst &Inst);
1631   bool validateMIMGGatherDMask(const MCInst &Inst);
1632   bool validateMovrels(const MCInst &Inst, const OperandVector &Operands);
1633   Optional<StringRef> validateMIMGDataSize(const MCInst &Inst);
1634   bool validateMIMGAddrSize(const MCInst &Inst);
1635   bool validateMIMGD16(const MCInst &Inst);
1636   bool validateMIMGDim(const MCInst &Inst);
1637   bool validateMIMGMSAA(const MCInst &Inst);
1638   bool validateOpSel(const MCInst &Inst);
1639   bool validateDPP(const MCInst &Inst, const OperandVector &Operands);
1640   bool validateVccOperand(unsigned Reg) const;
1641   bool validateVOPLiteral(const MCInst &Inst, const OperandVector &Operands);
1642   bool validateMAIAccWrite(const MCInst &Inst, const OperandVector &Operands);
1643   bool validateMFMA(const MCInst &Inst, const OperandVector &Operands);
1644   bool validateAGPRLdSt(const MCInst &Inst) const;
1645   bool validateVGPRAlign(const MCInst &Inst) const;
1646   bool validateBLGP(const MCInst &Inst, const OperandVector &Operands);
1647   bool validateGWS(const MCInst &Inst, const OperandVector &Operands);
1648   bool validateDivScale(const MCInst &Inst);
1649   bool validateCoherencyBits(const MCInst &Inst, const OperandVector &Operands,
1650                              const SMLoc &IDLoc);
1651   bool validateFlatLdsDMA(const MCInst &Inst, const OperandVector &Operands,
1652                           const SMLoc &IDLoc);
1653   bool validateExeczVcczOperands(const OperandVector &Operands);
1654   Optional<StringRef> validateLdsDirect(const MCInst &Inst);
1655   unsigned getConstantBusLimit(unsigned Opcode) const;
1656   bool usesConstantBus(const MCInst &Inst, unsigned OpIdx);
1657   bool isInlineConstant(const MCInst &Inst, unsigned OpIdx) const;
1658   unsigned findImplicitSGPRReadInVOP(const MCInst &Inst) const;
1659 
1660   bool isSupportedMnemo(StringRef Mnemo,
1661                         const FeatureBitset &FBS);
1662   bool isSupportedMnemo(StringRef Mnemo,
1663                         const FeatureBitset &FBS,
1664                         ArrayRef<unsigned> Variants);
1665   bool checkUnsupportedInstruction(StringRef Name, const SMLoc &IDLoc);
1666 
1667   bool isId(const StringRef Id) const;
1668   bool isId(const AsmToken &Token, const StringRef Id) const;
1669   bool isToken(const AsmToken::TokenKind Kind) const;
1670   bool trySkipId(const StringRef Id);
1671   bool trySkipId(const StringRef Pref, const StringRef Id);
1672   bool trySkipId(const StringRef Id, const AsmToken::TokenKind Kind);
1673   bool trySkipToken(const AsmToken::TokenKind Kind);
1674   bool skipToken(const AsmToken::TokenKind Kind, const StringRef ErrMsg);
1675   bool parseString(StringRef &Val, const StringRef ErrMsg = "expected a string");
1676   bool parseId(StringRef &Val, const StringRef ErrMsg = "");
1677 
1678   void peekTokens(MutableArrayRef<AsmToken> Tokens);
1679   AsmToken::TokenKind getTokenKind() const;
1680   bool parseExpr(int64_t &Imm, StringRef Expected = "");
1681   bool parseExpr(OperandVector &Operands);
1682   StringRef getTokenStr() const;
1683   AsmToken peekToken();
1684   AsmToken getToken() const;
1685   SMLoc getLoc() const;
1686   void lex();
1687 
1688 public:
1689   void onBeginOfFile() override;
1690 
1691   OperandMatchResultTy parseOptionalOperand(OperandVector &Operands);
1692   OperandMatchResultTy parseOptionalOpr(OperandVector &Operands);
1693 
1694   OperandMatchResultTy parseExpTgt(OperandVector &Operands);
1695   OperandMatchResultTy parseSendMsgOp(OperandVector &Operands);
1696   OperandMatchResultTy parseInterpSlot(OperandVector &Operands);
1697   OperandMatchResultTy parseInterpAttr(OperandVector &Operands);
1698   OperandMatchResultTy parseSOppBrTarget(OperandVector &Operands);
1699   OperandMatchResultTy parseBoolReg(OperandVector &Operands);
1700 
1701   bool parseSwizzleOperand(int64_t &Op,
1702                            const unsigned MinVal,
1703                            const unsigned MaxVal,
1704                            const StringRef ErrMsg,
1705                            SMLoc &Loc);
1706   bool parseSwizzleOperands(const unsigned OpNum, int64_t* Op,
1707                             const unsigned MinVal,
1708                             const unsigned MaxVal,
1709                             const StringRef ErrMsg);
1710   OperandMatchResultTy parseSwizzleOp(OperandVector &Operands);
1711   bool parseSwizzleOffset(int64_t &Imm);
1712   bool parseSwizzleMacro(int64_t &Imm);
1713   bool parseSwizzleQuadPerm(int64_t &Imm);
1714   bool parseSwizzleBitmaskPerm(int64_t &Imm);
1715   bool parseSwizzleBroadcast(int64_t &Imm);
1716   bool parseSwizzleSwap(int64_t &Imm);
1717   bool parseSwizzleReverse(int64_t &Imm);
1718 
1719   OperandMatchResultTy parseGPRIdxMode(OperandVector &Operands);
1720   int64_t parseGPRIdxMacro();
1721 
1722   void cvtMubuf(MCInst &Inst, const OperandVector &Operands) { cvtMubufImpl(Inst, Operands, false); }
1723   void cvtMubufAtomic(MCInst &Inst, const OperandVector &Operands) { cvtMubufImpl(Inst, Operands, true); }
1724   void cvtMubufLds(MCInst &Inst, const OperandVector &Operands) { cvtMubufImpl(Inst, Operands, false, true); }
1725   void cvtMtbuf(MCInst &Inst, const OperandVector &Operands);
1726 
1727   AMDGPUOperand::Ptr defaultCPol() const;
1728 
1729   AMDGPUOperand::Ptr defaultSMRDOffset8() const;
1730   AMDGPUOperand::Ptr defaultSMEMOffset() const;
1731   AMDGPUOperand::Ptr defaultSMRDLiteralOffset() const;
1732   AMDGPUOperand::Ptr defaultFlatOffset() const;
1733 
1734   OperandMatchResultTy parseOModOperand(OperandVector &Operands);
1735 
1736   void cvtVOP3(MCInst &Inst, const OperandVector &Operands,
1737                OptionalImmIndexMap &OptionalIdx);
1738   void cvtVOP3OpSel(MCInst &Inst, const OperandVector &Operands);
1739   void cvtVOP3(MCInst &Inst, const OperandVector &Operands);
1740   void cvtVOP3P(MCInst &Inst, const OperandVector &Operands);
1741   void cvtVOP3P(MCInst &Inst, const OperandVector &Operands,
1742                 OptionalImmIndexMap &OptionalIdx);
1743 
1744   void cvtVOP3Interp(MCInst &Inst, const OperandVector &Operands);
1745   void cvtVINTERP(MCInst &Inst, const OperandVector &Operands);
1746 
1747   void cvtMIMG(MCInst &Inst, const OperandVector &Operands,
1748                bool IsAtomic = false);
1749   void cvtMIMGAtomic(MCInst &Inst, const OperandVector &Operands);
1750   void cvtIntersectRay(MCInst &Inst, const OperandVector &Operands);
1751 
1752   void cvtSMEMAtomic(MCInst &Inst, const OperandVector &Operands);
1753 
1754   bool parseDimId(unsigned &Encoding);
1755   OperandMatchResultTy parseDim(OperandVector &Operands);
1756   OperandMatchResultTy parseDPP8(OperandVector &Operands);
1757   OperandMatchResultTy parseDPPCtrl(OperandVector &Operands);
1758   bool isSupportedDPPCtrl(StringRef Ctrl, const OperandVector &Operands);
1759   int64_t parseDPPCtrlSel(StringRef Ctrl);
1760   int64_t parseDPPCtrlPerm();
1761   AMDGPUOperand::Ptr defaultRowMask() const;
1762   AMDGPUOperand::Ptr defaultBankMask() const;
1763   AMDGPUOperand::Ptr defaultBoundCtrl() const;
1764   AMDGPUOperand::Ptr defaultFI() const;
1765   void cvtDPP(MCInst &Inst, const OperandVector &Operands, bool IsDPP8 = false);
1766   void cvtDPP8(MCInst &Inst, const OperandVector &Operands) {
1767     cvtDPP(Inst, Operands, true);
1768   }
1769   void cvtVOPCNoDstDPP(MCInst &Inst, const OperandVector &Operands,
1770                        bool IsDPP8 = false);
1771   void cvtVOPCNoDstDPP8(MCInst &Inst, const OperandVector &Operands) {
1772     cvtVOPCNoDstDPP(Inst, Operands, true);
1773   }
1774   void cvtVOP3DPP(MCInst &Inst, const OperandVector &Operands,
1775                   bool IsDPP8 = false);
1776   void cvtVOP3DPP8(MCInst &Inst, const OperandVector &Operands) {
1777     cvtVOP3DPP(Inst, Operands, true);
1778   }
1779   void cvtVOPC64NoDstDPP(MCInst &Inst, const OperandVector &Operands,
1780                          bool IsDPP8 = false);
1781   void cvtVOPC64NoDstDPP8(MCInst &Inst, const OperandVector &Operands) {
1782     cvtVOPC64NoDstDPP(Inst, Operands, true);
1783   }
1784 
1785   OperandMatchResultTy parseSDWASel(OperandVector &Operands, StringRef Prefix,
1786                                     AMDGPUOperand::ImmTy Type);
1787   OperandMatchResultTy parseSDWADstUnused(OperandVector &Operands);
1788   void cvtSdwaVOP1(MCInst &Inst, const OperandVector &Operands);
1789   void cvtSdwaVOP2(MCInst &Inst, const OperandVector &Operands);
1790   void cvtSdwaVOP2b(MCInst &Inst, const OperandVector &Operands);
1791   void cvtSdwaVOP2e(MCInst &Inst, const OperandVector &Operands);
1792   void cvtSdwaVOPC(MCInst &Inst, const OperandVector &Operands);
1793   void cvtSDWA(MCInst &Inst, const OperandVector &Operands,
1794                uint64_t BasicInstType,
1795                bool SkipDstVcc = false,
1796                bool SkipSrcVcc = false);
1797 
1798   AMDGPUOperand::Ptr defaultBLGP() const;
1799   AMDGPUOperand::Ptr defaultCBSZ() const;
1800   AMDGPUOperand::Ptr defaultABID() const;
1801 
1802   OperandMatchResultTy parseEndpgmOp(OperandVector &Operands);
1803   AMDGPUOperand::Ptr defaultEndpgmImmOperands() const;
1804 
1805   AMDGPUOperand::Ptr defaultWaitVDST() const;
1806   AMDGPUOperand::Ptr defaultWaitEXP() const;
1807 };
1808 
1809 struct OptionalOperand {
1810   const char *Name;
1811   AMDGPUOperand::ImmTy Type;
1812   bool IsBit;
1813   bool (*ConvertResult)(int64_t&);
1814 };
1815 
1816 } // end anonymous namespace
1817 
1818 // May be called with integer type with equivalent bitwidth.
1819 static const fltSemantics *getFltSemantics(unsigned Size) {
1820   switch (Size) {
1821   case 4:
1822     return &APFloat::IEEEsingle();
1823   case 8:
1824     return &APFloat::IEEEdouble();
1825   case 2:
1826     return &APFloat::IEEEhalf();
1827   default:
1828     llvm_unreachable("unsupported fp type");
1829   }
1830 }
1831 
1832 static const fltSemantics *getFltSemantics(MVT VT) {
1833   return getFltSemantics(VT.getSizeInBits() / 8);
1834 }
1835 
1836 static const fltSemantics *getOpFltSemantics(uint8_t OperandType) {
1837   switch (OperandType) {
1838   case AMDGPU::OPERAND_REG_IMM_INT32:
1839   case AMDGPU::OPERAND_REG_IMM_FP32:
1840   case AMDGPU::OPERAND_REG_IMM_FP32_DEFERRED:
1841   case AMDGPU::OPERAND_REG_INLINE_C_INT32:
1842   case AMDGPU::OPERAND_REG_INLINE_C_FP32:
1843   case AMDGPU::OPERAND_REG_INLINE_AC_INT32:
1844   case AMDGPU::OPERAND_REG_INLINE_AC_FP32:
1845   case AMDGPU::OPERAND_REG_INLINE_C_V2FP32:
1846   case AMDGPU::OPERAND_REG_IMM_V2FP32:
1847   case AMDGPU::OPERAND_REG_INLINE_C_V2INT32:
1848   case AMDGPU::OPERAND_REG_IMM_V2INT32:
1849   case AMDGPU::OPERAND_KIMM32:
1850     return &APFloat::IEEEsingle();
1851   case AMDGPU::OPERAND_REG_IMM_INT64:
1852   case AMDGPU::OPERAND_REG_IMM_FP64:
1853   case AMDGPU::OPERAND_REG_INLINE_C_INT64:
1854   case AMDGPU::OPERAND_REG_INLINE_C_FP64:
1855   case AMDGPU::OPERAND_REG_INLINE_AC_FP64:
1856     return &APFloat::IEEEdouble();
1857   case AMDGPU::OPERAND_REG_IMM_INT16:
1858   case AMDGPU::OPERAND_REG_IMM_FP16:
1859   case AMDGPU::OPERAND_REG_IMM_FP16_DEFERRED:
1860   case AMDGPU::OPERAND_REG_INLINE_C_INT16:
1861   case AMDGPU::OPERAND_REG_INLINE_C_FP16:
1862   case AMDGPU::OPERAND_REG_INLINE_C_V2INT16:
1863   case AMDGPU::OPERAND_REG_INLINE_C_V2FP16:
1864   case AMDGPU::OPERAND_REG_INLINE_AC_INT16:
1865   case AMDGPU::OPERAND_REG_INLINE_AC_FP16:
1866   case AMDGPU::OPERAND_REG_INLINE_AC_V2INT16:
1867   case AMDGPU::OPERAND_REG_INLINE_AC_V2FP16:
1868   case AMDGPU::OPERAND_REG_IMM_V2INT16:
1869   case AMDGPU::OPERAND_REG_IMM_V2FP16:
1870   case AMDGPU::OPERAND_KIMM16:
1871     return &APFloat::IEEEhalf();
1872   default:
1873     llvm_unreachable("unsupported fp type");
1874   }
1875 }
1876 
1877 //===----------------------------------------------------------------------===//
1878 // Operand
1879 //===----------------------------------------------------------------------===//
1880 
1881 static bool canLosslesslyConvertToFPType(APFloat &FPLiteral, MVT VT) {
1882   bool Lost;
1883 
1884   // Convert literal to single precision
1885   APFloat::opStatus Status = FPLiteral.convert(*getFltSemantics(VT),
1886                                                APFloat::rmNearestTiesToEven,
1887                                                &Lost);
1888   // We allow precision lost but not overflow or underflow
1889   if (Status != APFloat::opOK &&
1890       Lost &&
1891       ((Status & APFloat::opOverflow)  != 0 ||
1892        (Status & APFloat::opUnderflow) != 0)) {
1893     return false;
1894   }
1895 
1896   return true;
1897 }
1898 
1899 static bool isSafeTruncation(int64_t Val, unsigned Size) {
1900   return isUIntN(Size, Val) || isIntN(Size, Val);
1901 }
1902 
1903 static bool isInlineableLiteralOp16(int64_t Val, MVT VT, bool HasInv2Pi) {
1904   if (VT.getScalarType() == MVT::i16) {
1905     // FP immediate values are broken.
1906     return isInlinableIntLiteral(Val);
1907   }
1908 
1909   // f16/v2f16 operands work correctly for all values.
1910   return AMDGPU::isInlinableLiteral16(Val, HasInv2Pi);
1911 }
1912 
1913 bool AMDGPUOperand::isInlinableImm(MVT type) const {
1914 
1915   // This is a hack to enable named inline values like
1916   // shared_base with both 32-bit and 64-bit operands.
1917   // Note that these values are defined as
1918   // 32-bit operands only.
1919   if (isInlineValue()) {
1920     return true;
1921   }
1922 
1923   if (!isImmTy(ImmTyNone)) {
1924     // Only plain immediates are inlinable (e.g. "clamp" attribute is not)
1925     return false;
1926   }
1927   // TODO: We should avoid using host float here. It would be better to
1928   // check the float bit values which is what a few other places do.
1929   // We've had bot failures before due to weird NaN support on mips hosts.
1930 
1931   APInt Literal(64, Imm.Val);
1932 
1933   if (Imm.IsFPImm) { // We got fp literal token
1934     if (type == MVT::f64 || type == MVT::i64) { // Expected 64-bit operand
1935       return AMDGPU::isInlinableLiteral64(Imm.Val,
1936                                           AsmParser->hasInv2PiInlineImm());
1937     }
1938 
1939     APFloat FPLiteral(APFloat::IEEEdouble(), APInt(64, Imm.Val));
1940     if (!canLosslesslyConvertToFPType(FPLiteral, type))
1941       return false;
1942 
1943     if (type.getScalarSizeInBits() == 16) {
1944       return isInlineableLiteralOp16(
1945         static_cast<int16_t>(FPLiteral.bitcastToAPInt().getZExtValue()),
1946         type, AsmParser->hasInv2PiInlineImm());
1947     }
1948 
1949     // Check if single precision literal is inlinable
1950     return AMDGPU::isInlinableLiteral32(
1951       static_cast<int32_t>(FPLiteral.bitcastToAPInt().getZExtValue()),
1952       AsmParser->hasInv2PiInlineImm());
1953   }
1954 
1955   // We got int literal token.
1956   if (type == MVT::f64 || type == MVT::i64) { // Expected 64-bit operand
1957     return AMDGPU::isInlinableLiteral64(Imm.Val,
1958                                         AsmParser->hasInv2PiInlineImm());
1959   }
1960 
1961   if (!isSafeTruncation(Imm.Val, type.getScalarSizeInBits())) {
1962     return false;
1963   }
1964 
1965   if (type.getScalarSizeInBits() == 16) {
1966     return isInlineableLiteralOp16(
1967       static_cast<int16_t>(Literal.getLoBits(16).getSExtValue()),
1968       type, AsmParser->hasInv2PiInlineImm());
1969   }
1970 
1971   return AMDGPU::isInlinableLiteral32(
1972     static_cast<int32_t>(Literal.getLoBits(32).getZExtValue()),
1973     AsmParser->hasInv2PiInlineImm());
1974 }
1975 
1976 bool AMDGPUOperand::isLiteralImm(MVT type) const {
1977   // Check that this immediate can be added as literal
1978   if (!isImmTy(ImmTyNone)) {
1979     return false;
1980   }
1981 
1982   if (!Imm.IsFPImm) {
1983     // We got int literal token.
1984 
1985     if (type == MVT::f64 && hasFPModifiers()) {
1986       // Cannot apply fp modifiers to int literals preserving the same semantics
1987       // for VOP1/2/C and VOP3 because of integer truncation. To avoid ambiguity,
1988       // disable these cases.
1989       return false;
1990     }
1991 
1992     unsigned Size = type.getSizeInBits();
1993     if (Size == 64)
1994       Size = 32;
1995 
1996     // FIXME: 64-bit operands can zero extend, sign extend, or pad zeroes for FP
1997     // types.
1998     return isSafeTruncation(Imm.Val, Size);
1999   }
2000 
2001   // We got fp literal token
2002   if (type == MVT::f64) { // Expected 64-bit fp operand
2003     // We would set low 64-bits of literal to zeroes but we accept this literals
2004     return true;
2005   }
2006 
2007   if (type == MVT::i64) { // Expected 64-bit int operand
2008     // We don't allow fp literals in 64-bit integer instructions. It is
2009     // unclear how we should encode them.
2010     return false;
2011   }
2012 
2013   // We allow fp literals with f16x2 operands assuming that the specified
2014   // literal goes into the lower half and the upper half is zero. We also
2015   // require that the literal may be losslessly converted to f16.
2016   MVT ExpectedType = (type == MVT::v2f16)? MVT::f16 :
2017                      (type == MVT::v2i16)? MVT::i16 :
2018                      (type == MVT::v2f32)? MVT::f32 : type;
2019 
2020   APFloat FPLiteral(APFloat::IEEEdouble(), APInt(64, Imm.Val));
2021   return canLosslesslyConvertToFPType(FPLiteral, ExpectedType);
2022 }
2023 
2024 bool AMDGPUOperand::isRegClass(unsigned RCID) const {
2025   return isRegKind() && AsmParser->getMRI()->getRegClass(RCID).contains(getReg());
2026 }
2027 
2028 bool AMDGPUOperand::isVRegWithInputMods() const {
2029   return isRegClass(AMDGPU::VGPR_32RegClassID) ||
2030          // GFX90A allows DPP on 64-bit operands.
2031          (isRegClass(AMDGPU::VReg_64RegClassID) &&
2032           AsmParser->getFeatureBits()[AMDGPU::Feature64BitDPP]);
2033 }
2034 
2035 bool AMDGPUOperand::isSDWAOperand(MVT type) const {
2036   if (AsmParser->isVI())
2037     return isVReg32();
2038   else if (AsmParser->isGFX9Plus())
2039     return isRegClass(AMDGPU::VS_32RegClassID) || isInlinableImm(type);
2040   else
2041     return false;
2042 }
2043 
2044 bool AMDGPUOperand::isSDWAFP16Operand() const {
2045   return isSDWAOperand(MVT::f16);
2046 }
2047 
2048 bool AMDGPUOperand::isSDWAFP32Operand() const {
2049   return isSDWAOperand(MVT::f32);
2050 }
2051 
2052 bool AMDGPUOperand::isSDWAInt16Operand() const {
2053   return isSDWAOperand(MVT::i16);
2054 }
2055 
2056 bool AMDGPUOperand::isSDWAInt32Operand() const {
2057   return isSDWAOperand(MVT::i32);
2058 }
2059 
2060 bool AMDGPUOperand::isBoolReg() const {
2061   auto FB = AsmParser->getFeatureBits();
2062   return isReg() && ((FB[AMDGPU::FeatureWavefrontSize64] && isSCSrcB64()) ||
2063                      (FB[AMDGPU::FeatureWavefrontSize32] && isSCSrcB32()));
2064 }
2065 
2066 uint64_t AMDGPUOperand::applyInputFPModifiers(uint64_t Val, unsigned Size) const
2067 {
2068   assert(isImmTy(ImmTyNone) && Imm.Mods.hasFPModifiers());
2069   assert(Size == 2 || Size == 4 || Size == 8);
2070 
2071   const uint64_t FpSignMask = (1ULL << (Size * 8 - 1));
2072 
2073   if (Imm.Mods.Abs) {
2074     Val &= ~FpSignMask;
2075   }
2076   if (Imm.Mods.Neg) {
2077     Val ^= FpSignMask;
2078   }
2079 
2080   return Val;
2081 }
2082 
2083 void AMDGPUOperand::addImmOperands(MCInst &Inst, unsigned N, bool ApplyModifiers) const {
2084   if (AMDGPU::isSISrcOperand(AsmParser->getMII()->get(Inst.getOpcode()),
2085                              Inst.getNumOperands())) {
2086     addLiteralImmOperand(Inst, Imm.Val,
2087                          ApplyModifiers &
2088                          isImmTy(ImmTyNone) && Imm.Mods.hasFPModifiers());
2089   } else {
2090     assert(!isImmTy(ImmTyNone) || !hasModifiers());
2091     Inst.addOperand(MCOperand::createImm(Imm.Val));
2092     setImmKindNone();
2093   }
2094 }
2095 
2096 void AMDGPUOperand::addLiteralImmOperand(MCInst &Inst, int64_t Val, bool ApplyModifiers) const {
2097   const auto& InstDesc = AsmParser->getMII()->get(Inst.getOpcode());
2098   auto OpNum = Inst.getNumOperands();
2099   // Check that this operand accepts literals
2100   assert(AMDGPU::isSISrcOperand(InstDesc, OpNum));
2101 
2102   if (ApplyModifiers) {
2103     assert(AMDGPU::isSISrcFPOperand(InstDesc, OpNum));
2104     const unsigned Size = Imm.IsFPImm ? sizeof(double) : getOperandSize(InstDesc, OpNum);
2105     Val = applyInputFPModifiers(Val, Size);
2106   }
2107 
2108   APInt Literal(64, Val);
2109   uint8_t OpTy = InstDesc.OpInfo[OpNum].OperandType;
2110 
2111   if (Imm.IsFPImm) { // We got fp literal token
2112     switch (OpTy) {
2113     case AMDGPU::OPERAND_REG_IMM_INT64:
2114     case AMDGPU::OPERAND_REG_IMM_FP64:
2115     case AMDGPU::OPERAND_REG_INLINE_C_INT64:
2116     case AMDGPU::OPERAND_REG_INLINE_C_FP64:
2117     case AMDGPU::OPERAND_REG_INLINE_AC_FP64:
2118       if (AMDGPU::isInlinableLiteral64(Literal.getZExtValue(),
2119                                        AsmParser->hasInv2PiInlineImm())) {
2120         Inst.addOperand(MCOperand::createImm(Literal.getZExtValue()));
2121         setImmKindConst();
2122         return;
2123       }
2124 
2125       // Non-inlineable
2126       if (AMDGPU::isSISrcFPOperand(InstDesc, OpNum)) { // Expected 64-bit fp operand
2127         // For fp operands we check if low 32 bits are zeros
2128         if (Literal.getLoBits(32) != 0) {
2129           const_cast<AMDGPUAsmParser *>(AsmParser)->Warning(Inst.getLoc(),
2130           "Can't encode literal as exact 64-bit floating-point operand. "
2131           "Low 32-bits will be set to zero");
2132         }
2133 
2134         Inst.addOperand(MCOperand::createImm(Literal.lshr(32).getZExtValue()));
2135         setImmKindLiteral();
2136         return;
2137       }
2138 
2139       // We don't allow fp literals in 64-bit integer instructions. It is
2140       // unclear how we should encode them. This case should be checked earlier
2141       // in predicate methods (isLiteralImm())
2142       llvm_unreachable("fp literal in 64-bit integer instruction.");
2143 
2144     case AMDGPU::OPERAND_REG_IMM_INT32:
2145     case AMDGPU::OPERAND_REG_IMM_FP32:
2146     case AMDGPU::OPERAND_REG_IMM_FP32_DEFERRED:
2147     case AMDGPU::OPERAND_REG_INLINE_C_INT32:
2148     case AMDGPU::OPERAND_REG_INLINE_C_FP32:
2149     case AMDGPU::OPERAND_REG_INLINE_AC_INT32:
2150     case AMDGPU::OPERAND_REG_INLINE_AC_FP32:
2151     case AMDGPU::OPERAND_REG_IMM_INT16:
2152     case AMDGPU::OPERAND_REG_IMM_FP16:
2153     case AMDGPU::OPERAND_REG_IMM_FP16_DEFERRED:
2154     case AMDGPU::OPERAND_REG_INLINE_C_INT16:
2155     case AMDGPU::OPERAND_REG_INLINE_C_FP16:
2156     case AMDGPU::OPERAND_REG_INLINE_C_V2INT16:
2157     case AMDGPU::OPERAND_REG_INLINE_C_V2FP16:
2158     case AMDGPU::OPERAND_REG_INLINE_AC_INT16:
2159     case AMDGPU::OPERAND_REG_INLINE_AC_FP16:
2160     case AMDGPU::OPERAND_REG_INLINE_AC_V2INT16:
2161     case AMDGPU::OPERAND_REG_INLINE_AC_V2FP16:
2162     case AMDGPU::OPERAND_REG_IMM_V2INT16:
2163     case AMDGPU::OPERAND_REG_IMM_V2FP16:
2164     case AMDGPU::OPERAND_REG_INLINE_C_V2FP32:
2165     case AMDGPU::OPERAND_REG_IMM_V2FP32:
2166     case AMDGPU::OPERAND_REG_INLINE_C_V2INT32:
2167     case AMDGPU::OPERAND_REG_IMM_V2INT32:
2168     case AMDGPU::OPERAND_KIMM32:
2169     case AMDGPU::OPERAND_KIMM16: {
2170       bool lost;
2171       APFloat FPLiteral(APFloat::IEEEdouble(), Literal);
2172       // Convert literal to single precision
2173       FPLiteral.convert(*getOpFltSemantics(OpTy),
2174                         APFloat::rmNearestTiesToEven, &lost);
2175       // We allow precision lost but not overflow or underflow. This should be
2176       // checked earlier in isLiteralImm()
2177 
2178       uint64_t ImmVal = FPLiteral.bitcastToAPInt().getZExtValue();
2179       Inst.addOperand(MCOperand::createImm(ImmVal));
2180       setImmKindLiteral();
2181       return;
2182     }
2183     default:
2184       llvm_unreachable("invalid operand size");
2185     }
2186 
2187     return;
2188   }
2189 
2190   // We got int literal token.
2191   // Only sign extend inline immediates.
2192   switch (OpTy) {
2193   case AMDGPU::OPERAND_REG_IMM_INT32:
2194   case AMDGPU::OPERAND_REG_IMM_FP32:
2195   case AMDGPU::OPERAND_REG_IMM_FP32_DEFERRED:
2196   case AMDGPU::OPERAND_REG_INLINE_C_INT32:
2197   case AMDGPU::OPERAND_REG_INLINE_C_FP32:
2198   case AMDGPU::OPERAND_REG_INLINE_AC_INT32:
2199   case AMDGPU::OPERAND_REG_INLINE_AC_FP32:
2200   case AMDGPU::OPERAND_REG_IMM_V2INT16:
2201   case AMDGPU::OPERAND_REG_IMM_V2FP16:
2202   case AMDGPU::OPERAND_REG_IMM_V2FP32:
2203   case AMDGPU::OPERAND_REG_INLINE_C_V2FP32:
2204   case AMDGPU::OPERAND_REG_IMM_V2INT32:
2205   case AMDGPU::OPERAND_REG_INLINE_C_V2INT32:
2206     if (isSafeTruncation(Val, 32) &&
2207         AMDGPU::isInlinableLiteral32(static_cast<int32_t>(Val),
2208                                      AsmParser->hasInv2PiInlineImm())) {
2209       Inst.addOperand(MCOperand::createImm(Val));
2210       setImmKindConst();
2211       return;
2212     }
2213 
2214     Inst.addOperand(MCOperand::createImm(Val & 0xffffffff));
2215     setImmKindLiteral();
2216     return;
2217 
2218   case AMDGPU::OPERAND_REG_IMM_INT64:
2219   case AMDGPU::OPERAND_REG_IMM_FP64:
2220   case AMDGPU::OPERAND_REG_INLINE_C_INT64:
2221   case AMDGPU::OPERAND_REG_INLINE_C_FP64:
2222   case AMDGPU::OPERAND_REG_INLINE_AC_FP64:
2223     if (AMDGPU::isInlinableLiteral64(Val, AsmParser->hasInv2PiInlineImm())) {
2224       Inst.addOperand(MCOperand::createImm(Val));
2225       setImmKindConst();
2226       return;
2227     }
2228 
2229     Inst.addOperand(MCOperand::createImm(Lo_32(Val)));
2230     setImmKindLiteral();
2231     return;
2232 
2233   case AMDGPU::OPERAND_REG_IMM_INT16:
2234   case AMDGPU::OPERAND_REG_IMM_FP16:
2235   case AMDGPU::OPERAND_REG_IMM_FP16_DEFERRED:
2236   case AMDGPU::OPERAND_REG_INLINE_C_INT16:
2237   case AMDGPU::OPERAND_REG_INLINE_C_FP16:
2238   case AMDGPU::OPERAND_REG_INLINE_AC_INT16:
2239   case AMDGPU::OPERAND_REG_INLINE_AC_FP16:
2240     if (isSafeTruncation(Val, 16) &&
2241         AMDGPU::isInlinableLiteral16(static_cast<int16_t>(Val),
2242                                      AsmParser->hasInv2PiInlineImm())) {
2243       Inst.addOperand(MCOperand::createImm(Val));
2244       setImmKindConst();
2245       return;
2246     }
2247 
2248     Inst.addOperand(MCOperand::createImm(Val & 0xffff));
2249     setImmKindLiteral();
2250     return;
2251 
2252   case AMDGPU::OPERAND_REG_INLINE_C_V2INT16:
2253   case AMDGPU::OPERAND_REG_INLINE_C_V2FP16:
2254   case AMDGPU::OPERAND_REG_INLINE_AC_V2INT16:
2255   case AMDGPU::OPERAND_REG_INLINE_AC_V2FP16: {
2256     assert(isSafeTruncation(Val, 16));
2257     assert(AMDGPU::isInlinableLiteral16(static_cast<int16_t>(Val),
2258                                         AsmParser->hasInv2PiInlineImm()));
2259 
2260     Inst.addOperand(MCOperand::createImm(Val));
2261     return;
2262   }
2263   case AMDGPU::OPERAND_KIMM32:
2264     Inst.addOperand(MCOperand::createImm(Literal.getLoBits(32).getZExtValue()));
2265     setImmKindNone();
2266     return;
2267   case AMDGPU::OPERAND_KIMM16:
2268     Inst.addOperand(MCOperand::createImm(Literal.getLoBits(16).getZExtValue()));
2269     setImmKindNone();
2270     return;
2271   default:
2272     llvm_unreachable("invalid operand size");
2273   }
2274 }
2275 
2276 template <unsigned Bitwidth>
2277 void AMDGPUOperand::addKImmFPOperands(MCInst &Inst, unsigned N) const {
2278   APInt Literal(64, Imm.Val);
2279   setImmKindNone();
2280 
2281   if (!Imm.IsFPImm) {
2282     // We got int literal token.
2283     Inst.addOperand(MCOperand::createImm(Literal.getLoBits(Bitwidth).getZExtValue()));
2284     return;
2285   }
2286 
2287   bool Lost;
2288   APFloat FPLiteral(APFloat::IEEEdouble(), Literal);
2289   FPLiteral.convert(*getFltSemantics(Bitwidth / 8),
2290                     APFloat::rmNearestTiesToEven, &Lost);
2291   Inst.addOperand(MCOperand::createImm(FPLiteral.bitcastToAPInt().getZExtValue()));
2292 }
2293 
2294 void AMDGPUOperand::addRegOperands(MCInst &Inst, unsigned N) const {
2295   Inst.addOperand(MCOperand::createReg(AMDGPU::getMCReg(getReg(), AsmParser->getSTI())));
2296 }
2297 
2298 static bool isInlineValue(unsigned Reg) {
2299   switch (Reg) {
2300   case AMDGPU::SRC_SHARED_BASE:
2301   case AMDGPU::SRC_SHARED_LIMIT:
2302   case AMDGPU::SRC_PRIVATE_BASE:
2303   case AMDGPU::SRC_PRIVATE_LIMIT:
2304   case AMDGPU::SRC_POPS_EXITING_WAVE_ID:
2305     return true;
2306   case AMDGPU::SRC_VCCZ:
2307   case AMDGPU::SRC_EXECZ:
2308   case AMDGPU::SRC_SCC:
2309     return true;
2310   case AMDGPU::SGPR_NULL:
2311     return true;
2312   default:
2313     return false;
2314   }
2315 }
2316 
2317 bool AMDGPUOperand::isInlineValue() const {
2318   return isRegKind() && ::isInlineValue(getReg());
2319 }
2320 
2321 //===----------------------------------------------------------------------===//
2322 // AsmParser
2323 //===----------------------------------------------------------------------===//
2324 
2325 static int getRegClass(RegisterKind Is, unsigned RegWidth) {
2326   if (Is == IS_VGPR) {
2327     switch (RegWidth) {
2328       default: return -1;
2329       case 32:
2330         return AMDGPU::VGPR_32RegClassID;
2331       case 64:
2332         return AMDGPU::VReg_64RegClassID;
2333       case 96:
2334         return AMDGPU::VReg_96RegClassID;
2335       case 128:
2336         return AMDGPU::VReg_128RegClassID;
2337       case 160:
2338         return AMDGPU::VReg_160RegClassID;
2339       case 192:
2340         return AMDGPU::VReg_192RegClassID;
2341       case 224:
2342         return AMDGPU::VReg_224RegClassID;
2343       case 256:
2344         return AMDGPU::VReg_256RegClassID;
2345       case 512:
2346         return AMDGPU::VReg_512RegClassID;
2347       case 1024:
2348         return AMDGPU::VReg_1024RegClassID;
2349     }
2350   } else if (Is == IS_TTMP) {
2351     switch (RegWidth) {
2352       default: return -1;
2353       case 32:
2354         return AMDGPU::TTMP_32RegClassID;
2355       case 64:
2356         return AMDGPU::TTMP_64RegClassID;
2357       case 128:
2358         return AMDGPU::TTMP_128RegClassID;
2359       case 256:
2360         return AMDGPU::TTMP_256RegClassID;
2361       case 512:
2362         return AMDGPU::TTMP_512RegClassID;
2363     }
2364   } else if (Is == IS_SGPR) {
2365     switch (RegWidth) {
2366       default: return -1;
2367       case 32:
2368         return AMDGPU::SGPR_32RegClassID;
2369       case 64:
2370         return AMDGPU::SGPR_64RegClassID;
2371       case 96:
2372         return AMDGPU::SGPR_96RegClassID;
2373       case 128:
2374         return AMDGPU::SGPR_128RegClassID;
2375       case 160:
2376         return AMDGPU::SGPR_160RegClassID;
2377       case 192:
2378         return AMDGPU::SGPR_192RegClassID;
2379       case 224:
2380         return AMDGPU::SGPR_224RegClassID;
2381       case 256:
2382         return AMDGPU::SGPR_256RegClassID;
2383       case 512:
2384         return AMDGPU::SGPR_512RegClassID;
2385     }
2386   } else if (Is == IS_AGPR) {
2387     switch (RegWidth) {
2388       default: return -1;
2389       case 32:
2390         return AMDGPU::AGPR_32RegClassID;
2391       case 64:
2392         return AMDGPU::AReg_64RegClassID;
2393       case 96:
2394         return AMDGPU::AReg_96RegClassID;
2395       case 128:
2396         return AMDGPU::AReg_128RegClassID;
2397       case 160:
2398         return AMDGPU::AReg_160RegClassID;
2399       case 192:
2400         return AMDGPU::AReg_192RegClassID;
2401       case 224:
2402         return AMDGPU::AReg_224RegClassID;
2403       case 256:
2404         return AMDGPU::AReg_256RegClassID;
2405       case 512:
2406         return AMDGPU::AReg_512RegClassID;
2407       case 1024:
2408         return AMDGPU::AReg_1024RegClassID;
2409     }
2410   }
2411   return -1;
2412 }
2413 
2414 static unsigned getSpecialRegForName(StringRef RegName) {
2415   return StringSwitch<unsigned>(RegName)
2416     .Case("exec", AMDGPU::EXEC)
2417     .Case("vcc", AMDGPU::VCC)
2418     .Case("flat_scratch", AMDGPU::FLAT_SCR)
2419     .Case("xnack_mask", AMDGPU::XNACK_MASK)
2420     .Case("shared_base", AMDGPU::SRC_SHARED_BASE)
2421     .Case("src_shared_base", AMDGPU::SRC_SHARED_BASE)
2422     .Case("shared_limit", AMDGPU::SRC_SHARED_LIMIT)
2423     .Case("src_shared_limit", AMDGPU::SRC_SHARED_LIMIT)
2424     .Case("private_base", AMDGPU::SRC_PRIVATE_BASE)
2425     .Case("src_private_base", AMDGPU::SRC_PRIVATE_BASE)
2426     .Case("private_limit", AMDGPU::SRC_PRIVATE_LIMIT)
2427     .Case("src_private_limit", AMDGPU::SRC_PRIVATE_LIMIT)
2428     .Case("pops_exiting_wave_id", AMDGPU::SRC_POPS_EXITING_WAVE_ID)
2429     .Case("src_pops_exiting_wave_id", AMDGPU::SRC_POPS_EXITING_WAVE_ID)
2430     .Case("lds_direct", AMDGPU::LDS_DIRECT)
2431     .Case("src_lds_direct", AMDGPU::LDS_DIRECT)
2432     .Case("m0", AMDGPU::M0)
2433     .Case("vccz", AMDGPU::SRC_VCCZ)
2434     .Case("src_vccz", AMDGPU::SRC_VCCZ)
2435     .Case("execz", AMDGPU::SRC_EXECZ)
2436     .Case("src_execz", AMDGPU::SRC_EXECZ)
2437     .Case("scc", AMDGPU::SRC_SCC)
2438     .Case("src_scc", AMDGPU::SRC_SCC)
2439     .Case("tba", AMDGPU::TBA)
2440     .Case("tma", AMDGPU::TMA)
2441     .Case("flat_scratch_lo", AMDGPU::FLAT_SCR_LO)
2442     .Case("flat_scratch_hi", AMDGPU::FLAT_SCR_HI)
2443     .Case("xnack_mask_lo", AMDGPU::XNACK_MASK_LO)
2444     .Case("xnack_mask_hi", AMDGPU::XNACK_MASK_HI)
2445     .Case("vcc_lo", AMDGPU::VCC_LO)
2446     .Case("vcc_hi", AMDGPU::VCC_HI)
2447     .Case("exec_lo", AMDGPU::EXEC_LO)
2448     .Case("exec_hi", AMDGPU::EXEC_HI)
2449     .Case("tma_lo", AMDGPU::TMA_LO)
2450     .Case("tma_hi", AMDGPU::TMA_HI)
2451     .Case("tba_lo", AMDGPU::TBA_LO)
2452     .Case("tba_hi", AMDGPU::TBA_HI)
2453     .Case("pc", AMDGPU::PC_REG)
2454     .Case("null", AMDGPU::SGPR_NULL)
2455     .Default(AMDGPU::NoRegister);
2456 }
2457 
2458 bool AMDGPUAsmParser::ParseRegister(unsigned &RegNo, SMLoc &StartLoc,
2459                                     SMLoc &EndLoc, bool RestoreOnFailure) {
2460   auto R = parseRegister();
2461   if (!R) return true;
2462   assert(R->isReg());
2463   RegNo = R->getReg();
2464   StartLoc = R->getStartLoc();
2465   EndLoc = R->getEndLoc();
2466   return false;
2467 }
2468 
2469 bool AMDGPUAsmParser::ParseRegister(unsigned &RegNo, SMLoc &StartLoc,
2470                                     SMLoc &EndLoc) {
2471   return ParseRegister(RegNo, StartLoc, EndLoc, /*RestoreOnFailure=*/false);
2472 }
2473 
2474 OperandMatchResultTy AMDGPUAsmParser::tryParseRegister(unsigned &RegNo,
2475                                                        SMLoc &StartLoc,
2476                                                        SMLoc &EndLoc) {
2477   bool Result =
2478       ParseRegister(RegNo, StartLoc, EndLoc, /*RestoreOnFailure=*/true);
2479   bool PendingErrors = getParser().hasPendingError();
2480   getParser().clearPendingErrors();
2481   if (PendingErrors)
2482     return MatchOperand_ParseFail;
2483   if (Result)
2484     return MatchOperand_NoMatch;
2485   return MatchOperand_Success;
2486 }
2487 
2488 bool AMDGPUAsmParser::AddNextRegisterToList(unsigned &Reg, unsigned &RegWidth,
2489                                             RegisterKind RegKind, unsigned Reg1,
2490                                             SMLoc Loc) {
2491   switch (RegKind) {
2492   case IS_SPECIAL:
2493     if (Reg == AMDGPU::EXEC_LO && Reg1 == AMDGPU::EXEC_HI) {
2494       Reg = AMDGPU::EXEC;
2495       RegWidth = 64;
2496       return true;
2497     }
2498     if (Reg == AMDGPU::FLAT_SCR_LO && Reg1 == AMDGPU::FLAT_SCR_HI) {
2499       Reg = AMDGPU::FLAT_SCR;
2500       RegWidth = 64;
2501       return true;
2502     }
2503     if (Reg == AMDGPU::XNACK_MASK_LO && Reg1 == AMDGPU::XNACK_MASK_HI) {
2504       Reg = AMDGPU::XNACK_MASK;
2505       RegWidth = 64;
2506       return true;
2507     }
2508     if (Reg == AMDGPU::VCC_LO && Reg1 == AMDGPU::VCC_HI) {
2509       Reg = AMDGPU::VCC;
2510       RegWidth = 64;
2511       return true;
2512     }
2513     if (Reg == AMDGPU::TBA_LO && Reg1 == AMDGPU::TBA_HI) {
2514       Reg = AMDGPU::TBA;
2515       RegWidth = 64;
2516       return true;
2517     }
2518     if (Reg == AMDGPU::TMA_LO && Reg1 == AMDGPU::TMA_HI) {
2519       Reg = AMDGPU::TMA;
2520       RegWidth = 64;
2521       return true;
2522     }
2523     Error(Loc, "register does not fit in the list");
2524     return false;
2525   case IS_VGPR:
2526   case IS_SGPR:
2527   case IS_AGPR:
2528   case IS_TTMP:
2529     if (Reg1 != Reg + RegWidth / 32) {
2530       Error(Loc, "registers in a list must have consecutive indices");
2531       return false;
2532     }
2533     RegWidth += 32;
2534     return true;
2535   default:
2536     llvm_unreachable("unexpected register kind");
2537   }
2538 }
2539 
2540 struct RegInfo {
2541   StringLiteral Name;
2542   RegisterKind Kind;
2543 };
2544 
2545 static constexpr RegInfo RegularRegisters[] = {
2546   {{"v"},    IS_VGPR},
2547   {{"s"},    IS_SGPR},
2548   {{"ttmp"}, IS_TTMP},
2549   {{"acc"},  IS_AGPR},
2550   {{"a"},    IS_AGPR},
2551 };
2552 
2553 static bool isRegularReg(RegisterKind Kind) {
2554   return Kind == IS_VGPR ||
2555          Kind == IS_SGPR ||
2556          Kind == IS_TTMP ||
2557          Kind == IS_AGPR;
2558 }
2559 
2560 static const RegInfo* getRegularRegInfo(StringRef Str) {
2561   for (const RegInfo &Reg : RegularRegisters)
2562     if (Str.startswith(Reg.Name))
2563       return &Reg;
2564   return nullptr;
2565 }
2566 
2567 static bool getRegNum(StringRef Str, unsigned& Num) {
2568   return !Str.getAsInteger(10, Num);
2569 }
2570 
2571 bool
2572 AMDGPUAsmParser::isRegister(const AsmToken &Token,
2573                             const AsmToken &NextToken) const {
2574 
2575   // A list of consecutive registers: [s0,s1,s2,s3]
2576   if (Token.is(AsmToken::LBrac))
2577     return true;
2578 
2579   if (!Token.is(AsmToken::Identifier))
2580     return false;
2581 
2582   // A single register like s0 or a range of registers like s[0:1]
2583 
2584   StringRef Str = Token.getString();
2585   const RegInfo *Reg = getRegularRegInfo(Str);
2586   if (Reg) {
2587     StringRef RegName = Reg->Name;
2588     StringRef RegSuffix = Str.substr(RegName.size());
2589     if (!RegSuffix.empty()) {
2590       unsigned Num;
2591       // A single register with an index: rXX
2592       if (getRegNum(RegSuffix, Num))
2593         return true;
2594     } else {
2595       // A range of registers: r[XX:YY].
2596       if (NextToken.is(AsmToken::LBrac))
2597         return true;
2598     }
2599   }
2600 
2601   return getSpecialRegForName(Str) != AMDGPU::NoRegister;
2602 }
2603 
2604 bool
2605 AMDGPUAsmParser::isRegister()
2606 {
2607   return isRegister(getToken(), peekToken());
2608 }
2609 
2610 unsigned
2611 AMDGPUAsmParser::getRegularReg(RegisterKind RegKind,
2612                                unsigned RegNum,
2613                                unsigned RegWidth,
2614                                SMLoc Loc) {
2615 
2616   assert(isRegularReg(RegKind));
2617 
2618   unsigned AlignSize = 1;
2619   if (RegKind == IS_SGPR || RegKind == IS_TTMP) {
2620     // SGPR and TTMP registers must be aligned.
2621     // Max required alignment is 4 dwords.
2622     AlignSize = std::min(RegWidth / 32, 4u);
2623   }
2624 
2625   if (RegNum % AlignSize != 0) {
2626     Error(Loc, "invalid register alignment");
2627     return AMDGPU::NoRegister;
2628   }
2629 
2630   unsigned RegIdx = RegNum / AlignSize;
2631   int RCID = getRegClass(RegKind, RegWidth);
2632   if (RCID == -1) {
2633     Error(Loc, "invalid or unsupported register size");
2634     return AMDGPU::NoRegister;
2635   }
2636 
2637   const MCRegisterInfo *TRI = getContext().getRegisterInfo();
2638   const MCRegisterClass RC = TRI->getRegClass(RCID);
2639   if (RegIdx >= RC.getNumRegs()) {
2640     Error(Loc, "register index is out of range");
2641     return AMDGPU::NoRegister;
2642   }
2643 
2644   return RC.getRegister(RegIdx);
2645 }
2646 
2647 bool AMDGPUAsmParser::ParseRegRange(unsigned &Num, unsigned &RegWidth) {
2648   int64_t RegLo, RegHi;
2649   if (!skipToken(AsmToken::LBrac, "missing register index"))
2650     return false;
2651 
2652   SMLoc FirstIdxLoc = getLoc();
2653   SMLoc SecondIdxLoc;
2654 
2655   if (!parseExpr(RegLo))
2656     return false;
2657 
2658   if (trySkipToken(AsmToken::Colon)) {
2659     SecondIdxLoc = getLoc();
2660     if (!parseExpr(RegHi))
2661       return false;
2662   } else {
2663     RegHi = RegLo;
2664   }
2665 
2666   if (!skipToken(AsmToken::RBrac, "expected a closing square bracket"))
2667     return false;
2668 
2669   if (!isUInt<32>(RegLo)) {
2670     Error(FirstIdxLoc, "invalid register index");
2671     return false;
2672   }
2673 
2674   if (!isUInt<32>(RegHi)) {
2675     Error(SecondIdxLoc, "invalid register index");
2676     return false;
2677   }
2678 
2679   if (RegLo > RegHi) {
2680     Error(FirstIdxLoc, "first register index should not exceed second index");
2681     return false;
2682   }
2683 
2684   Num = static_cast<unsigned>(RegLo);
2685   RegWidth = 32 * ((RegHi - RegLo) + 1);
2686   return true;
2687 }
2688 
2689 unsigned AMDGPUAsmParser::ParseSpecialReg(RegisterKind &RegKind,
2690                                           unsigned &RegNum, unsigned &RegWidth,
2691                                           SmallVectorImpl<AsmToken> &Tokens) {
2692   assert(isToken(AsmToken::Identifier));
2693   unsigned Reg = getSpecialRegForName(getTokenStr());
2694   if (Reg) {
2695     RegNum = 0;
2696     RegWidth = 32;
2697     RegKind = IS_SPECIAL;
2698     Tokens.push_back(getToken());
2699     lex(); // skip register name
2700   }
2701   return Reg;
2702 }
2703 
2704 unsigned AMDGPUAsmParser::ParseRegularReg(RegisterKind &RegKind,
2705                                           unsigned &RegNum, unsigned &RegWidth,
2706                                           SmallVectorImpl<AsmToken> &Tokens) {
2707   assert(isToken(AsmToken::Identifier));
2708   StringRef RegName = getTokenStr();
2709   auto Loc = getLoc();
2710 
2711   const RegInfo *RI = getRegularRegInfo(RegName);
2712   if (!RI) {
2713     Error(Loc, "invalid register name");
2714     return AMDGPU::NoRegister;
2715   }
2716 
2717   Tokens.push_back(getToken());
2718   lex(); // skip register name
2719 
2720   RegKind = RI->Kind;
2721   StringRef RegSuffix = RegName.substr(RI->Name.size());
2722   if (!RegSuffix.empty()) {
2723     // Single 32-bit register: vXX.
2724     if (!getRegNum(RegSuffix, RegNum)) {
2725       Error(Loc, "invalid register index");
2726       return AMDGPU::NoRegister;
2727     }
2728     RegWidth = 32;
2729   } else {
2730     // Range of registers: v[XX:YY]. ":YY" is optional.
2731     if (!ParseRegRange(RegNum, RegWidth))
2732       return AMDGPU::NoRegister;
2733   }
2734 
2735   return getRegularReg(RegKind, RegNum, RegWidth, Loc);
2736 }
2737 
2738 unsigned AMDGPUAsmParser::ParseRegList(RegisterKind &RegKind, unsigned &RegNum,
2739                                        unsigned &RegWidth,
2740                                        SmallVectorImpl<AsmToken> &Tokens) {
2741   unsigned Reg = AMDGPU::NoRegister;
2742   auto ListLoc = getLoc();
2743 
2744   if (!skipToken(AsmToken::LBrac,
2745                  "expected a register or a list of registers")) {
2746     return AMDGPU::NoRegister;
2747   }
2748 
2749   // List of consecutive registers, e.g.: [s0,s1,s2,s3]
2750 
2751   auto Loc = getLoc();
2752   if (!ParseAMDGPURegister(RegKind, Reg, RegNum, RegWidth))
2753     return AMDGPU::NoRegister;
2754   if (RegWidth != 32) {
2755     Error(Loc, "expected a single 32-bit register");
2756     return AMDGPU::NoRegister;
2757   }
2758 
2759   for (; trySkipToken(AsmToken::Comma); ) {
2760     RegisterKind NextRegKind;
2761     unsigned NextReg, NextRegNum, NextRegWidth;
2762     Loc = getLoc();
2763 
2764     if (!ParseAMDGPURegister(NextRegKind, NextReg,
2765                              NextRegNum, NextRegWidth,
2766                              Tokens)) {
2767       return AMDGPU::NoRegister;
2768     }
2769     if (NextRegWidth != 32) {
2770       Error(Loc, "expected a single 32-bit register");
2771       return AMDGPU::NoRegister;
2772     }
2773     if (NextRegKind != RegKind) {
2774       Error(Loc, "registers in a list must be of the same kind");
2775       return AMDGPU::NoRegister;
2776     }
2777     if (!AddNextRegisterToList(Reg, RegWidth, RegKind, NextReg, Loc))
2778       return AMDGPU::NoRegister;
2779   }
2780 
2781   if (!skipToken(AsmToken::RBrac,
2782                  "expected a comma or a closing square bracket")) {
2783     return AMDGPU::NoRegister;
2784   }
2785 
2786   if (isRegularReg(RegKind))
2787     Reg = getRegularReg(RegKind, RegNum, RegWidth, ListLoc);
2788 
2789   return Reg;
2790 }
2791 
2792 bool AMDGPUAsmParser::ParseAMDGPURegister(RegisterKind &RegKind, unsigned &Reg,
2793                                           unsigned &RegNum, unsigned &RegWidth,
2794                                           SmallVectorImpl<AsmToken> &Tokens) {
2795   auto Loc = getLoc();
2796   Reg = AMDGPU::NoRegister;
2797 
2798   if (isToken(AsmToken::Identifier)) {
2799     Reg = ParseSpecialReg(RegKind, RegNum, RegWidth, Tokens);
2800     if (Reg == AMDGPU::NoRegister)
2801       Reg = ParseRegularReg(RegKind, RegNum, RegWidth, Tokens);
2802   } else {
2803     Reg = ParseRegList(RegKind, RegNum, RegWidth, Tokens);
2804   }
2805 
2806   const MCRegisterInfo *TRI = getContext().getRegisterInfo();
2807   if (Reg == AMDGPU::NoRegister) {
2808     assert(Parser.hasPendingError());
2809     return false;
2810   }
2811 
2812   if (!subtargetHasRegister(*TRI, Reg)) {
2813     if (Reg == AMDGPU::SGPR_NULL) {
2814       Error(Loc, "'null' operand is not supported on this GPU");
2815     } else {
2816       Error(Loc, "register not available on this GPU");
2817     }
2818     return false;
2819   }
2820 
2821   return true;
2822 }
2823 
2824 bool AMDGPUAsmParser::ParseAMDGPURegister(RegisterKind &RegKind, unsigned &Reg,
2825                                           unsigned &RegNum, unsigned &RegWidth,
2826                                           bool RestoreOnFailure /*=false*/) {
2827   Reg = AMDGPU::NoRegister;
2828 
2829   SmallVector<AsmToken, 1> Tokens;
2830   if (ParseAMDGPURegister(RegKind, Reg, RegNum, RegWidth, Tokens)) {
2831     if (RestoreOnFailure) {
2832       while (!Tokens.empty()) {
2833         getLexer().UnLex(Tokens.pop_back_val());
2834       }
2835     }
2836     return true;
2837   }
2838   return false;
2839 }
2840 
2841 Optional<StringRef>
2842 AMDGPUAsmParser::getGprCountSymbolName(RegisterKind RegKind) {
2843   switch (RegKind) {
2844   case IS_VGPR:
2845     return StringRef(".amdgcn.next_free_vgpr");
2846   case IS_SGPR:
2847     return StringRef(".amdgcn.next_free_sgpr");
2848   default:
2849     return None;
2850   }
2851 }
2852 
2853 void AMDGPUAsmParser::initializeGprCountSymbol(RegisterKind RegKind) {
2854   auto SymbolName = getGprCountSymbolName(RegKind);
2855   assert(SymbolName && "initializing invalid register kind");
2856   MCSymbol *Sym = getContext().getOrCreateSymbol(*SymbolName);
2857   Sym->setVariableValue(MCConstantExpr::create(0, getContext()));
2858 }
2859 
2860 bool AMDGPUAsmParser::updateGprCountSymbols(RegisterKind RegKind,
2861                                             unsigned DwordRegIndex,
2862                                             unsigned RegWidth) {
2863   // Symbols are only defined for GCN targets
2864   if (AMDGPU::getIsaVersion(getSTI().getCPU()).Major < 6)
2865     return true;
2866 
2867   auto SymbolName = getGprCountSymbolName(RegKind);
2868   if (!SymbolName)
2869     return true;
2870   MCSymbol *Sym = getContext().getOrCreateSymbol(*SymbolName);
2871 
2872   int64_t NewMax = DwordRegIndex + divideCeil(RegWidth, 32) - 1;
2873   int64_t OldCount;
2874 
2875   if (!Sym->isVariable())
2876     return !Error(getLoc(),
2877                   ".amdgcn.next_free_{v,s}gpr symbols must be variable");
2878   if (!Sym->getVariableValue(false)->evaluateAsAbsolute(OldCount))
2879     return !Error(
2880         getLoc(),
2881         ".amdgcn.next_free_{v,s}gpr symbols must be absolute expressions");
2882 
2883   if (OldCount <= NewMax)
2884     Sym->setVariableValue(MCConstantExpr::create(NewMax + 1, getContext()));
2885 
2886   return true;
2887 }
2888 
2889 std::unique_ptr<AMDGPUOperand>
2890 AMDGPUAsmParser::parseRegister(bool RestoreOnFailure) {
2891   const auto &Tok = getToken();
2892   SMLoc StartLoc = Tok.getLoc();
2893   SMLoc EndLoc = Tok.getEndLoc();
2894   RegisterKind RegKind;
2895   unsigned Reg, RegNum, RegWidth;
2896 
2897   if (!ParseAMDGPURegister(RegKind, Reg, RegNum, RegWidth)) {
2898     return nullptr;
2899   }
2900   if (isHsaAbiVersion3AndAbove(&getSTI())) {
2901     if (!updateGprCountSymbols(RegKind, RegNum, RegWidth))
2902       return nullptr;
2903   } else
2904     KernelScope.usesRegister(RegKind, RegNum, RegWidth);
2905   return AMDGPUOperand::CreateReg(this, Reg, StartLoc, EndLoc);
2906 }
2907 
2908 OperandMatchResultTy
2909 AMDGPUAsmParser::parseImm(OperandVector &Operands, bool HasSP3AbsModifier) {
2910   // TODO: add syntactic sugar for 1/(2*PI)
2911 
2912   assert(!isRegister());
2913   assert(!isModifier());
2914 
2915   const auto& Tok = getToken();
2916   const auto& NextTok = peekToken();
2917   bool IsReal = Tok.is(AsmToken::Real);
2918   SMLoc S = getLoc();
2919   bool Negate = false;
2920 
2921   if (!IsReal && Tok.is(AsmToken::Minus) && NextTok.is(AsmToken::Real)) {
2922     lex();
2923     IsReal = true;
2924     Negate = true;
2925   }
2926 
2927   if (IsReal) {
2928     // Floating-point expressions are not supported.
2929     // Can only allow floating-point literals with an
2930     // optional sign.
2931 
2932     StringRef Num = getTokenStr();
2933     lex();
2934 
2935     APFloat RealVal(APFloat::IEEEdouble());
2936     auto roundMode = APFloat::rmNearestTiesToEven;
2937     if (errorToBool(RealVal.convertFromString(Num, roundMode).takeError())) {
2938       return MatchOperand_ParseFail;
2939     }
2940     if (Negate)
2941       RealVal.changeSign();
2942 
2943     Operands.push_back(
2944       AMDGPUOperand::CreateImm(this, RealVal.bitcastToAPInt().getZExtValue(), S,
2945                                AMDGPUOperand::ImmTyNone, true));
2946 
2947     return MatchOperand_Success;
2948 
2949   } else {
2950     int64_t IntVal;
2951     const MCExpr *Expr;
2952     SMLoc S = getLoc();
2953 
2954     if (HasSP3AbsModifier) {
2955       // This is a workaround for handling expressions
2956       // as arguments of SP3 'abs' modifier, for example:
2957       //     |1.0|
2958       //     |-1|
2959       //     |1+x|
2960       // This syntax is not compatible with syntax of standard
2961       // MC expressions (due to the trailing '|').
2962       SMLoc EndLoc;
2963       if (getParser().parsePrimaryExpr(Expr, EndLoc, nullptr))
2964         return MatchOperand_ParseFail;
2965     } else {
2966       if (Parser.parseExpression(Expr))
2967         return MatchOperand_ParseFail;
2968     }
2969 
2970     if (Expr->evaluateAsAbsolute(IntVal)) {
2971       Operands.push_back(AMDGPUOperand::CreateImm(this, IntVal, S));
2972     } else {
2973       Operands.push_back(AMDGPUOperand::CreateExpr(this, Expr, S));
2974     }
2975 
2976     return MatchOperand_Success;
2977   }
2978 
2979   return MatchOperand_NoMatch;
2980 }
2981 
2982 OperandMatchResultTy
2983 AMDGPUAsmParser::parseReg(OperandVector &Operands) {
2984   if (!isRegister())
2985     return MatchOperand_NoMatch;
2986 
2987   if (auto R = parseRegister()) {
2988     assert(R->isReg());
2989     Operands.push_back(std::move(R));
2990     return MatchOperand_Success;
2991   }
2992   return MatchOperand_ParseFail;
2993 }
2994 
2995 OperandMatchResultTy
2996 AMDGPUAsmParser::parseRegOrImm(OperandVector &Operands, bool HasSP3AbsMod) {
2997   auto res = parseReg(Operands);
2998   if (res != MatchOperand_NoMatch) {
2999     return res;
3000   } else if (isModifier()) {
3001     return MatchOperand_NoMatch;
3002   } else {
3003     return parseImm(Operands, HasSP3AbsMod);
3004   }
3005 }
3006 
3007 bool
3008 AMDGPUAsmParser::isNamedOperandModifier(const AsmToken &Token, const AsmToken &NextToken) const {
3009   if (Token.is(AsmToken::Identifier) && NextToken.is(AsmToken::LParen)) {
3010     const auto &str = Token.getString();
3011     return str == "abs" || str == "neg" || str == "sext";
3012   }
3013   return false;
3014 }
3015 
3016 bool
3017 AMDGPUAsmParser::isOpcodeModifierWithVal(const AsmToken &Token, const AsmToken &NextToken) const {
3018   return Token.is(AsmToken::Identifier) && NextToken.is(AsmToken::Colon);
3019 }
3020 
3021 bool
3022 AMDGPUAsmParser::isOperandModifier(const AsmToken &Token, const AsmToken &NextToken) const {
3023   return isNamedOperandModifier(Token, NextToken) || Token.is(AsmToken::Pipe);
3024 }
3025 
3026 bool
3027 AMDGPUAsmParser::isRegOrOperandModifier(const AsmToken &Token, const AsmToken &NextToken) const {
3028   return isRegister(Token, NextToken) || isOperandModifier(Token, NextToken);
3029 }
3030 
3031 // Check if this is an operand modifier or an opcode modifier
3032 // which may look like an expression but it is not. We should
3033 // avoid parsing these modifiers as expressions. Currently
3034 // recognized sequences are:
3035 //   |...|
3036 //   abs(...)
3037 //   neg(...)
3038 //   sext(...)
3039 //   -reg
3040 //   -|...|
3041 //   -abs(...)
3042 //   name:...
3043 // Note that simple opcode modifiers like 'gds' may be parsed as
3044 // expressions; this is a special case. See getExpressionAsToken.
3045 //
3046 bool
3047 AMDGPUAsmParser::isModifier() {
3048 
3049   AsmToken Tok = getToken();
3050   AsmToken NextToken[2];
3051   peekTokens(NextToken);
3052 
3053   return isOperandModifier(Tok, NextToken[0]) ||
3054          (Tok.is(AsmToken::Minus) && isRegOrOperandModifier(NextToken[0], NextToken[1])) ||
3055          isOpcodeModifierWithVal(Tok, NextToken[0]);
3056 }
3057 
3058 // Check if the current token is an SP3 'neg' modifier.
3059 // Currently this modifier is allowed in the following context:
3060 //
3061 // 1. Before a register, e.g. "-v0", "-v[...]" or "-[v0,v1]".
3062 // 2. Before an 'abs' modifier: -abs(...)
3063 // 3. Before an SP3 'abs' modifier: -|...|
3064 //
3065 // In all other cases "-" is handled as a part
3066 // of an expression that follows the sign.
3067 //
3068 // Note: When "-" is followed by an integer literal,
3069 // this is interpreted as integer negation rather
3070 // than a floating-point NEG modifier applied to N.
3071 // Beside being contr-intuitive, such use of floating-point
3072 // NEG modifier would have resulted in different meaning
3073 // of integer literals used with VOP1/2/C and VOP3,
3074 // for example:
3075 //    v_exp_f32_e32 v5, -1 // VOP1: src0 = 0xFFFFFFFF
3076 //    v_exp_f32_e64 v5, -1 // VOP3: src0 = 0x80000001
3077 // Negative fp literals with preceding "-" are
3078 // handled likewise for uniformity
3079 //
3080 bool
3081 AMDGPUAsmParser::parseSP3NegModifier() {
3082 
3083   AsmToken NextToken[2];
3084   peekTokens(NextToken);
3085 
3086   if (isToken(AsmToken::Minus) &&
3087       (isRegister(NextToken[0], NextToken[1]) ||
3088        NextToken[0].is(AsmToken::Pipe) ||
3089        isId(NextToken[0], "abs"))) {
3090     lex();
3091     return true;
3092   }
3093 
3094   return false;
3095 }
3096 
3097 OperandMatchResultTy
3098 AMDGPUAsmParser::parseRegOrImmWithFPInputMods(OperandVector &Operands,
3099                                               bool AllowImm) {
3100   bool Neg, SP3Neg;
3101   bool Abs, SP3Abs;
3102   SMLoc Loc;
3103 
3104   // Disable ambiguous constructs like '--1' etc. Should use neg(-1) instead.
3105   if (isToken(AsmToken::Minus) && peekToken().is(AsmToken::Minus)) {
3106     Error(getLoc(), "invalid syntax, expected 'neg' modifier");
3107     return MatchOperand_ParseFail;
3108   }
3109 
3110   SP3Neg = parseSP3NegModifier();
3111 
3112   Loc = getLoc();
3113   Neg = trySkipId("neg");
3114   if (Neg && SP3Neg) {
3115     Error(Loc, "expected register or immediate");
3116     return MatchOperand_ParseFail;
3117   }
3118   if (Neg && !skipToken(AsmToken::LParen, "expected left paren after neg"))
3119     return MatchOperand_ParseFail;
3120 
3121   Abs = trySkipId("abs");
3122   if (Abs && !skipToken(AsmToken::LParen, "expected left paren after abs"))
3123     return MatchOperand_ParseFail;
3124 
3125   Loc = getLoc();
3126   SP3Abs = trySkipToken(AsmToken::Pipe);
3127   if (Abs && SP3Abs) {
3128     Error(Loc, "expected register or immediate");
3129     return MatchOperand_ParseFail;
3130   }
3131 
3132   OperandMatchResultTy Res;
3133   if (AllowImm) {
3134     Res = parseRegOrImm(Operands, SP3Abs);
3135   } else {
3136     Res = parseReg(Operands);
3137   }
3138   if (Res != MatchOperand_Success) {
3139     return (SP3Neg || Neg || SP3Abs || Abs)? MatchOperand_ParseFail : Res;
3140   }
3141 
3142   if (SP3Abs && !skipToken(AsmToken::Pipe, "expected vertical bar"))
3143     return MatchOperand_ParseFail;
3144   if (Abs && !skipToken(AsmToken::RParen, "expected closing parentheses"))
3145     return MatchOperand_ParseFail;
3146   if (Neg && !skipToken(AsmToken::RParen, "expected closing parentheses"))
3147     return MatchOperand_ParseFail;
3148 
3149   AMDGPUOperand::Modifiers Mods;
3150   Mods.Abs = Abs || SP3Abs;
3151   Mods.Neg = Neg || SP3Neg;
3152 
3153   if (Mods.hasFPModifiers()) {
3154     AMDGPUOperand &Op = static_cast<AMDGPUOperand &>(*Operands.back());
3155     if (Op.isExpr()) {
3156       Error(Op.getStartLoc(), "expected an absolute expression");
3157       return MatchOperand_ParseFail;
3158     }
3159     Op.setModifiers(Mods);
3160   }
3161   return MatchOperand_Success;
3162 }
3163 
3164 OperandMatchResultTy
3165 AMDGPUAsmParser::parseRegOrImmWithIntInputMods(OperandVector &Operands,
3166                                                bool AllowImm) {
3167   bool Sext = trySkipId("sext");
3168   if (Sext && !skipToken(AsmToken::LParen, "expected left paren after sext"))
3169     return MatchOperand_ParseFail;
3170 
3171   OperandMatchResultTy Res;
3172   if (AllowImm) {
3173     Res = parseRegOrImm(Operands);
3174   } else {
3175     Res = parseReg(Operands);
3176   }
3177   if (Res != MatchOperand_Success) {
3178     return Sext? MatchOperand_ParseFail : Res;
3179   }
3180 
3181   if (Sext && !skipToken(AsmToken::RParen, "expected closing parentheses"))
3182     return MatchOperand_ParseFail;
3183 
3184   AMDGPUOperand::Modifiers Mods;
3185   Mods.Sext = Sext;
3186 
3187   if (Mods.hasIntModifiers()) {
3188     AMDGPUOperand &Op = static_cast<AMDGPUOperand &>(*Operands.back());
3189     if (Op.isExpr()) {
3190       Error(Op.getStartLoc(), "expected an absolute expression");
3191       return MatchOperand_ParseFail;
3192     }
3193     Op.setModifiers(Mods);
3194   }
3195 
3196   return MatchOperand_Success;
3197 }
3198 
3199 OperandMatchResultTy
3200 AMDGPUAsmParser::parseRegWithFPInputMods(OperandVector &Operands) {
3201   return parseRegOrImmWithFPInputMods(Operands, false);
3202 }
3203 
3204 OperandMatchResultTy
3205 AMDGPUAsmParser::parseRegWithIntInputMods(OperandVector &Operands) {
3206   return parseRegOrImmWithIntInputMods(Operands, false);
3207 }
3208 
3209 OperandMatchResultTy AMDGPUAsmParser::parseVReg32OrOff(OperandVector &Operands) {
3210   auto Loc = getLoc();
3211   if (trySkipId("off")) {
3212     Operands.push_back(AMDGPUOperand::CreateImm(this, 0, Loc,
3213                                                 AMDGPUOperand::ImmTyOff, false));
3214     return MatchOperand_Success;
3215   }
3216 
3217   if (!isRegister())
3218     return MatchOperand_NoMatch;
3219 
3220   std::unique_ptr<AMDGPUOperand> Reg = parseRegister();
3221   if (Reg) {
3222     Operands.push_back(std::move(Reg));
3223     return MatchOperand_Success;
3224   }
3225 
3226   return MatchOperand_ParseFail;
3227 
3228 }
3229 
3230 unsigned AMDGPUAsmParser::checkTargetMatchPredicate(MCInst &Inst) {
3231   uint64_t TSFlags = MII.get(Inst.getOpcode()).TSFlags;
3232 
3233   if ((getForcedEncodingSize() == 32 && (TSFlags & SIInstrFlags::VOP3)) ||
3234       (getForcedEncodingSize() == 64 && !(TSFlags & SIInstrFlags::VOP3)) ||
3235       (isForcedDPP() && !(TSFlags & SIInstrFlags::DPP)) ||
3236       (isForcedSDWA() && !(TSFlags & SIInstrFlags::SDWA)) )
3237     return Match_InvalidOperand;
3238 
3239   if ((TSFlags & SIInstrFlags::VOP3) &&
3240       (TSFlags & SIInstrFlags::VOPAsmPrefer32Bit) &&
3241       getForcedEncodingSize() != 64)
3242     return Match_PreferE32;
3243 
3244   if (Inst.getOpcode() == AMDGPU::V_MAC_F32_sdwa_vi ||
3245       Inst.getOpcode() == AMDGPU::V_MAC_F16_sdwa_vi) {
3246     // v_mac_f32/16 allow only dst_sel == DWORD;
3247     auto OpNum =
3248         AMDGPU::getNamedOperandIdx(Inst.getOpcode(), AMDGPU::OpName::dst_sel);
3249     const auto &Op = Inst.getOperand(OpNum);
3250     if (!Op.isImm() || Op.getImm() != AMDGPU::SDWA::SdwaSel::DWORD) {
3251       return Match_InvalidOperand;
3252     }
3253   }
3254 
3255   return Match_Success;
3256 }
3257 
3258 static ArrayRef<unsigned> getAllVariants() {
3259   static const unsigned Variants[] = {
3260     AMDGPUAsmVariants::DEFAULT, AMDGPUAsmVariants::VOP3,
3261     AMDGPUAsmVariants::SDWA, AMDGPUAsmVariants::SDWA9,
3262     AMDGPUAsmVariants::DPP, AMDGPUAsmVariants::VOP3_DPP
3263   };
3264 
3265   return makeArrayRef(Variants);
3266 }
3267 
3268 // What asm variants we should check
3269 ArrayRef<unsigned> AMDGPUAsmParser::getMatchedVariants() const {
3270   if (isForcedDPP() && isForcedVOP3()) {
3271     static const unsigned Variants[] = {AMDGPUAsmVariants::VOP3_DPP};
3272     return makeArrayRef(Variants);
3273   }
3274   if (getForcedEncodingSize() == 32) {
3275     static const unsigned Variants[] = {AMDGPUAsmVariants::DEFAULT};
3276     return makeArrayRef(Variants);
3277   }
3278 
3279   if (isForcedVOP3()) {
3280     static const unsigned Variants[] = {AMDGPUAsmVariants::VOP3};
3281     return makeArrayRef(Variants);
3282   }
3283 
3284   if (isForcedSDWA()) {
3285     static const unsigned Variants[] = {AMDGPUAsmVariants::SDWA,
3286                                         AMDGPUAsmVariants::SDWA9};
3287     return makeArrayRef(Variants);
3288   }
3289 
3290   if (isForcedDPP()) {
3291     static const unsigned Variants[] = {AMDGPUAsmVariants::DPP};
3292     return makeArrayRef(Variants);
3293   }
3294 
3295   return getAllVariants();
3296 }
3297 
3298 StringRef AMDGPUAsmParser::getMatchedVariantName() const {
3299   if (isForcedDPP() && isForcedVOP3())
3300     return "e64_dpp";
3301 
3302   if (getForcedEncodingSize() == 32)
3303     return "e32";
3304 
3305   if (isForcedVOP3())
3306     return "e64";
3307 
3308   if (isForcedSDWA())
3309     return "sdwa";
3310 
3311   if (isForcedDPP())
3312     return "dpp";
3313 
3314   return "";
3315 }
3316 
3317 unsigned AMDGPUAsmParser::findImplicitSGPRReadInVOP(const MCInst &Inst) const {
3318   const MCInstrDesc &Desc = MII.get(Inst.getOpcode());
3319   const unsigned Num = Desc.getNumImplicitUses();
3320   for (unsigned i = 0; i < Num; ++i) {
3321     unsigned Reg = Desc.ImplicitUses[i];
3322     switch (Reg) {
3323     case AMDGPU::FLAT_SCR:
3324     case AMDGPU::VCC:
3325     case AMDGPU::VCC_LO:
3326     case AMDGPU::VCC_HI:
3327     case AMDGPU::M0:
3328       return Reg;
3329     default:
3330       break;
3331     }
3332   }
3333   return AMDGPU::NoRegister;
3334 }
3335 
3336 // NB: This code is correct only when used to check constant
3337 // bus limitations because GFX7 support no f16 inline constants.
3338 // Note that there are no cases when a GFX7 opcode violates
3339 // constant bus limitations due to the use of an f16 constant.
3340 bool AMDGPUAsmParser::isInlineConstant(const MCInst &Inst,
3341                                        unsigned OpIdx) const {
3342   const MCInstrDesc &Desc = MII.get(Inst.getOpcode());
3343 
3344   if (!AMDGPU::isSISrcOperand(Desc, OpIdx)) {
3345     return false;
3346   }
3347 
3348   const MCOperand &MO = Inst.getOperand(OpIdx);
3349 
3350   int64_t Val = MO.getImm();
3351   auto OpSize = AMDGPU::getOperandSize(Desc, OpIdx);
3352 
3353   switch (OpSize) { // expected operand size
3354   case 8:
3355     return AMDGPU::isInlinableLiteral64(Val, hasInv2PiInlineImm());
3356   case 4:
3357     return AMDGPU::isInlinableLiteral32(Val, hasInv2PiInlineImm());
3358   case 2: {
3359     const unsigned OperandType = Desc.OpInfo[OpIdx].OperandType;
3360     if (OperandType == AMDGPU::OPERAND_REG_IMM_INT16 ||
3361         OperandType == AMDGPU::OPERAND_REG_INLINE_C_INT16 ||
3362         OperandType == AMDGPU::OPERAND_REG_INLINE_AC_INT16)
3363       return AMDGPU::isInlinableIntLiteral(Val);
3364 
3365     if (OperandType == AMDGPU::OPERAND_REG_INLINE_C_V2INT16 ||
3366         OperandType == AMDGPU::OPERAND_REG_INLINE_AC_V2INT16 ||
3367         OperandType == AMDGPU::OPERAND_REG_IMM_V2INT16)
3368       return AMDGPU::isInlinableIntLiteralV216(Val);
3369 
3370     if (OperandType == AMDGPU::OPERAND_REG_INLINE_C_V2FP16 ||
3371         OperandType == AMDGPU::OPERAND_REG_INLINE_AC_V2FP16 ||
3372         OperandType == AMDGPU::OPERAND_REG_IMM_V2FP16)
3373       return AMDGPU::isInlinableLiteralV216(Val, hasInv2PiInlineImm());
3374 
3375     return AMDGPU::isInlinableLiteral16(Val, hasInv2PiInlineImm());
3376   }
3377   default:
3378     llvm_unreachable("invalid operand size");
3379   }
3380 }
3381 
3382 unsigned AMDGPUAsmParser::getConstantBusLimit(unsigned Opcode) const {
3383   if (!isGFX10Plus())
3384     return 1;
3385 
3386   switch (Opcode) {
3387   // 64-bit shift instructions can use only one scalar value input
3388   case AMDGPU::V_LSHLREV_B64_e64:
3389   case AMDGPU::V_LSHLREV_B64_gfx10:
3390   case AMDGPU::V_LSHLREV_B64_e64_gfx11:
3391   case AMDGPU::V_LSHRREV_B64_e64:
3392   case AMDGPU::V_LSHRREV_B64_gfx10:
3393   case AMDGPU::V_LSHRREV_B64_e64_gfx11:
3394   case AMDGPU::V_ASHRREV_I64_e64:
3395   case AMDGPU::V_ASHRREV_I64_gfx10:
3396   case AMDGPU::V_ASHRREV_I64_e64_gfx11:
3397   case AMDGPU::V_LSHL_B64_e64:
3398   case AMDGPU::V_LSHR_B64_e64:
3399   case AMDGPU::V_ASHR_I64_e64:
3400     return 1;
3401   default:
3402     return 2;
3403   }
3404 }
3405 
3406 bool AMDGPUAsmParser::usesConstantBus(const MCInst &Inst, unsigned OpIdx) {
3407   const MCOperand &MO = Inst.getOperand(OpIdx);
3408   if (MO.isImm()) {
3409     return !isInlineConstant(Inst, OpIdx);
3410   } else if (MO.isReg()) {
3411     auto Reg = MO.getReg();
3412     const MCRegisterInfo *TRI = getContext().getRegisterInfo();
3413     auto PReg = mc2PseudoReg(Reg);
3414     return isSGPR(PReg, TRI) && PReg != SGPR_NULL;
3415   } else {
3416     return true;
3417   }
3418 }
3419 
3420 bool
3421 AMDGPUAsmParser::validateConstantBusLimitations(const MCInst &Inst,
3422                                                 const OperandVector &Operands) {
3423   const unsigned Opcode = Inst.getOpcode();
3424   const MCInstrDesc &Desc = MII.get(Opcode);
3425   unsigned LastSGPR = AMDGPU::NoRegister;
3426   unsigned ConstantBusUseCount = 0;
3427   unsigned NumLiterals = 0;
3428   unsigned LiteralSize;
3429 
3430   if (Desc.TSFlags &
3431       (SIInstrFlags::VOPC |
3432        SIInstrFlags::VOP1 | SIInstrFlags::VOP2 |
3433        SIInstrFlags::VOP3 | SIInstrFlags::VOP3P |
3434        SIInstrFlags::SDWA)) {
3435     // Check special imm operands (used by madmk, etc)
3436     if (AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::imm) != -1) {
3437       ++NumLiterals;
3438       LiteralSize = 4;
3439     }
3440 
3441     SmallDenseSet<unsigned> SGPRsUsed;
3442     unsigned SGPRUsed = findImplicitSGPRReadInVOP(Inst);
3443     if (SGPRUsed != AMDGPU::NoRegister) {
3444       SGPRsUsed.insert(SGPRUsed);
3445       ++ConstantBusUseCount;
3446     }
3447 
3448     const int Src0Idx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::src0);
3449     const int Src1Idx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::src1);
3450     const int Src2Idx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::src2);
3451 
3452     const int OpIndices[] = { Src0Idx, Src1Idx, Src2Idx };
3453 
3454     for (int OpIdx : OpIndices) {
3455       if (OpIdx == -1) break;
3456 
3457       const MCOperand &MO = Inst.getOperand(OpIdx);
3458       if (usesConstantBus(Inst, OpIdx)) {
3459         if (MO.isReg()) {
3460           LastSGPR = mc2PseudoReg(MO.getReg());
3461           // Pairs of registers with a partial intersections like these
3462           //   s0, s[0:1]
3463           //   flat_scratch_lo, flat_scratch
3464           //   flat_scratch_lo, flat_scratch_hi
3465           // are theoretically valid but they are disabled anyway.
3466           // Note that this code mimics SIInstrInfo::verifyInstruction
3467           if (!SGPRsUsed.count(LastSGPR)) {
3468             SGPRsUsed.insert(LastSGPR);
3469             ++ConstantBusUseCount;
3470           }
3471         } else { // Expression or a literal
3472 
3473           if (Desc.OpInfo[OpIdx].OperandType == MCOI::OPERAND_IMMEDIATE)
3474             continue; // special operand like VINTERP attr_chan
3475 
3476           // An instruction may use only one literal.
3477           // This has been validated on the previous step.
3478           // See validateVOPLiteral.
3479           // This literal may be used as more than one operand.
3480           // If all these operands are of the same size,
3481           // this literal counts as one scalar value.
3482           // Otherwise it counts as 2 scalar values.
3483           // See "GFX10 Shader Programming", section 3.6.2.3.
3484 
3485           unsigned Size = AMDGPU::getOperandSize(Desc, OpIdx);
3486           if (Size < 4) Size = 4;
3487 
3488           if (NumLiterals == 0) {
3489             NumLiterals = 1;
3490             LiteralSize = Size;
3491           } else if (LiteralSize != Size) {
3492             NumLiterals = 2;
3493           }
3494         }
3495       }
3496     }
3497   }
3498   ConstantBusUseCount += NumLiterals;
3499 
3500   if (ConstantBusUseCount <= getConstantBusLimit(Opcode))
3501     return true;
3502 
3503   SMLoc LitLoc = getLitLoc(Operands);
3504   SMLoc RegLoc = getRegLoc(LastSGPR, Operands);
3505   SMLoc Loc = (LitLoc.getPointer() < RegLoc.getPointer()) ? RegLoc : LitLoc;
3506   Error(Loc, "invalid operand (violates constant bus restrictions)");
3507   return false;
3508 }
3509 
3510 bool
3511 AMDGPUAsmParser::validateEarlyClobberLimitations(const MCInst &Inst,
3512                                                  const OperandVector &Operands) {
3513   const unsigned Opcode = Inst.getOpcode();
3514   const MCInstrDesc &Desc = MII.get(Opcode);
3515 
3516   const int DstIdx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::vdst);
3517   if (DstIdx == -1 ||
3518       Desc.getOperandConstraint(DstIdx, MCOI::EARLY_CLOBBER) == -1) {
3519     return true;
3520   }
3521 
3522   const MCRegisterInfo *TRI = getContext().getRegisterInfo();
3523 
3524   const int Src0Idx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::src0);
3525   const int Src1Idx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::src1);
3526   const int Src2Idx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::src2);
3527 
3528   assert(DstIdx != -1);
3529   const MCOperand &Dst = Inst.getOperand(DstIdx);
3530   assert(Dst.isReg());
3531 
3532   const int SrcIndices[] = { Src0Idx, Src1Idx, Src2Idx };
3533 
3534   for (int SrcIdx : SrcIndices) {
3535     if (SrcIdx == -1) break;
3536     const MCOperand &Src = Inst.getOperand(SrcIdx);
3537     if (Src.isReg()) {
3538       if (TRI->regsOverlap(Dst.getReg(), Src.getReg())) {
3539         const unsigned SrcReg = mc2PseudoReg(Src.getReg());
3540         Error(getRegLoc(SrcReg, Operands),
3541           "destination must be different than all sources");
3542         return false;
3543       }
3544     }
3545   }
3546 
3547   return true;
3548 }
3549 
3550 bool AMDGPUAsmParser::validateIntClampSupported(const MCInst &Inst) {
3551 
3552   const unsigned Opc = Inst.getOpcode();
3553   const MCInstrDesc &Desc = MII.get(Opc);
3554 
3555   if ((Desc.TSFlags & SIInstrFlags::IntClamp) != 0 && !hasIntClamp()) {
3556     int ClampIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::clamp);
3557     assert(ClampIdx != -1);
3558     return Inst.getOperand(ClampIdx).getImm() == 0;
3559   }
3560 
3561   return true;
3562 }
3563 
3564 Optional<StringRef> AMDGPUAsmParser::validateMIMGDataSize(const MCInst &Inst) {
3565 
3566   const unsigned Opc = Inst.getOpcode();
3567   const MCInstrDesc &Desc = MII.get(Opc);
3568 
3569   if ((Desc.TSFlags & SIInstrFlags::MIMG) == 0)
3570     return None;
3571 
3572   int VDataIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::vdata);
3573   int DMaskIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::dmask);
3574   int TFEIdx   = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::tfe);
3575 
3576   assert(VDataIdx != -1);
3577 
3578   if (DMaskIdx == -1 || TFEIdx == -1) // intersect_ray
3579     return None;
3580 
3581   unsigned VDataSize = AMDGPU::getRegOperandSize(getMRI(), Desc, VDataIdx);
3582   unsigned TFESize = (TFEIdx != -1 && Inst.getOperand(TFEIdx).getImm()) ? 1 : 0;
3583   unsigned DMask = Inst.getOperand(DMaskIdx).getImm() & 0xf;
3584   if (DMask == 0)
3585     DMask = 1;
3586 
3587   bool isPackedD16 = false;
3588   unsigned DataSize =
3589     (Desc.TSFlags & SIInstrFlags::Gather4) ? 4 : countPopulation(DMask);
3590   if (hasPackedD16()) {
3591     int D16Idx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::d16);
3592     isPackedD16 = D16Idx >= 0;
3593     if (isPackedD16 && Inst.getOperand(D16Idx).getImm())
3594       DataSize = (DataSize + 1) / 2;
3595   }
3596 
3597   if ((VDataSize / 4) == DataSize + TFESize)
3598     return None;
3599 
3600   return StringRef(isPackedD16
3601                        ? "image data size does not match dmask, d16 and tfe"
3602                        : "image data size does not match dmask and tfe");
3603 }
3604 
3605 bool AMDGPUAsmParser::validateMIMGAddrSize(const MCInst &Inst) {
3606   const unsigned Opc = Inst.getOpcode();
3607   const MCInstrDesc &Desc = MII.get(Opc);
3608 
3609   if ((Desc.TSFlags & SIInstrFlags::MIMG) == 0 || !isGFX10Plus())
3610     return true;
3611 
3612   const AMDGPU::MIMGInfo *Info = AMDGPU::getMIMGInfo(Opc);
3613 
3614   const AMDGPU::MIMGBaseOpcodeInfo *BaseOpcode =
3615       AMDGPU::getMIMGBaseOpcodeInfo(Info->BaseOpcode);
3616   int VAddr0Idx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::vaddr0);
3617   int SrsrcIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::srsrc);
3618   int DimIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::dim);
3619   int A16Idx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::a16);
3620 
3621   assert(VAddr0Idx != -1);
3622   assert(SrsrcIdx != -1);
3623   assert(SrsrcIdx > VAddr0Idx);
3624 
3625   if (DimIdx == -1)
3626     return true; // intersect_ray
3627 
3628   unsigned Dim = Inst.getOperand(DimIdx).getImm();
3629   const AMDGPU::MIMGDimInfo *DimInfo = AMDGPU::getMIMGDimInfoByEncoding(Dim);
3630   bool IsNSA = SrsrcIdx - VAddr0Idx > 1;
3631   unsigned ActualAddrSize =
3632       IsNSA ? SrsrcIdx - VAddr0Idx
3633             : AMDGPU::getRegOperandSize(getMRI(), Desc, VAddr0Idx) / 4;
3634   bool IsA16 = (A16Idx != -1 && Inst.getOperand(A16Idx).getImm());
3635 
3636   unsigned ExpectedAddrSize =
3637       AMDGPU::getAddrSizeMIMGOp(BaseOpcode, DimInfo, IsA16, hasG16());
3638 
3639   if (!IsNSA) {
3640     if (ExpectedAddrSize > 8)
3641       ExpectedAddrSize = 16;
3642 
3643     // Allow oversized 8 VGPR vaddr when only 5/6/7 VGPRs are required.
3644     // This provides backward compatibility for assembly created
3645     // before 160b/192b/224b types were directly supported.
3646     if (ActualAddrSize == 8 && (ExpectedAddrSize >= 5 && ExpectedAddrSize <= 7))
3647       return true;
3648   }
3649 
3650   return ActualAddrSize == ExpectedAddrSize;
3651 }
3652 
3653 bool AMDGPUAsmParser::validateMIMGAtomicDMask(const MCInst &Inst) {
3654 
3655   const unsigned Opc = Inst.getOpcode();
3656   const MCInstrDesc &Desc = MII.get(Opc);
3657 
3658   if ((Desc.TSFlags & SIInstrFlags::MIMG) == 0)
3659     return true;
3660   if (!Desc.mayLoad() || !Desc.mayStore())
3661     return true; // Not atomic
3662 
3663   int DMaskIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::dmask);
3664   unsigned DMask = Inst.getOperand(DMaskIdx).getImm() & 0xf;
3665 
3666   // This is an incomplete check because image_atomic_cmpswap
3667   // may only use 0x3 and 0xf while other atomic operations
3668   // may use 0x1 and 0x3. However these limitations are
3669   // verified when we check that dmask matches dst size.
3670   return DMask == 0x1 || DMask == 0x3 || DMask == 0xf;
3671 }
3672 
3673 bool AMDGPUAsmParser::validateMIMGGatherDMask(const MCInst &Inst) {
3674 
3675   const unsigned Opc = Inst.getOpcode();
3676   const MCInstrDesc &Desc = MII.get(Opc);
3677 
3678   if ((Desc.TSFlags & SIInstrFlags::Gather4) == 0)
3679     return true;
3680 
3681   int DMaskIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::dmask);
3682   unsigned DMask = Inst.getOperand(DMaskIdx).getImm() & 0xf;
3683 
3684   // GATHER4 instructions use dmask in a different fashion compared to
3685   // other MIMG instructions. The only useful DMASK values are
3686   // 1=red, 2=green, 4=blue, 8=alpha. (e.g. 1 returns
3687   // (red,red,red,red) etc.) The ISA document doesn't mention
3688   // this.
3689   return DMask == 0x1 || DMask == 0x2 || DMask == 0x4 || DMask == 0x8;
3690 }
3691 
3692 bool AMDGPUAsmParser::validateMIMGMSAA(const MCInst &Inst) {
3693   const unsigned Opc = Inst.getOpcode();
3694   const MCInstrDesc &Desc = MII.get(Opc);
3695 
3696   if ((Desc.TSFlags & SIInstrFlags::MIMG) == 0)
3697     return true;
3698 
3699   const AMDGPU::MIMGInfo *Info = AMDGPU::getMIMGInfo(Opc);
3700   const AMDGPU::MIMGBaseOpcodeInfo *BaseOpcode =
3701       AMDGPU::getMIMGBaseOpcodeInfo(Info->BaseOpcode);
3702 
3703   if (!BaseOpcode->MSAA)
3704     return true;
3705 
3706   int DimIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::dim);
3707   assert(DimIdx != -1);
3708 
3709   unsigned Dim = Inst.getOperand(DimIdx).getImm();
3710   const AMDGPU::MIMGDimInfo *DimInfo = AMDGPU::getMIMGDimInfoByEncoding(Dim);
3711 
3712   return DimInfo->MSAA;
3713 }
3714 
3715 static bool IsMovrelsSDWAOpcode(const unsigned Opcode)
3716 {
3717   switch (Opcode) {
3718   case AMDGPU::V_MOVRELS_B32_sdwa_gfx10:
3719   case AMDGPU::V_MOVRELSD_B32_sdwa_gfx10:
3720   case AMDGPU::V_MOVRELSD_2_B32_sdwa_gfx10:
3721     return true;
3722   default:
3723     return false;
3724   }
3725 }
3726 
3727 // movrels* opcodes should only allow VGPRS as src0.
3728 // This is specified in .td description for vop1/vop3,
3729 // but sdwa is handled differently. See isSDWAOperand.
3730 bool AMDGPUAsmParser::validateMovrels(const MCInst &Inst,
3731                                       const OperandVector &Operands) {
3732 
3733   const unsigned Opc = Inst.getOpcode();
3734   const MCInstrDesc &Desc = MII.get(Opc);
3735 
3736   if ((Desc.TSFlags & SIInstrFlags::SDWA) == 0 || !IsMovrelsSDWAOpcode(Opc))
3737     return true;
3738 
3739   const int Src0Idx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src0);
3740   assert(Src0Idx != -1);
3741 
3742   SMLoc ErrLoc;
3743   const MCOperand &Src0 = Inst.getOperand(Src0Idx);
3744   if (Src0.isReg()) {
3745     auto Reg = mc2PseudoReg(Src0.getReg());
3746     const MCRegisterInfo *TRI = getContext().getRegisterInfo();
3747     if (!isSGPR(Reg, TRI))
3748       return true;
3749     ErrLoc = getRegLoc(Reg, Operands);
3750   } else {
3751     ErrLoc = getConstLoc(Operands);
3752   }
3753 
3754   Error(ErrLoc, "source operand must be a VGPR");
3755   return false;
3756 }
3757 
3758 bool AMDGPUAsmParser::validateMAIAccWrite(const MCInst &Inst,
3759                                           const OperandVector &Operands) {
3760 
3761   const unsigned Opc = Inst.getOpcode();
3762 
3763   if (Opc != AMDGPU::V_ACCVGPR_WRITE_B32_vi)
3764     return true;
3765 
3766   const int Src0Idx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src0);
3767   assert(Src0Idx != -1);
3768 
3769   const MCOperand &Src0 = Inst.getOperand(Src0Idx);
3770   if (!Src0.isReg())
3771     return true;
3772 
3773   auto Reg = mc2PseudoReg(Src0.getReg());
3774   const MCRegisterInfo *TRI = getContext().getRegisterInfo();
3775   if (!isGFX90A() && isSGPR(Reg, TRI)) {
3776     Error(getRegLoc(Reg, Operands),
3777           "source operand must be either a VGPR or an inline constant");
3778     return false;
3779   }
3780 
3781   return true;
3782 }
3783 
3784 bool AMDGPUAsmParser::validateMFMA(const MCInst &Inst,
3785                                    const OperandVector &Operands) {
3786   const unsigned Opc = Inst.getOpcode();
3787   const MCInstrDesc &Desc = MII.get(Opc);
3788 
3789   if ((Desc.TSFlags & SIInstrFlags::IsMAI) == 0)
3790     return true;
3791 
3792   const int Src2Idx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src2);
3793   if (Src2Idx == -1)
3794     return true;
3795 
3796   const MCOperand &Src2 = Inst.getOperand(Src2Idx);
3797   if (!Src2.isReg())
3798     return true;
3799 
3800   MCRegister Src2Reg = Src2.getReg();
3801   MCRegister DstReg = Inst.getOperand(0).getReg();
3802   if (Src2Reg == DstReg)
3803     return true;
3804 
3805   const MCRegisterInfo *TRI = getContext().getRegisterInfo();
3806   if (TRI->getRegClass(Desc.OpInfo[0].RegClass).getSizeInBits() <= 128)
3807     return true;
3808 
3809   if (TRI->regsOverlap(Src2Reg, DstReg)) {
3810     Error(getRegLoc(mc2PseudoReg(Src2Reg), Operands),
3811           "source 2 operand must not partially overlap with dst");
3812     return false;
3813   }
3814 
3815   return true;
3816 }
3817 
3818 bool AMDGPUAsmParser::validateDivScale(const MCInst &Inst) {
3819   switch (Inst.getOpcode()) {
3820   default:
3821     return true;
3822   case V_DIV_SCALE_F32_gfx6_gfx7:
3823   case V_DIV_SCALE_F32_vi:
3824   case V_DIV_SCALE_F32_gfx10:
3825   case V_DIV_SCALE_F64_gfx6_gfx7:
3826   case V_DIV_SCALE_F64_vi:
3827   case V_DIV_SCALE_F64_gfx10:
3828     break;
3829   }
3830 
3831   // TODO: Check that src0 = src1 or src2.
3832 
3833   for (auto Name : {AMDGPU::OpName::src0_modifiers,
3834                     AMDGPU::OpName::src2_modifiers,
3835                     AMDGPU::OpName::src2_modifiers}) {
3836     if (Inst.getOperand(AMDGPU::getNamedOperandIdx(Inst.getOpcode(), Name))
3837             .getImm() &
3838         SISrcMods::ABS) {
3839       return false;
3840     }
3841   }
3842 
3843   return true;
3844 }
3845 
3846 bool AMDGPUAsmParser::validateMIMGD16(const MCInst &Inst) {
3847 
3848   const unsigned Opc = Inst.getOpcode();
3849   const MCInstrDesc &Desc = MII.get(Opc);
3850 
3851   if ((Desc.TSFlags & SIInstrFlags::MIMG) == 0)
3852     return true;
3853 
3854   int D16Idx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::d16);
3855   if (D16Idx >= 0 && Inst.getOperand(D16Idx).getImm()) {
3856     if (isCI() || isSI())
3857       return false;
3858   }
3859 
3860   return true;
3861 }
3862 
3863 bool AMDGPUAsmParser::validateMIMGDim(const MCInst &Inst) {
3864   const unsigned Opc = Inst.getOpcode();
3865   const MCInstrDesc &Desc = MII.get(Opc);
3866 
3867   if ((Desc.TSFlags & SIInstrFlags::MIMG) == 0)
3868     return true;
3869 
3870   int DimIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::dim);
3871   if (DimIdx < 0)
3872     return true;
3873 
3874   long Imm = Inst.getOperand(DimIdx).getImm();
3875   if (Imm < 0 || Imm >= 8)
3876     return false;
3877 
3878   return true;
3879 }
3880 
3881 static bool IsRevOpcode(const unsigned Opcode)
3882 {
3883   switch (Opcode) {
3884   case AMDGPU::V_SUBREV_F32_e32:
3885   case AMDGPU::V_SUBREV_F32_e64:
3886   case AMDGPU::V_SUBREV_F32_e32_gfx10:
3887   case AMDGPU::V_SUBREV_F32_e32_gfx6_gfx7:
3888   case AMDGPU::V_SUBREV_F32_e32_vi:
3889   case AMDGPU::V_SUBREV_F32_e64_gfx10:
3890   case AMDGPU::V_SUBREV_F32_e64_gfx6_gfx7:
3891   case AMDGPU::V_SUBREV_F32_e64_vi:
3892 
3893   case AMDGPU::V_SUBREV_CO_U32_e32:
3894   case AMDGPU::V_SUBREV_CO_U32_e64:
3895   case AMDGPU::V_SUBREV_I32_e32_gfx6_gfx7:
3896   case AMDGPU::V_SUBREV_I32_e64_gfx6_gfx7:
3897 
3898   case AMDGPU::V_SUBBREV_U32_e32:
3899   case AMDGPU::V_SUBBREV_U32_e64:
3900   case AMDGPU::V_SUBBREV_U32_e32_gfx6_gfx7:
3901   case AMDGPU::V_SUBBREV_U32_e32_vi:
3902   case AMDGPU::V_SUBBREV_U32_e64_gfx6_gfx7:
3903   case AMDGPU::V_SUBBREV_U32_e64_vi:
3904 
3905   case AMDGPU::V_SUBREV_U32_e32:
3906   case AMDGPU::V_SUBREV_U32_e64:
3907   case AMDGPU::V_SUBREV_U32_e32_gfx9:
3908   case AMDGPU::V_SUBREV_U32_e32_vi:
3909   case AMDGPU::V_SUBREV_U32_e64_gfx9:
3910   case AMDGPU::V_SUBREV_U32_e64_vi:
3911 
3912   case AMDGPU::V_SUBREV_F16_e32:
3913   case AMDGPU::V_SUBREV_F16_e64:
3914   case AMDGPU::V_SUBREV_F16_e32_gfx10:
3915   case AMDGPU::V_SUBREV_F16_e32_vi:
3916   case AMDGPU::V_SUBREV_F16_e64_gfx10:
3917   case AMDGPU::V_SUBREV_F16_e64_vi:
3918 
3919   case AMDGPU::V_SUBREV_U16_e32:
3920   case AMDGPU::V_SUBREV_U16_e64:
3921   case AMDGPU::V_SUBREV_U16_e32_vi:
3922   case AMDGPU::V_SUBREV_U16_e64_vi:
3923 
3924   case AMDGPU::V_SUBREV_CO_U32_e32_gfx9:
3925   case AMDGPU::V_SUBREV_CO_U32_e64_gfx10:
3926   case AMDGPU::V_SUBREV_CO_U32_e64_gfx9:
3927 
3928   case AMDGPU::V_SUBBREV_CO_U32_e32_gfx9:
3929   case AMDGPU::V_SUBBREV_CO_U32_e64_gfx9:
3930 
3931   case AMDGPU::V_SUBREV_NC_U32_e32_gfx10:
3932   case AMDGPU::V_SUBREV_NC_U32_e64_gfx10:
3933 
3934   case AMDGPU::V_SUBREV_CO_CI_U32_e32_gfx10:
3935   case AMDGPU::V_SUBREV_CO_CI_U32_e64_gfx10:
3936 
3937   case AMDGPU::V_LSHRREV_B32_e32:
3938   case AMDGPU::V_LSHRREV_B32_e64:
3939   case AMDGPU::V_LSHRREV_B32_e32_gfx6_gfx7:
3940   case AMDGPU::V_LSHRREV_B32_e64_gfx6_gfx7:
3941   case AMDGPU::V_LSHRREV_B32_e32_vi:
3942   case AMDGPU::V_LSHRREV_B32_e64_vi:
3943   case AMDGPU::V_LSHRREV_B32_e32_gfx10:
3944   case AMDGPU::V_LSHRREV_B32_e64_gfx10:
3945 
3946   case AMDGPU::V_ASHRREV_I32_e32:
3947   case AMDGPU::V_ASHRREV_I32_e64:
3948   case AMDGPU::V_ASHRREV_I32_e32_gfx10:
3949   case AMDGPU::V_ASHRREV_I32_e32_gfx6_gfx7:
3950   case AMDGPU::V_ASHRREV_I32_e32_vi:
3951   case AMDGPU::V_ASHRREV_I32_e64_gfx10:
3952   case AMDGPU::V_ASHRREV_I32_e64_gfx6_gfx7:
3953   case AMDGPU::V_ASHRREV_I32_e64_vi:
3954 
3955   case AMDGPU::V_LSHLREV_B32_e32:
3956   case AMDGPU::V_LSHLREV_B32_e64:
3957   case AMDGPU::V_LSHLREV_B32_e32_gfx10:
3958   case AMDGPU::V_LSHLREV_B32_e32_gfx6_gfx7:
3959   case AMDGPU::V_LSHLREV_B32_e32_vi:
3960   case AMDGPU::V_LSHLREV_B32_e64_gfx10:
3961   case AMDGPU::V_LSHLREV_B32_e64_gfx6_gfx7:
3962   case AMDGPU::V_LSHLREV_B32_e64_vi:
3963 
3964   case AMDGPU::V_LSHLREV_B16_e32:
3965   case AMDGPU::V_LSHLREV_B16_e64:
3966   case AMDGPU::V_LSHLREV_B16_e32_vi:
3967   case AMDGPU::V_LSHLREV_B16_e64_vi:
3968   case AMDGPU::V_LSHLREV_B16_gfx10:
3969 
3970   case AMDGPU::V_LSHRREV_B16_e32:
3971   case AMDGPU::V_LSHRREV_B16_e64:
3972   case AMDGPU::V_LSHRREV_B16_e32_vi:
3973   case AMDGPU::V_LSHRREV_B16_e64_vi:
3974   case AMDGPU::V_LSHRREV_B16_gfx10:
3975 
3976   case AMDGPU::V_ASHRREV_I16_e32:
3977   case AMDGPU::V_ASHRREV_I16_e64:
3978   case AMDGPU::V_ASHRREV_I16_e32_vi:
3979   case AMDGPU::V_ASHRREV_I16_e64_vi:
3980   case AMDGPU::V_ASHRREV_I16_gfx10:
3981 
3982   case AMDGPU::V_LSHLREV_B64_e64:
3983   case AMDGPU::V_LSHLREV_B64_gfx10:
3984   case AMDGPU::V_LSHLREV_B64_vi:
3985 
3986   case AMDGPU::V_LSHRREV_B64_e64:
3987   case AMDGPU::V_LSHRREV_B64_gfx10:
3988   case AMDGPU::V_LSHRREV_B64_vi:
3989 
3990   case AMDGPU::V_ASHRREV_I64_e64:
3991   case AMDGPU::V_ASHRREV_I64_gfx10:
3992   case AMDGPU::V_ASHRREV_I64_vi:
3993 
3994   case AMDGPU::V_PK_LSHLREV_B16:
3995   case AMDGPU::V_PK_LSHLREV_B16_gfx10:
3996   case AMDGPU::V_PK_LSHLREV_B16_vi:
3997 
3998   case AMDGPU::V_PK_LSHRREV_B16:
3999   case AMDGPU::V_PK_LSHRREV_B16_gfx10:
4000   case AMDGPU::V_PK_LSHRREV_B16_vi:
4001   case AMDGPU::V_PK_ASHRREV_I16:
4002   case AMDGPU::V_PK_ASHRREV_I16_gfx10:
4003   case AMDGPU::V_PK_ASHRREV_I16_vi:
4004     return true;
4005   default:
4006     return false;
4007   }
4008 }
4009 
4010 Optional<StringRef> AMDGPUAsmParser::validateLdsDirect(const MCInst &Inst) {
4011 
4012   using namespace SIInstrFlags;
4013   const unsigned Opcode = Inst.getOpcode();
4014   const MCInstrDesc &Desc = MII.get(Opcode);
4015 
4016   // lds_direct register is defined so that it can be used
4017   // with 9-bit operands only. Ignore encodings which do not accept these.
4018   const auto Enc = VOP1 | VOP2 | VOP3 | VOPC | VOP3P | SIInstrFlags::SDWA;
4019   if ((Desc.TSFlags & Enc) == 0)
4020     return None;
4021 
4022   for (auto SrcName : {OpName::src0, OpName::src1, OpName::src2}) {
4023     auto SrcIdx = getNamedOperandIdx(Opcode, SrcName);
4024     if (SrcIdx == -1)
4025       break;
4026     const auto &Src = Inst.getOperand(SrcIdx);
4027     if (Src.isReg() && Src.getReg() == LDS_DIRECT) {
4028 
4029       if (isGFX90A() || isGFX11Plus())
4030         return StringRef("lds_direct is not supported on this GPU");
4031 
4032       if (IsRevOpcode(Opcode) || (Desc.TSFlags & SIInstrFlags::SDWA))
4033         return StringRef("lds_direct cannot be used with this instruction");
4034 
4035       if (SrcName != OpName::src0)
4036         return StringRef("lds_direct may be used as src0 only");
4037     }
4038   }
4039 
4040   return None;
4041 }
4042 
4043 SMLoc AMDGPUAsmParser::getFlatOffsetLoc(const OperandVector &Operands) const {
4044   for (unsigned i = 1, e = Operands.size(); i != e; ++i) {
4045     AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[i]);
4046     if (Op.isFlatOffset())
4047       return Op.getStartLoc();
4048   }
4049   return getLoc();
4050 }
4051 
4052 bool AMDGPUAsmParser::validateFlatOffset(const MCInst &Inst,
4053                                          const OperandVector &Operands) {
4054   uint64_t TSFlags = MII.get(Inst.getOpcode()).TSFlags;
4055   if ((TSFlags & SIInstrFlags::FLAT) == 0)
4056     return true;
4057 
4058   auto Opcode = Inst.getOpcode();
4059   auto OpNum = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::offset);
4060   assert(OpNum != -1);
4061 
4062   const auto &Op = Inst.getOperand(OpNum);
4063   if (!hasFlatOffsets() && Op.getImm() != 0) {
4064     Error(getFlatOffsetLoc(Operands),
4065           "flat offset modifier is not supported on this GPU");
4066     return false;
4067   }
4068 
4069   // For FLAT segment the offset must be positive;
4070   // MSB is ignored and forced to zero.
4071   if (TSFlags & (SIInstrFlags::FlatGlobal | SIInstrFlags::FlatScratch)) {
4072     unsigned OffsetSize = AMDGPU::getNumFlatOffsetBits(getSTI(), true);
4073     if (!isIntN(OffsetSize, Op.getImm())) {
4074       Error(getFlatOffsetLoc(Operands),
4075             Twine("expected a ") + Twine(OffsetSize) + "-bit signed offset");
4076       return false;
4077     }
4078   } else {
4079     unsigned OffsetSize = AMDGPU::getNumFlatOffsetBits(getSTI(), false);
4080     if (!isUIntN(OffsetSize, Op.getImm())) {
4081       Error(getFlatOffsetLoc(Operands),
4082             Twine("expected a ") + Twine(OffsetSize) + "-bit unsigned offset");
4083       return false;
4084     }
4085   }
4086 
4087   return true;
4088 }
4089 
4090 SMLoc AMDGPUAsmParser::getSMEMOffsetLoc(const OperandVector &Operands) const {
4091   // Start with second operand because SMEM Offset cannot be dst or src0.
4092   for (unsigned i = 2, e = Operands.size(); i != e; ++i) {
4093     AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[i]);
4094     if (Op.isSMEMOffset())
4095       return Op.getStartLoc();
4096   }
4097   return getLoc();
4098 }
4099 
4100 bool AMDGPUAsmParser::validateSMEMOffset(const MCInst &Inst,
4101                                          const OperandVector &Operands) {
4102   if (isCI() || isSI())
4103     return true;
4104 
4105   uint64_t TSFlags = MII.get(Inst.getOpcode()).TSFlags;
4106   if ((TSFlags & SIInstrFlags::SMRD) == 0)
4107     return true;
4108 
4109   auto Opcode = Inst.getOpcode();
4110   auto OpNum = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::offset);
4111   if (OpNum == -1)
4112     return true;
4113 
4114   const auto &Op = Inst.getOperand(OpNum);
4115   if (!Op.isImm())
4116     return true;
4117 
4118   uint64_t Offset = Op.getImm();
4119   bool IsBuffer = AMDGPU::getSMEMIsBuffer(Opcode);
4120   if (AMDGPU::isLegalSMRDEncodedUnsignedOffset(getSTI(), Offset) ||
4121       AMDGPU::isLegalSMRDEncodedSignedOffset(getSTI(), Offset, IsBuffer))
4122     return true;
4123 
4124   Error(getSMEMOffsetLoc(Operands),
4125         (isVI() || IsBuffer) ? "expected a 20-bit unsigned offset" :
4126                                "expected a 21-bit signed offset");
4127 
4128   return false;
4129 }
4130 
4131 bool AMDGPUAsmParser::validateSOPLiteral(const MCInst &Inst) const {
4132   unsigned Opcode = Inst.getOpcode();
4133   const MCInstrDesc &Desc = MII.get(Opcode);
4134   if (!(Desc.TSFlags & (SIInstrFlags::SOP2 | SIInstrFlags::SOPC)))
4135     return true;
4136 
4137   const int Src0Idx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::src0);
4138   const int Src1Idx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::src1);
4139 
4140   const int OpIndices[] = { Src0Idx, Src1Idx };
4141 
4142   unsigned NumExprs = 0;
4143   unsigned NumLiterals = 0;
4144   uint32_t LiteralValue;
4145 
4146   for (int OpIdx : OpIndices) {
4147     if (OpIdx == -1) break;
4148 
4149     const MCOperand &MO = Inst.getOperand(OpIdx);
4150     // Exclude special imm operands (like that used by s_set_gpr_idx_on)
4151     if (AMDGPU::isSISrcOperand(Desc, OpIdx)) {
4152       if (MO.isImm() && !isInlineConstant(Inst, OpIdx)) {
4153         uint32_t Value = static_cast<uint32_t>(MO.getImm());
4154         if (NumLiterals == 0 || LiteralValue != Value) {
4155           LiteralValue = Value;
4156           ++NumLiterals;
4157         }
4158       } else if (MO.isExpr()) {
4159         ++NumExprs;
4160       }
4161     }
4162   }
4163 
4164   return NumLiterals + NumExprs <= 1;
4165 }
4166 
4167 bool AMDGPUAsmParser::validateOpSel(const MCInst &Inst) {
4168   const unsigned Opc = Inst.getOpcode();
4169   if (Opc == AMDGPU::V_PERMLANE16_B32_gfx10 ||
4170       Opc == AMDGPU::V_PERMLANEX16_B32_gfx10) {
4171     int OpSelIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::op_sel);
4172     unsigned OpSel = Inst.getOperand(OpSelIdx).getImm();
4173 
4174     if (OpSel & ~3)
4175       return false;
4176   }
4177 
4178   if (isGFX940() && (MII.get(Opc).TSFlags & SIInstrFlags::IsDOT)) {
4179     int OpSelIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::op_sel);
4180     if (OpSelIdx != -1) {
4181       if (Inst.getOperand(OpSelIdx).getImm() != 0)
4182         return false;
4183     }
4184     int OpSelHiIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::op_sel_hi);
4185     if (OpSelHiIdx != -1) {
4186       if (Inst.getOperand(OpSelHiIdx).getImm() != -1)
4187         return false;
4188     }
4189   }
4190 
4191   return true;
4192 }
4193 
4194 bool AMDGPUAsmParser::validateDPP(const MCInst &Inst,
4195                                   const OperandVector &Operands) {
4196   const unsigned Opc = Inst.getOpcode();
4197   int DppCtrlIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::dpp_ctrl);
4198   if (DppCtrlIdx < 0)
4199     return true;
4200   unsigned DppCtrl = Inst.getOperand(DppCtrlIdx).getImm();
4201 
4202   if (!AMDGPU::isLegal64BitDPPControl(DppCtrl)) {
4203     // DPP64 is supported for row_newbcast only.
4204     int Src0Idx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src0);
4205     if (Src0Idx >= 0 &&
4206         getMRI()->getSubReg(Inst.getOperand(Src0Idx).getReg(), AMDGPU::sub1)) {
4207       SMLoc S = getImmLoc(AMDGPUOperand::ImmTyDppCtrl, Operands);
4208       Error(S, "64 bit dpp only supports row_newbcast");
4209       return false;
4210     }
4211   }
4212 
4213   return true;
4214 }
4215 
4216 // Check if VCC register matches wavefront size
4217 bool AMDGPUAsmParser::validateVccOperand(unsigned Reg) const {
4218   auto FB = getFeatureBits();
4219   return (FB[AMDGPU::FeatureWavefrontSize64] && Reg == AMDGPU::VCC) ||
4220     (FB[AMDGPU::FeatureWavefrontSize32] && Reg == AMDGPU::VCC_LO);
4221 }
4222 
4223 // One unique literal can be used. VOP3 literal is only allowed in GFX10+
4224 bool AMDGPUAsmParser::validateVOPLiteral(const MCInst &Inst,
4225                                          const OperandVector &Operands) {
4226   unsigned Opcode = Inst.getOpcode();
4227   const MCInstrDesc &Desc = MII.get(Opcode);
4228   const int ImmIdx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::imm);
4229   if (!(Desc.TSFlags & (SIInstrFlags::VOP3 | SIInstrFlags::VOP3P)) &&
4230       ImmIdx == -1)
4231     return true;
4232 
4233   const int Src0Idx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::src0);
4234   const int Src1Idx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::src1);
4235   const int Src2Idx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::src2);
4236 
4237   const int OpIndices[] = {Src0Idx, Src1Idx, Src2Idx, ImmIdx};
4238 
4239   unsigned NumExprs = 0;
4240   unsigned NumLiterals = 0;
4241   uint32_t LiteralValue;
4242 
4243   for (int OpIdx : OpIndices) {
4244     if (OpIdx == -1)
4245       continue;
4246 
4247     const MCOperand &MO = Inst.getOperand(OpIdx);
4248     if (!MO.isImm() && !MO.isExpr())
4249       continue;
4250     if (!AMDGPU::isSISrcOperand(Desc, OpIdx))
4251       continue;
4252 
4253     if (OpIdx == Src2Idx && (Desc.TSFlags & SIInstrFlags::IsMAI) &&
4254         getFeatureBits()[AMDGPU::FeatureMFMAInlineLiteralBug]) {
4255       Error(getConstLoc(Operands),
4256             "inline constants are not allowed for this operand");
4257       return false;
4258     }
4259 
4260     if (MO.isImm() && !isInlineConstant(Inst, OpIdx)) {
4261       uint32_t Value = static_cast<uint32_t>(MO.getImm());
4262       if (NumLiterals == 0 || LiteralValue != Value) {
4263         LiteralValue = Value;
4264         ++NumLiterals;
4265       }
4266     } else if (MO.isExpr()) {
4267       ++NumExprs;
4268     }
4269   }
4270   NumLiterals += NumExprs;
4271 
4272   if (!NumLiterals)
4273     return true;
4274 
4275   if (ImmIdx == -1 && !getFeatureBits()[AMDGPU::FeatureVOP3Literal]) {
4276     Error(getLitLoc(Operands), "literal operands are not supported");
4277     return false;
4278   }
4279 
4280   if (NumLiterals > 1) {
4281     Error(getLitLoc(Operands), "only one literal operand is allowed");
4282     return false;
4283   }
4284 
4285   return true;
4286 }
4287 
4288 // Returns -1 if not a register, 0 if VGPR and 1 if AGPR.
4289 static int IsAGPROperand(const MCInst &Inst, uint16_t NameIdx,
4290                          const MCRegisterInfo *MRI) {
4291   int OpIdx = AMDGPU::getNamedOperandIdx(Inst.getOpcode(), NameIdx);
4292   if (OpIdx < 0)
4293     return -1;
4294 
4295   const MCOperand &Op = Inst.getOperand(OpIdx);
4296   if (!Op.isReg())
4297     return -1;
4298 
4299   unsigned Sub = MRI->getSubReg(Op.getReg(), AMDGPU::sub0);
4300   auto Reg = Sub ? Sub : Op.getReg();
4301   const MCRegisterClass &AGPR32 = MRI->getRegClass(AMDGPU::AGPR_32RegClassID);
4302   return AGPR32.contains(Reg) ? 1 : 0;
4303 }
4304 
4305 bool AMDGPUAsmParser::validateAGPRLdSt(const MCInst &Inst) const {
4306   uint64_t TSFlags = MII.get(Inst.getOpcode()).TSFlags;
4307   if ((TSFlags & (SIInstrFlags::FLAT | SIInstrFlags::MUBUF |
4308                   SIInstrFlags::MTBUF | SIInstrFlags::MIMG |
4309                   SIInstrFlags::DS)) == 0)
4310     return true;
4311 
4312   uint16_t DataNameIdx = (TSFlags & SIInstrFlags::DS) ? AMDGPU::OpName::data0
4313                                                       : AMDGPU::OpName::vdata;
4314 
4315   const MCRegisterInfo *MRI = getMRI();
4316   int DstAreg = IsAGPROperand(Inst, AMDGPU::OpName::vdst, MRI);
4317   int DataAreg = IsAGPROperand(Inst, DataNameIdx, MRI);
4318 
4319   if ((TSFlags & SIInstrFlags::DS) && DataAreg >= 0) {
4320     int Data2Areg = IsAGPROperand(Inst, AMDGPU::OpName::data1, MRI);
4321     if (Data2Areg >= 0 && Data2Areg != DataAreg)
4322       return false;
4323   }
4324 
4325   auto FB = getFeatureBits();
4326   if (FB[AMDGPU::FeatureGFX90AInsts]) {
4327     if (DataAreg < 0 || DstAreg < 0)
4328       return true;
4329     return DstAreg == DataAreg;
4330   }
4331 
4332   return DstAreg < 1 && DataAreg < 1;
4333 }
4334 
4335 bool AMDGPUAsmParser::validateVGPRAlign(const MCInst &Inst) const {
4336   auto FB = getFeatureBits();
4337   if (!FB[AMDGPU::FeatureGFX90AInsts])
4338     return true;
4339 
4340   const MCRegisterInfo *MRI = getMRI();
4341   const MCRegisterClass &VGPR32 = MRI->getRegClass(AMDGPU::VGPR_32RegClassID);
4342   const MCRegisterClass &AGPR32 = MRI->getRegClass(AMDGPU::AGPR_32RegClassID);
4343   for (unsigned I = 0, E = Inst.getNumOperands(); I != E; ++I) {
4344     const MCOperand &Op = Inst.getOperand(I);
4345     if (!Op.isReg())
4346       continue;
4347 
4348     unsigned Sub = MRI->getSubReg(Op.getReg(), AMDGPU::sub0);
4349     if (!Sub)
4350       continue;
4351 
4352     if (VGPR32.contains(Sub) && ((Sub - AMDGPU::VGPR0) & 1))
4353       return false;
4354     if (AGPR32.contains(Sub) && ((Sub - AMDGPU::AGPR0) & 1))
4355       return false;
4356   }
4357 
4358   return true;
4359 }
4360 
4361 SMLoc AMDGPUAsmParser::getBLGPLoc(const OperandVector &Operands) const {
4362   for (unsigned i = 1, e = Operands.size(); i != e; ++i) {
4363     AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[i]);
4364     if (Op.isBLGP())
4365       return Op.getStartLoc();
4366   }
4367   return SMLoc();
4368 }
4369 
4370 bool AMDGPUAsmParser::validateBLGP(const MCInst &Inst,
4371                                    const OperandVector &Operands) {
4372   unsigned Opc = Inst.getOpcode();
4373   int BlgpIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::blgp);
4374   if (BlgpIdx == -1)
4375     return true;
4376   SMLoc BLGPLoc = getBLGPLoc(Operands);
4377   if (!BLGPLoc.isValid())
4378     return true;
4379   bool IsNeg = StringRef(BLGPLoc.getPointer()).startswith("neg:");
4380   auto FB = getFeatureBits();
4381   bool UsesNeg = false;
4382   if (FB[AMDGPU::FeatureGFX940Insts]) {
4383     switch (Opc) {
4384     case AMDGPU::V_MFMA_F64_16X16X4F64_gfx940_acd:
4385     case AMDGPU::V_MFMA_F64_16X16X4F64_gfx940_vcd:
4386     case AMDGPU::V_MFMA_F64_4X4X4F64_gfx940_acd:
4387     case AMDGPU::V_MFMA_F64_4X4X4F64_gfx940_vcd:
4388       UsesNeg = true;
4389     }
4390   }
4391 
4392   if (IsNeg == UsesNeg)
4393     return true;
4394 
4395   Error(BLGPLoc,
4396         UsesNeg ? "invalid modifier: blgp is not supported"
4397                 : "invalid modifier: neg is not supported");
4398 
4399   return false;
4400 }
4401 
4402 // gfx90a has an undocumented limitation:
4403 // DS_GWS opcodes must use even aligned registers.
4404 bool AMDGPUAsmParser::validateGWS(const MCInst &Inst,
4405                                   const OperandVector &Operands) {
4406   if (!getFeatureBits()[AMDGPU::FeatureGFX90AInsts])
4407     return true;
4408 
4409   int Opc = Inst.getOpcode();
4410   if (Opc != AMDGPU::DS_GWS_INIT_vi && Opc != AMDGPU::DS_GWS_BARRIER_vi &&
4411       Opc != AMDGPU::DS_GWS_SEMA_BR_vi)
4412     return true;
4413 
4414   const MCRegisterInfo *MRI = getMRI();
4415   const MCRegisterClass &VGPR32 = MRI->getRegClass(AMDGPU::VGPR_32RegClassID);
4416   int Data0Pos =
4417       AMDGPU::getNamedOperandIdx(Inst.getOpcode(), AMDGPU::OpName::data0);
4418   assert(Data0Pos != -1);
4419   auto Reg = Inst.getOperand(Data0Pos).getReg();
4420   auto RegIdx = Reg - (VGPR32.contains(Reg) ? AMDGPU::VGPR0 : AMDGPU::AGPR0);
4421   if (RegIdx & 1) {
4422     SMLoc RegLoc = getRegLoc(Reg, Operands);
4423     Error(RegLoc, "vgpr must be even aligned");
4424     return false;
4425   }
4426 
4427   return true;
4428 }
4429 
4430 bool AMDGPUAsmParser::validateCoherencyBits(const MCInst &Inst,
4431                                             const OperandVector &Operands,
4432                                             const SMLoc &IDLoc) {
4433   int CPolPos = AMDGPU::getNamedOperandIdx(Inst.getOpcode(),
4434                                            AMDGPU::OpName::cpol);
4435   if (CPolPos == -1)
4436     return true;
4437 
4438   unsigned CPol = Inst.getOperand(CPolPos).getImm();
4439 
4440   uint64_t TSFlags = MII.get(Inst.getOpcode()).TSFlags;
4441   if (TSFlags & SIInstrFlags::SMRD) {
4442     if (CPol && (isSI() || isCI())) {
4443       SMLoc S = getImmLoc(AMDGPUOperand::ImmTyCPol, Operands);
4444       Error(S, "cache policy is not supported for SMRD instructions");
4445       return false;
4446     }
4447     if (CPol & ~(AMDGPU::CPol::GLC | AMDGPU::CPol::DLC)) {
4448       Error(IDLoc, "invalid cache policy for SMEM instruction");
4449       return false;
4450     }
4451   }
4452 
4453   if (isGFX90A() && !isGFX940() && (CPol & CPol::SCC)) {
4454     SMLoc S = getImmLoc(AMDGPUOperand::ImmTyCPol, Operands);
4455     StringRef CStr(S.getPointer());
4456     S = SMLoc::getFromPointer(&CStr.data()[CStr.find("scc")]);
4457     Error(S, "scc is not supported on this GPU");
4458     return false;
4459   }
4460 
4461   if (!(TSFlags & (SIInstrFlags::IsAtomicNoRet | SIInstrFlags::IsAtomicRet)))
4462     return true;
4463 
4464   if (TSFlags & SIInstrFlags::IsAtomicRet) {
4465     if (!(TSFlags & SIInstrFlags::MIMG) && !(CPol & CPol::GLC)) {
4466       Error(IDLoc, isGFX940() ? "instruction must use sc0"
4467                               : "instruction must use glc");
4468       return false;
4469     }
4470   } else {
4471     if (CPol & CPol::GLC) {
4472       SMLoc S = getImmLoc(AMDGPUOperand::ImmTyCPol, Operands);
4473       StringRef CStr(S.getPointer());
4474       S = SMLoc::getFromPointer(
4475           &CStr.data()[CStr.find(isGFX940() ? "sc0" : "glc")]);
4476       Error(S, isGFX940() ? "instruction must not use sc0"
4477                           : "instruction must not use glc");
4478       return false;
4479     }
4480   }
4481 
4482   return true;
4483 }
4484 
4485 bool AMDGPUAsmParser::validateFlatLdsDMA(const MCInst &Inst,
4486                                          const OperandVector &Operands,
4487                                          const SMLoc &IDLoc) {
4488   if (isGFX940())
4489     return true;
4490 
4491   uint64_t TSFlags = MII.get(Inst.getOpcode()).TSFlags;
4492   if ((TSFlags & (SIInstrFlags::VALU | SIInstrFlags::FLAT)) !=
4493       (SIInstrFlags::VALU | SIInstrFlags::FLAT))
4494     return true;
4495   // This is FLAT LDS DMA.
4496 
4497   SMLoc S = getImmLoc(AMDGPUOperand::ImmTyLDS, Operands);
4498   StringRef CStr(S.getPointer());
4499   if (!CStr.startswith("lds")) {
4500     // This is incorrectly selected LDS DMA version of a FLAT load opcode.
4501     // And LDS version should have 'lds' modifier, but it follows optional
4502     // operands so its absense is ignored by the matcher.
4503     Error(IDLoc, "invalid operands for instruction");
4504     return false;
4505   }
4506 
4507   return true;
4508 }
4509 
4510 bool AMDGPUAsmParser::validateExeczVcczOperands(const OperandVector &Operands) {
4511   if (!isGFX11Plus())
4512     return true;
4513   for (auto &Operand : Operands) {
4514     if (!Operand->isReg())
4515       continue;
4516     unsigned Reg = Operand->getReg();
4517     if (Reg == SRC_EXECZ || Reg == SRC_VCCZ) {
4518       Error(getRegLoc(Reg, Operands),
4519             "execz and vccz are not supported on this GPU");
4520       return false;
4521     }
4522   }
4523   return true;
4524 }
4525 
4526 bool AMDGPUAsmParser::validateInstruction(const MCInst &Inst,
4527                                           const SMLoc &IDLoc,
4528                                           const OperandVector &Operands) {
4529   if (auto ErrMsg = validateLdsDirect(Inst)) {
4530     Error(getRegLoc(LDS_DIRECT, Operands), *ErrMsg);
4531     return false;
4532   }
4533   if (!validateSOPLiteral(Inst)) {
4534     Error(getLitLoc(Operands),
4535       "only one literal operand is allowed");
4536     return false;
4537   }
4538   if (!validateVOPLiteral(Inst, Operands)) {
4539     return false;
4540   }
4541   if (!validateConstantBusLimitations(Inst, Operands)) {
4542     return false;
4543   }
4544   if (!validateEarlyClobberLimitations(Inst, Operands)) {
4545     return false;
4546   }
4547   if (!validateIntClampSupported(Inst)) {
4548     Error(getImmLoc(AMDGPUOperand::ImmTyClampSI, Operands),
4549       "integer clamping is not supported on this GPU");
4550     return false;
4551   }
4552   if (!validateOpSel(Inst)) {
4553     Error(getImmLoc(AMDGPUOperand::ImmTyOpSel, Operands),
4554       "invalid op_sel operand");
4555     return false;
4556   }
4557   if (!validateDPP(Inst, Operands)) {
4558     return false;
4559   }
4560   // For MUBUF/MTBUF d16 is a part of opcode, so there is nothing to validate.
4561   if (!validateMIMGD16(Inst)) {
4562     Error(getImmLoc(AMDGPUOperand::ImmTyD16, Operands),
4563       "d16 modifier is not supported on this GPU");
4564     return false;
4565   }
4566   if (!validateMIMGDim(Inst)) {
4567     Error(IDLoc, "dim modifier is required on this GPU");
4568     return false;
4569   }
4570   if (!validateMIMGMSAA(Inst)) {
4571     Error(getImmLoc(AMDGPUOperand::ImmTyDim, Operands),
4572           "invalid dim; must be MSAA type");
4573     return false;
4574   }
4575   if (auto ErrMsg = validateMIMGDataSize(Inst)) {
4576     Error(IDLoc, *ErrMsg);
4577     return false;
4578   }
4579   if (!validateMIMGAddrSize(Inst)) {
4580     Error(IDLoc,
4581       "image address size does not match dim and a16");
4582     return false;
4583   }
4584   if (!validateMIMGAtomicDMask(Inst)) {
4585     Error(getImmLoc(AMDGPUOperand::ImmTyDMask, Operands),
4586       "invalid atomic image dmask");
4587     return false;
4588   }
4589   if (!validateMIMGGatherDMask(Inst)) {
4590     Error(getImmLoc(AMDGPUOperand::ImmTyDMask, Operands),
4591       "invalid image_gather dmask: only one bit must be set");
4592     return false;
4593   }
4594   if (!validateMovrels(Inst, Operands)) {
4595     return false;
4596   }
4597   if (!validateFlatOffset(Inst, Operands)) {
4598     return false;
4599   }
4600   if (!validateSMEMOffset(Inst, Operands)) {
4601     return false;
4602   }
4603   if (!validateMAIAccWrite(Inst, Operands)) {
4604     return false;
4605   }
4606   if (!validateMFMA(Inst, Operands)) {
4607     return false;
4608   }
4609   if (!validateCoherencyBits(Inst, Operands, IDLoc)) {
4610     return false;
4611   }
4612 
4613   if (!validateAGPRLdSt(Inst)) {
4614     Error(IDLoc, getFeatureBits()[AMDGPU::FeatureGFX90AInsts]
4615     ? "invalid register class: data and dst should be all VGPR or AGPR"
4616     : "invalid register class: agpr loads and stores not supported on this GPU"
4617     );
4618     return false;
4619   }
4620   if (!validateVGPRAlign(Inst)) {
4621     Error(IDLoc,
4622       "invalid register class: vgpr tuples must be 64 bit aligned");
4623     return false;
4624   }
4625   if (!validateGWS(Inst, Operands)) {
4626     return false;
4627   }
4628 
4629   if (!validateBLGP(Inst, Operands)) {
4630     return false;
4631   }
4632 
4633   if (!validateDivScale(Inst)) {
4634     Error(IDLoc, "ABS not allowed in VOP3B instructions");
4635     return false;
4636   }
4637   if (!validateCoherencyBits(Inst, Operands, IDLoc)) {
4638     return false;
4639   }
4640   if (!validateExeczVcczOperands(Operands)) {
4641     return false;
4642   }
4643 
4644   if (!validateFlatLdsDMA(Inst, Operands, IDLoc)) {
4645     return false;
4646   }
4647 
4648   return true;
4649 }
4650 
4651 static std::string AMDGPUMnemonicSpellCheck(StringRef S,
4652                                             const FeatureBitset &FBS,
4653                                             unsigned VariantID = 0);
4654 
4655 static bool AMDGPUCheckMnemonic(StringRef Mnemonic,
4656                                 const FeatureBitset &AvailableFeatures,
4657                                 unsigned VariantID);
4658 
4659 bool AMDGPUAsmParser::isSupportedMnemo(StringRef Mnemo,
4660                                        const FeatureBitset &FBS) {
4661   return isSupportedMnemo(Mnemo, FBS, getAllVariants());
4662 }
4663 
4664 bool AMDGPUAsmParser::isSupportedMnemo(StringRef Mnemo,
4665                                        const FeatureBitset &FBS,
4666                                        ArrayRef<unsigned> Variants) {
4667   for (auto Variant : Variants) {
4668     if (AMDGPUCheckMnemonic(Mnemo, FBS, Variant))
4669       return true;
4670   }
4671 
4672   return false;
4673 }
4674 
4675 bool AMDGPUAsmParser::checkUnsupportedInstruction(StringRef Mnemo,
4676                                                   const SMLoc &IDLoc) {
4677   FeatureBitset FBS = ComputeAvailableFeatures(getSTI().getFeatureBits());
4678 
4679   // Check if requested instruction variant is supported.
4680   if (isSupportedMnemo(Mnemo, FBS, getMatchedVariants()))
4681     return false;
4682 
4683   // This instruction is not supported.
4684   // Clear any other pending errors because they are no longer relevant.
4685   getParser().clearPendingErrors();
4686 
4687   // Requested instruction variant is not supported.
4688   // Check if any other variants are supported.
4689   StringRef VariantName = getMatchedVariantName();
4690   if (!VariantName.empty() && isSupportedMnemo(Mnemo, FBS)) {
4691     return Error(IDLoc,
4692                  Twine(VariantName,
4693                        " variant of this instruction is not supported"));
4694   }
4695 
4696   // Finally check if this instruction is supported on any other GPU.
4697   if (isSupportedMnemo(Mnemo, FeatureBitset().set())) {
4698     return Error(IDLoc, "instruction not supported on this GPU");
4699   }
4700 
4701   // Instruction not supported on any GPU. Probably a typo.
4702   std::string Suggestion = AMDGPUMnemonicSpellCheck(Mnemo, FBS);
4703   return Error(IDLoc, "invalid instruction" + Suggestion);
4704 }
4705 
4706 bool AMDGPUAsmParser::MatchAndEmitInstruction(SMLoc IDLoc, unsigned &Opcode,
4707                                               OperandVector &Operands,
4708                                               MCStreamer &Out,
4709                                               uint64_t &ErrorInfo,
4710                                               bool MatchingInlineAsm) {
4711   MCInst Inst;
4712   unsigned Result = Match_Success;
4713   for (auto Variant : getMatchedVariants()) {
4714     uint64_t EI;
4715     auto R = MatchInstructionImpl(Operands, Inst, EI, MatchingInlineAsm,
4716                                   Variant);
4717     // We order match statuses from least to most specific. We use most specific
4718     // status as resulting
4719     // Match_MnemonicFail < Match_InvalidOperand < Match_MissingFeature < Match_PreferE32
4720     if ((R == Match_Success) ||
4721         (R == Match_PreferE32) ||
4722         (R == Match_MissingFeature && Result != Match_PreferE32) ||
4723         (R == Match_InvalidOperand && Result != Match_MissingFeature
4724                                    && Result != Match_PreferE32) ||
4725         (R == Match_MnemonicFail   && Result != Match_InvalidOperand
4726                                    && Result != Match_MissingFeature
4727                                    && Result != Match_PreferE32)) {
4728       Result = R;
4729       ErrorInfo = EI;
4730     }
4731     if (R == Match_Success)
4732       break;
4733   }
4734 
4735   if (Result == Match_Success) {
4736     if (!validateInstruction(Inst, IDLoc, Operands)) {
4737       return true;
4738     }
4739     Inst.setLoc(IDLoc);
4740     Out.emitInstruction(Inst, getSTI());
4741     return false;
4742   }
4743 
4744   StringRef Mnemo = ((AMDGPUOperand &)*Operands[0]).getToken();
4745   if (checkUnsupportedInstruction(Mnemo, IDLoc)) {
4746     return true;
4747   }
4748 
4749   switch (Result) {
4750   default: break;
4751   case Match_MissingFeature:
4752     // It has been verified that the specified instruction
4753     // mnemonic is valid. A match was found but it requires
4754     // features which are not supported on this GPU.
4755     return Error(IDLoc, "operands are not valid for this GPU or mode");
4756 
4757   case Match_InvalidOperand: {
4758     SMLoc ErrorLoc = IDLoc;
4759     if (ErrorInfo != ~0ULL) {
4760       if (ErrorInfo >= Operands.size()) {
4761         return Error(IDLoc, "too few operands for instruction");
4762       }
4763       ErrorLoc = ((AMDGPUOperand &)*Operands[ErrorInfo]).getStartLoc();
4764       if (ErrorLoc == SMLoc())
4765         ErrorLoc = IDLoc;
4766     }
4767     return Error(ErrorLoc, "invalid operand for instruction");
4768   }
4769 
4770   case Match_PreferE32:
4771     return Error(IDLoc, "internal error: instruction without _e64 suffix "
4772                         "should be encoded as e32");
4773   case Match_MnemonicFail:
4774     llvm_unreachable("Invalid instructions should have been handled already");
4775   }
4776   llvm_unreachable("Implement any new match types added!");
4777 }
4778 
4779 bool AMDGPUAsmParser::ParseAsAbsoluteExpression(uint32_t &Ret) {
4780   int64_t Tmp = -1;
4781   if (!isToken(AsmToken::Integer) && !isToken(AsmToken::Identifier)) {
4782     return true;
4783   }
4784   if (getParser().parseAbsoluteExpression(Tmp)) {
4785     return true;
4786   }
4787   Ret = static_cast<uint32_t>(Tmp);
4788   return false;
4789 }
4790 
4791 bool AMDGPUAsmParser::ParseDirectiveMajorMinor(uint32_t &Major,
4792                                                uint32_t &Minor) {
4793   if (ParseAsAbsoluteExpression(Major))
4794     return TokError("invalid major version");
4795 
4796   if (!trySkipToken(AsmToken::Comma))
4797     return TokError("minor version number required, comma expected");
4798 
4799   if (ParseAsAbsoluteExpression(Minor))
4800     return TokError("invalid minor version");
4801 
4802   return false;
4803 }
4804 
4805 bool AMDGPUAsmParser::ParseDirectiveAMDGCNTarget() {
4806   if (getSTI().getTargetTriple().getArch() != Triple::amdgcn)
4807     return TokError("directive only supported for amdgcn architecture");
4808 
4809   std::string TargetIDDirective;
4810   SMLoc TargetStart = getTok().getLoc();
4811   if (getParser().parseEscapedString(TargetIDDirective))
4812     return true;
4813 
4814   SMRange TargetRange = SMRange(TargetStart, getTok().getLoc());
4815   if (getTargetStreamer().getTargetID()->toString() != TargetIDDirective)
4816     return getParser().Error(TargetRange.Start,
4817         (Twine(".amdgcn_target directive's target id ") +
4818          Twine(TargetIDDirective) +
4819          Twine(" does not match the specified target id ") +
4820          Twine(getTargetStreamer().getTargetID()->toString())).str());
4821 
4822   return false;
4823 }
4824 
4825 bool AMDGPUAsmParser::OutOfRangeError(SMRange Range) {
4826   return Error(Range.Start, "value out of range", Range);
4827 }
4828 
4829 bool AMDGPUAsmParser::calculateGPRBlocks(
4830     const FeatureBitset &Features, bool VCCUsed, bool FlatScrUsed,
4831     bool XNACKUsed, Optional<bool> EnableWavefrontSize32, unsigned NextFreeVGPR,
4832     SMRange VGPRRange, unsigned NextFreeSGPR, SMRange SGPRRange,
4833     unsigned &VGPRBlocks, unsigned &SGPRBlocks) {
4834   // TODO(scott.linder): These calculations are duplicated from
4835   // AMDGPUAsmPrinter::getSIProgramInfo and could be unified.
4836   IsaVersion Version = getIsaVersion(getSTI().getCPU());
4837 
4838   unsigned NumVGPRs = NextFreeVGPR;
4839   unsigned NumSGPRs = NextFreeSGPR;
4840 
4841   if (Version.Major >= 10)
4842     NumSGPRs = 0;
4843   else {
4844     unsigned MaxAddressableNumSGPRs =
4845         IsaInfo::getAddressableNumSGPRs(&getSTI());
4846 
4847     if (Version.Major >= 8 && !Features.test(FeatureSGPRInitBug) &&
4848         NumSGPRs > MaxAddressableNumSGPRs)
4849       return OutOfRangeError(SGPRRange);
4850 
4851     NumSGPRs +=
4852         IsaInfo::getNumExtraSGPRs(&getSTI(), VCCUsed, FlatScrUsed, XNACKUsed);
4853 
4854     if ((Version.Major <= 7 || Features.test(FeatureSGPRInitBug)) &&
4855         NumSGPRs > MaxAddressableNumSGPRs)
4856       return OutOfRangeError(SGPRRange);
4857 
4858     if (Features.test(FeatureSGPRInitBug))
4859       NumSGPRs = IsaInfo::FIXED_NUM_SGPRS_FOR_INIT_BUG;
4860   }
4861 
4862   VGPRBlocks =
4863       IsaInfo::getNumVGPRBlocks(&getSTI(), NumVGPRs, EnableWavefrontSize32);
4864   SGPRBlocks = IsaInfo::getNumSGPRBlocks(&getSTI(), NumSGPRs);
4865 
4866   return false;
4867 }
4868 
4869 bool AMDGPUAsmParser::ParseDirectiveAMDHSAKernel() {
4870   if (getSTI().getTargetTriple().getArch() != Triple::amdgcn)
4871     return TokError("directive only supported for amdgcn architecture");
4872 
4873   if (getSTI().getTargetTriple().getOS() != Triple::AMDHSA)
4874     return TokError("directive only supported for amdhsa OS");
4875 
4876   StringRef KernelName;
4877   if (getParser().parseIdentifier(KernelName))
4878     return true;
4879 
4880   kernel_descriptor_t KD = getDefaultAmdhsaKernelDescriptor(&getSTI());
4881 
4882   StringSet<> Seen;
4883 
4884   IsaVersion IVersion = getIsaVersion(getSTI().getCPU());
4885 
4886   SMRange VGPRRange;
4887   uint64_t NextFreeVGPR = 0;
4888   uint64_t AccumOffset = 0;
4889   uint64_t SharedVGPRCount = 0;
4890   SMRange SGPRRange;
4891   uint64_t NextFreeSGPR = 0;
4892 
4893   // Count the number of user SGPRs implied from the enabled feature bits.
4894   unsigned ImpliedUserSGPRCount = 0;
4895 
4896   // Track if the asm explicitly contains the directive for the user SGPR
4897   // count.
4898   Optional<unsigned> ExplicitUserSGPRCount;
4899   bool ReserveVCC = true;
4900   bool ReserveFlatScr = true;
4901   Optional<bool> EnableWavefrontSize32;
4902 
4903   while (true) {
4904     while (trySkipToken(AsmToken::EndOfStatement));
4905 
4906     StringRef ID;
4907     SMRange IDRange = getTok().getLocRange();
4908     if (!parseId(ID, "expected .amdhsa_ directive or .end_amdhsa_kernel"))
4909       return true;
4910 
4911     if (ID == ".end_amdhsa_kernel")
4912       break;
4913 
4914     if (Seen.find(ID) != Seen.end())
4915       return TokError(".amdhsa_ directives cannot be repeated");
4916     Seen.insert(ID);
4917 
4918     SMLoc ValStart = getLoc();
4919     int64_t IVal;
4920     if (getParser().parseAbsoluteExpression(IVal))
4921       return true;
4922     SMLoc ValEnd = getLoc();
4923     SMRange ValRange = SMRange(ValStart, ValEnd);
4924 
4925     if (IVal < 0)
4926       return OutOfRangeError(ValRange);
4927 
4928     uint64_t Val = IVal;
4929 
4930 #define PARSE_BITS_ENTRY(FIELD, ENTRY, VALUE, RANGE)                           \
4931   if (!isUInt<ENTRY##_WIDTH>(VALUE))                                           \
4932     return OutOfRangeError(RANGE);                                             \
4933   AMDHSA_BITS_SET(FIELD, ENTRY, VALUE);
4934 
4935     if (ID == ".amdhsa_group_segment_fixed_size") {
4936       if (!isUInt<sizeof(KD.group_segment_fixed_size) * CHAR_BIT>(Val))
4937         return OutOfRangeError(ValRange);
4938       KD.group_segment_fixed_size = Val;
4939     } else if (ID == ".amdhsa_private_segment_fixed_size") {
4940       if (!isUInt<sizeof(KD.private_segment_fixed_size) * CHAR_BIT>(Val))
4941         return OutOfRangeError(ValRange);
4942       KD.private_segment_fixed_size = Val;
4943     } else if (ID == ".amdhsa_kernarg_size") {
4944       if (!isUInt<sizeof(KD.kernarg_size) * CHAR_BIT>(Val))
4945         return OutOfRangeError(ValRange);
4946       KD.kernarg_size = Val;
4947     } else if (ID == ".amdhsa_user_sgpr_count") {
4948       ExplicitUserSGPRCount = Val;
4949     } else if (ID == ".amdhsa_user_sgpr_private_segment_buffer") {
4950       if (hasArchitectedFlatScratch())
4951         return Error(IDRange.Start,
4952                      "directive is not supported with architected flat scratch",
4953                      IDRange);
4954       PARSE_BITS_ENTRY(KD.kernel_code_properties,
4955                        KERNEL_CODE_PROPERTY_ENABLE_SGPR_PRIVATE_SEGMENT_BUFFER,
4956                        Val, ValRange);
4957       if (Val)
4958         ImpliedUserSGPRCount += 4;
4959     } else if (ID == ".amdhsa_user_sgpr_dispatch_ptr") {
4960       PARSE_BITS_ENTRY(KD.kernel_code_properties,
4961                        KERNEL_CODE_PROPERTY_ENABLE_SGPR_DISPATCH_PTR, Val,
4962                        ValRange);
4963       if (Val)
4964         ImpliedUserSGPRCount += 2;
4965     } else if (ID == ".amdhsa_user_sgpr_queue_ptr") {
4966       PARSE_BITS_ENTRY(KD.kernel_code_properties,
4967                        KERNEL_CODE_PROPERTY_ENABLE_SGPR_QUEUE_PTR, Val,
4968                        ValRange);
4969       if (Val)
4970         ImpliedUserSGPRCount += 2;
4971     } else if (ID == ".amdhsa_user_sgpr_kernarg_segment_ptr") {
4972       PARSE_BITS_ENTRY(KD.kernel_code_properties,
4973                        KERNEL_CODE_PROPERTY_ENABLE_SGPR_KERNARG_SEGMENT_PTR,
4974                        Val, ValRange);
4975       if (Val)
4976         ImpliedUserSGPRCount += 2;
4977     } else if (ID == ".amdhsa_user_sgpr_dispatch_id") {
4978       PARSE_BITS_ENTRY(KD.kernel_code_properties,
4979                        KERNEL_CODE_PROPERTY_ENABLE_SGPR_DISPATCH_ID, Val,
4980                        ValRange);
4981       if (Val)
4982         ImpliedUserSGPRCount += 2;
4983     } else if (ID == ".amdhsa_user_sgpr_flat_scratch_init") {
4984       if (hasArchitectedFlatScratch())
4985         return Error(IDRange.Start,
4986                      "directive is not supported with architected flat scratch",
4987                      IDRange);
4988       PARSE_BITS_ENTRY(KD.kernel_code_properties,
4989                        KERNEL_CODE_PROPERTY_ENABLE_SGPR_FLAT_SCRATCH_INIT, Val,
4990                        ValRange);
4991       if (Val)
4992         ImpliedUserSGPRCount += 2;
4993     } else if (ID == ".amdhsa_user_sgpr_private_segment_size") {
4994       PARSE_BITS_ENTRY(KD.kernel_code_properties,
4995                        KERNEL_CODE_PROPERTY_ENABLE_SGPR_PRIVATE_SEGMENT_SIZE,
4996                        Val, ValRange);
4997       if (Val)
4998         ImpliedUserSGPRCount += 1;
4999     } else if (ID == ".amdhsa_wavefront_size32") {
5000       if (IVersion.Major < 10)
5001         return Error(IDRange.Start, "directive requires gfx10+", IDRange);
5002       EnableWavefrontSize32 = Val;
5003       PARSE_BITS_ENTRY(KD.kernel_code_properties,
5004                        KERNEL_CODE_PROPERTY_ENABLE_WAVEFRONT_SIZE32,
5005                        Val, ValRange);
5006     } else if (ID == ".amdhsa_system_sgpr_private_segment_wavefront_offset") {
5007       if (hasArchitectedFlatScratch())
5008         return Error(IDRange.Start,
5009                      "directive is not supported with architected flat scratch",
5010                      IDRange);
5011       PARSE_BITS_ENTRY(KD.compute_pgm_rsrc2,
5012                        COMPUTE_PGM_RSRC2_ENABLE_PRIVATE_SEGMENT, Val, ValRange);
5013     } else if (ID == ".amdhsa_enable_private_segment") {
5014       if (!hasArchitectedFlatScratch())
5015         return Error(
5016             IDRange.Start,
5017             "directive is not supported without architected flat scratch",
5018             IDRange);
5019       PARSE_BITS_ENTRY(KD.compute_pgm_rsrc2,
5020                        COMPUTE_PGM_RSRC2_ENABLE_PRIVATE_SEGMENT, Val, ValRange);
5021     } else if (ID == ".amdhsa_system_sgpr_workgroup_id_x") {
5022       PARSE_BITS_ENTRY(KD.compute_pgm_rsrc2,
5023                        COMPUTE_PGM_RSRC2_ENABLE_SGPR_WORKGROUP_ID_X, Val,
5024                        ValRange);
5025     } else if (ID == ".amdhsa_system_sgpr_workgroup_id_y") {
5026       PARSE_BITS_ENTRY(KD.compute_pgm_rsrc2,
5027                        COMPUTE_PGM_RSRC2_ENABLE_SGPR_WORKGROUP_ID_Y, Val,
5028                        ValRange);
5029     } else if (ID == ".amdhsa_system_sgpr_workgroup_id_z") {
5030       PARSE_BITS_ENTRY(KD.compute_pgm_rsrc2,
5031                        COMPUTE_PGM_RSRC2_ENABLE_SGPR_WORKGROUP_ID_Z, Val,
5032                        ValRange);
5033     } else if (ID == ".amdhsa_system_sgpr_workgroup_info") {
5034       PARSE_BITS_ENTRY(KD.compute_pgm_rsrc2,
5035                        COMPUTE_PGM_RSRC2_ENABLE_SGPR_WORKGROUP_INFO, Val,
5036                        ValRange);
5037     } else if (ID == ".amdhsa_system_vgpr_workitem_id") {
5038       PARSE_BITS_ENTRY(KD.compute_pgm_rsrc2,
5039                        COMPUTE_PGM_RSRC2_ENABLE_VGPR_WORKITEM_ID, Val,
5040                        ValRange);
5041     } else if (ID == ".amdhsa_next_free_vgpr") {
5042       VGPRRange = ValRange;
5043       NextFreeVGPR = Val;
5044     } else if (ID == ".amdhsa_next_free_sgpr") {
5045       SGPRRange = ValRange;
5046       NextFreeSGPR = Val;
5047     } else if (ID == ".amdhsa_accum_offset") {
5048       if (!isGFX90A())
5049         return Error(IDRange.Start, "directive requires gfx90a+", IDRange);
5050       AccumOffset = Val;
5051     } else if (ID == ".amdhsa_reserve_vcc") {
5052       if (!isUInt<1>(Val))
5053         return OutOfRangeError(ValRange);
5054       ReserveVCC = Val;
5055     } else if (ID == ".amdhsa_reserve_flat_scratch") {
5056       if (IVersion.Major < 7)
5057         return Error(IDRange.Start, "directive requires gfx7+", IDRange);
5058       if (hasArchitectedFlatScratch())
5059         return Error(IDRange.Start,
5060                      "directive is not supported with architected flat scratch",
5061                      IDRange);
5062       if (!isUInt<1>(Val))
5063         return OutOfRangeError(ValRange);
5064       ReserveFlatScr = Val;
5065     } else if (ID == ".amdhsa_reserve_xnack_mask") {
5066       if (IVersion.Major < 8)
5067         return Error(IDRange.Start, "directive requires gfx8+", IDRange);
5068       if (!isUInt<1>(Val))
5069         return OutOfRangeError(ValRange);
5070       if (Val != getTargetStreamer().getTargetID()->isXnackOnOrAny())
5071         return getParser().Error(IDRange.Start, ".amdhsa_reserve_xnack_mask does not match target id",
5072                                  IDRange);
5073     } else if (ID == ".amdhsa_float_round_mode_32") {
5074       PARSE_BITS_ENTRY(KD.compute_pgm_rsrc1,
5075                        COMPUTE_PGM_RSRC1_FLOAT_ROUND_MODE_32, Val, ValRange);
5076     } else if (ID == ".amdhsa_float_round_mode_16_64") {
5077       PARSE_BITS_ENTRY(KD.compute_pgm_rsrc1,
5078                        COMPUTE_PGM_RSRC1_FLOAT_ROUND_MODE_16_64, Val, ValRange);
5079     } else if (ID == ".amdhsa_float_denorm_mode_32") {
5080       PARSE_BITS_ENTRY(KD.compute_pgm_rsrc1,
5081                        COMPUTE_PGM_RSRC1_FLOAT_DENORM_MODE_32, Val, ValRange);
5082     } else if (ID == ".amdhsa_float_denorm_mode_16_64") {
5083       PARSE_BITS_ENTRY(KD.compute_pgm_rsrc1,
5084                        COMPUTE_PGM_RSRC1_FLOAT_DENORM_MODE_16_64, Val,
5085                        ValRange);
5086     } else if (ID == ".amdhsa_dx10_clamp") {
5087       PARSE_BITS_ENTRY(KD.compute_pgm_rsrc1,
5088                        COMPUTE_PGM_RSRC1_ENABLE_DX10_CLAMP, Val, ValRange);
5089     } else if (ID == ".amdhsa_ieee_mode") {
5090       PARSE_BITS_ENTRY(KD.compute_pgm_rsrc1, COMPUTE_PGM_RSRC1_ENABLE_IEEE_MODE,
5091                        Val, ValRange);
5092     } else if (ID == ".amdhsa_fp16_overflow") {
5093       if (IVersion.Major < 9)
5094         return Error(IDRange.Start, "directive requires gfx9+", IDRange);
5095       PARSE_BITS_ENTRY(KD.compute_pgm_rsrc1, COMPUTE_PGM_RSRC1_FP16_OVFL, Val,
5096                        ValRange);
5097     } else if (ID == ".amdhsa_tg_split") {
5098       if (!isGFX90A())
5099         return Error(IDRange.Start, "directive requires gfx90a+", IDRange);
5100       PARSE_BITS_ENTRY(KD.compute_pgm_rsrc3, COMPUTE_PGM_RSRC3_GFX90A_TG_SPLIT, Val,
5101                        ValRange);
5102     } else if (ID == ".amdhsa_workgroup_processor_mode") {
5103       if (IVersion.Major < 10)
5104         return Error(IDRange.Start, "directive requires gfx10+", IDRange);
5105       PARSE_BITS_ENTRY(KD.compute_pgm_rsrc1, COMPUTE_PGM_RSRC1_WGP_MODE, Val,
5106                        ValRange);
5107     } else if (ID == ".amdhsa_memory_ordered") {
5108       if (IVersion.Major < 10)
5109         return Error(IDRange.Start, "directive requires gfx10+", IDRange);
5110       PARSE_BITS_ENTRY(KD.compute_pgm_rsrc1, COMPUTE_PGM_RSRC1_MEM_ORDERED, Val,
5111                        ValRange);
5112     } else if (ID == ".amdhsa_forward_progress") {
5113       if (IVersion.Major < 10)
5114         return Error(IDRange.Start, "directive requires gfx10+", IDRange);
5115       PARSE_BITS_ENTRY(KD.compute_pgm_rsrc1, COMPUTE_PGM_RSRC1_FWD_PROGRESS, Val,
5116                        ValRange);
5117     } else if (ID == ".amdhsa_shared_vgpr_count") {
5118       if (IVersion.Major < 10)
5119         return Error(IDRange.Start, "directive requires gfx10+", IDRange);
5120       SharedVGPRCount = Val;
5121       PARSE_BITS_ENTRY(KD.compute_pgm_rsrc3,
5122                        COMPUTE_PGM_RSRC3_GFX10_PLUS_SHARED_VGPR_COUNT, Val,
5123                        ValRange);
5124     } else if (ID == ".amdhsa_exception_fp_ieee_invalid_op") {
5125       PARSE_BITS_ENTRY(
5126           KD.compute_pgm_rsrc2,
5127           COMPUTE_PGM_RSRC2_ENABLE_EXCEPTION_IEEE_754_FP_INVALID_OPERATION, Val,
5128           ValRange);
5129     } else if (ID == ".amdhsa_exception_fp_denorm_src") {
5130       PARSE_BITS_ENTRY(KD.compute_pgm_rsrc2,
5131                        COMPUTE_PGM_RSRC2_ENABLE_EXCEPTION_FP_DENORMAL_SOURCE,
5132                        Val, ValRange);
5133     } else if (ID == ".amdhsa_exception_fp_ieee_div_zero") {
5134       PARSE_BITS_ENTRY(
5135           KD.compute_pgm_rsrc2,
5136           COMPUTE_PGM_RSRC2_ENABLE_EXCEPTION_IEEE_754_FP_DIVISION_BY_ZERO, Val,
5137           ValRange);
5138     } else if (ID == ".amdhsa_exception_fp_ieee_overflow") {
5139       PARSE_BITS_ENTRY(KD.compute_pgm_rsrc2,
5140                        COMPUTE_PGM_RSRC2_ENABLE_EXCEPTION_IEEE_754_FP_OVERFLOW,
5141                        Val, ValRange);
5142     } else if (ID == ".amdhsa_exception_fp_ieee_underflow") {
5143       PARSE_BITS_ENTRY(KD.compute_pgm_rsrc2,
5144                        COMPUTE_PGM_RSRC2_ENABLE_EXCEPTION_IEEE_754_FP_UNDERFLOW,
5145                        Val, ValRange);
5146     } else if (ID == ".amdhsa_exception_fp_ieee_inexact") {
5147       PARSE_BITS_ENTRY(KD.compute_pgm_rsrc2,
5148                        COMPUTE_PGM_RSRC2_ENABLE_EXCEPTION_IEEE_754_FP_INEXACT,
5149                        Val, ValRange);
5150     } else if (ID == ".amdhsa_exception_int_div_zero") {
5151       PARSE_BITS_ENTRY(KD.compute_pgm_rsrc2,
5152                        COMPUTE_PGM_RSRC2_ENABLE_EXCEPTION_INT_DIVIDE_BY_ZERO,
5153                        Val, ValRange);
5154     } else {
5155       return Error(IDRange.Start, "unknown .amdhsa_kernel directive", IDRange);
5156     }
5157 
5158 #undef PARSE_BITS_ENTRY
5159   }
5160 
5161   if (Seen.find(".amdhsa_next_free_vgpr") == Seen.end())
5162     return TokError(".amdhsa_next_free_vgpr directive is required");
5163 
5164   if (Seen.find(".amdhsa_next_free_sgpr") == Seen.end())
5165     return TokError(".amdhsa_next_free_sgpr directive is required");
5166 
5167   unsigned VGPRBlocks;
5168   unsigned SGPRBlocks;
5169   if (calculateGPRBlocks(getFeatureBits(), ReserveVCC, ReserveFlatScr,
5170                          getTargetStreamer().getTargetID()->isXnackOnOrAny(),
5171                          EnableWavefrontSize32, NextFreeVGPR,
5172                          VGPRRange, NextFreeSGPR, SGPRRange, VGPRBlocks,
5173                          SGPRBlocks))
5174     return true;
5175 
5176   if (!isUInt<COMPUTE_PGM_RSRC1_GRANULATED_WORKITEM_VGPR_COUNT_WIDTH>(
5177           VGPRBlocks))
5178     return OutOfRangeError(VGPRRange);
5179   AMDHSA_BITS_SET(KD.compute_pgm_rsrc1,
5180                   COMPUTE_PGM_RSRC1_GRANULATED_WORKITEM_VGPR_COUNT, VGPRBlocks);
5181 
5182   if (!isUInt<COMPUTE_PGM_RSRC1_GRANULATED_WAVEFRONT_SGPR_COUNT_WIDTH>(
5183           SGPRBlocks))
5184     return OutOfRangeError(SGPRRange);
5185   AMDHSA_BITS_SET(KD.compute_pgm_rsrc1,
5186                   COMPUTE_PGM_RSRC1_GRANULATED_WAVEFRONT_SGPR_COUNT,
5187                   SGPRBlocks);
5188 
5189   if (ExplicitUserSGPRCount && ImpliedUserSGPRCount > *ExplicitUserSGPRCount)
5190     return TokError("amdgpu_user_sgpr_count smaller than than implied by "
5191                     "enabled user SGPRs");
5192 
5193   unsigned UserSGPRCount =
5194       ExplicitUserSGPRCount ? *ExplicitUserSGPRCount : ImpliedUserSGPRCount;
5195 
5196   if (!isUInt<COMPUTE_PGM_RSRC2_USER_SGPR_COUNT_WIDTH>(UserSGPRCount))
5197     return TokError("too many user SGPRs enabled");
5198   AMDHSA_BITS_SET(KD.compute_pgm_rsrc2, COMPUTE_PGM_RSRC2_USER_SGPR_COUNT,
5199                   UserSGPRCount);
5200 
5201   if (isGFX90A()) {
5202     if (Seen.find(".amdhsa_accum_offset") == Seen.end())
5203       return TokError(".amdhsa_accum_offset directive is required");
5204     if (AccumOffset < 4 || AccumOffset > 256 || (AccumOffset & 3))
5205       return TokError("accum_offset should be in range [4..256] in "
5206                       "increments of 4");
5207     if (AccumOffset > alignTo(std::max((uint64_t)1, NextFreeVGPR), 4))
5208       return TokError("accum_offset exceeds total VGPR allocation");
5209     AMDHSA_BITS_SET(KD.compute_pgm_rsrc3, COMPUTE_PGM_RSRC3_GFX90A_ACCUM_OFFSET,
5210                     (AccumOffset / 4 - 1));
5211   }
5212 
5213   if (IVersion.Major == 10) {
5214     // SharedVGPRCount < 16 checked by PARSE_ENTRY_BITS
5215     if (SharedVGPRCount && EnableWavefrontSize32) {
5216       return TokError("shared_vgpr_count directive not valid on "
5217                       "wavefront size 32");
5218     }
5219     if (SharedVGPRCount * 2 + VGPRBlocks > 63) {
5220       return TokError("shared_vgpr_count*2 + "
5221                       "compute_pgm_rsrc1.GRANULATED_WORKITEM_VGPR_COUNT cannot "
5222                       "exceed 63\n");
5223     }
5224   }
5225 
5226   getTargetStreamer().EmitAmdhsaKernelDescriptor(
5227       getSTI(), KernelName, KD, NextFreeVGPR, NextFreeSGPR, ReserveVCC,
5228       ReserveFlatScr);
5229   return false;
5230 }
5231 
5232 bool AMDGPUAsmParser::ParseDirectiveHSACodeObjectVersion() {
5233   uint32_t Major;
5234   uint32_t Minor;
5235 
5236   if (ParseDirectiveMajorMinor(Major, Minor))
5237     return true;
5238 
5239   getTargetStreamer().EmitDirectiveHSACodeObjectVersion(Major, Minor);
5240   return false;
5241 }
5242 
5243 bool AMDGPUAsmParser::ParseDirectiveHSACodeObjectISA() {
5244   uint32_t Major;
5245   uint32_t Minor;
5246   uint32_t Stepping;
5247   StringRef VendorName;
5248   StringRef ArchName;
5249 
5250   // If this directive has no arguments, then use the ISA version for the
5251   // targeted GPU.
5252   if (isToken(AsmToken::EndOfStatement)) {
5253     AMDGPU::IsaVersion ISA = AMDGPU::getIsaVersion(getSTI().getCPU());
5254     getTargetStreamer().EmitDirectiveHSACodeObjectISAV2(ISA.Major, ISA.Minor,
5255                                                         ISA.Stepping,
5256                                                         "AMD", "AMDGPU");
5257     return false;
5258   }
5259 
5260   if (ParseDirectiveMajorMinor(Major, Minor))
5261     return true;
5262 
5263   if (!trySkipToken(AsmToken::Comma))
5264     return TokError("stepping version number required, comma expected");
5265 
5266   if (ParseAsAbsoluteExpression(Stepping))
5267     return TokError("invalid stepping version");
5268 
5269   if (!trySkipToken(AsmToken::Comma))
5270     return TokError("vendor name required, comma expected");
5271 
5272   if (!parseString(VendorName, "invalid vendor name"))
5273     return true;
5274 
5275   if (!trySkipToken(AsmToken::Comma))
5276     return TokError("arch name required, comma expected");
5277 
5278   if (!parseString(ArchName, "invalid arch name"))
5279     return true;
5280 
5281   getTargetStreamer().EmitDirectiveHSACodeObjectISAV2(Major, Minor, Stepping,
5282                                                       VendorName, ArchName);
5283   return false;
5284 }
5285 
5286 bool AMDGPUAsmParser::ParseAMDKernelCodeTValue(StringRef ID,
5287                                                amd_kernel_code_t &Header) {
5288   // max_scratch_backing_memory_byte_size is deprecated. Ignore it while parsing
5289   // assembly for backwards compatibility.
5290   if (ID == "max_scratch_backing_memory_byte_size") {
5291     Parser.eatToEndOfStatement();
5292     return false;
5293   }
5294 
5295   SmallString<40> ErrStr;
5296   raw_svector_ostream Err(ErrStr);
5297   if (!parseAmdKernelCodeField(ID, getParser(), Header, Err)) {
5298     return TokError(Err.str());
5299   }
5300   Lex();
5301 
5302   if (ID == "enable_wavefront_size32") {
5303     if (Header.code_properties & AMD_CODE_PROPERTY_ENABLE_WAVEFRONT_SIZE32) {
5304       if (!isGFX10Plus())
5305         return TokError("enable_wavefront_size32=1 is only allowed on GFX10+");
5306       if (!getFeatureBits()[AMDGPU::FeatureWavefrontSize32])
5307         return TokError("enable_wavefront_size32=1 requires +WavefrontSize32");
5308     } else {
5309       if (!getFeatureBits()[AMDGPU::FeatureWavefrontSize64])
5310         return TokError("enable_wavefront_size32=0 requires +WavefrontSize64");
5311     }
5312   }
5313 
5314   if (ID == "wavefront_size") {
5315     if (Header.wavefront_size == 5) {
5316       if (!isGFX10Plus())
5317         return TokError("wavefront_size=5 is only allowed on GFX10+");
5318       if (!getFeatureBits()[AMDGPU::FeatureWavefrontSize32])
5319         return TokError("wavefront_size=5 requires +WavefrontSize32");
5320     } else if (Header.wavefront_size == 6) {
5321       if (!getFeatureBits()[AMDGPU::FeatureWavefrontSize64])
5322         return TokError("wavefront_size=6 requires +WavefrontSize64");
5323     }
5324   }
5325 
5326   if (ID == "enable_wgp_mode") {
5327     if (G_00B848_WGP_MODE(Header.compute_pgm_resource_registers) &&
5328         !isGFX10Plus())
5329       return TokError("enable_wgp_mode=1 is only allowed on GFX10+");
5330   }
5331 
5332   if (ID == "enable_mem_ordered") {
5333     if (G_00B848_MEM_ORDERED(Header.compute_pgm_resource_registers) &&
5334         !isGFX10Plus())
5335       return TokError("enable_mem_ordered=1 is only allowed on GFX10+");
5336   }
5337 
5338   if (ID == "enable_fwd_progress") {
5339     if (G_00B848_FWD_PROGRESS(Header.compute_pgm_resource_registers) &&
5340         !isGFX10Plus())
5341       return TokError("enable_fwd_progress=1 is only allowed on GFX10+");
5342   }
5343 
5344   return false;
5345 }
5346 
5347 bool AMDGPUAsmParser::ParseDirectiveAMDKernelCodeT() {
5348   amd_kernel_code_t Header;
5349   AMDGPU::initDefaultAMDKernelCodeT(Header, &getSTI());
5350 
5351   while (true) {
5352     // Lex EndOfStatement.  This is in a while loop, because lexing a comment
5353     // will set the current token to EndOfStatement.
5354     while(trySkipToken(AsmToken::EndOfStatement));
5355 
5356     StringRef ID;
5357     if (!parseId(ID, "expected value identifier or .end_amd_kernel_code_t"))
5358       return true;
5359 
5360     if (ID == ".end_amd_kernel_code_t")
5361       break;
5362 
5363     if (ParseAMDKernelCodeTValue(ID, Header))
5364       return true;
5365   }
5366 
5367   getTargetStreamer().EmitAMDKernelCodeT(Header);
5368 
5369   return false;
5370 }
5371 
5372 bool AMDGPUAsmParser::ParseDirectiveAMDGPUHsaKernel() {
5373   StringRef KernelName;
5374   if (!parseId(KernelName, "expected symbol name"))
5375     return true;
5376 
5377   getTargetStreamer().EmitAMDGPUSymbolType(KernelName,
5378                                            ELF::STT_AMDGPU_HSA_KERNEL);
5379 
5380   KernelScope.initialize(getContext());
5381   return false;
5382 }
5383 
5384 bool AMDGPUAsmParser::ParseDirectiveISAVersion() {
5385   if (getSTI().getTargetTriple().getArch() != Triple::amdgcn) {
5386     return Error(getLoc(),
5387                  ".amd_amdgpu_isa directive is not available on non-amdgcn "
5388                  "architectures");
5389   }
5390 
5391   auto TargetIDDirective = getLexer().getTok().getStringContents();
5392   if (getTargetStreamer().getTargetID()->toString() != TargetIDDirective)
5393     return Error(getParser().getTok().getLoc(), "target id must match options");
5394 
5395   getTargetStreamer().EmitISAVersion();
5396   Lex();
5397 
5398   return false;
5399 }
5400 
5401 bool AMDGPUAsmParser::ParseDirectiveHSAMetadata() {
5402   const char *AssemblerDirectiveBegin;
5403   const char *AssemblerDirectiveEnd;
5404   std::tie(AssemblerDirectiveBegin, AssemblerDirectiveEnd) =
5405       isHsaAbiVersion3AndAbove(&getSTI())
5406           ? std::make_tuple(HSAMD::V3::AssemblerDirectiveBegin,
5407                             HSAMD::V3::AssemblerDirectiveEnd)
5408           : std::make_tuple(HSAMD::AssemblerDirectiveBegin,
5409                             HSAMD::AssemblerDirectiveEnd);
5410 
5411   if (getSTI().getTargetTriple().getOS() != Triple::AMDHSA) {
5412     return Error(getLoc(),
5413                  (Twine(AssemblerDirectiveBegin) + Twine(" directive is "
5414                  "not available on non-amdhsa OSes")).str());
5415   }
5416 
5417   std::string HSAMetadataString;
5418   if (ParseToEndDirective(AssemblerDirectiveBegin, AssemblerDirectiveEnd,
5419                           HSAMetadataString))
5420     return true;
5421 
5422   if (isHsaAbiVersion3AndAbove(&getSTI())) {
5423     if (!getTargetStreamer().EmitHSAMetadataV3(HSAMetadataString))
5424       return Error(getLoc(), "invalid HSA metadata");
5425   } else {
5426     if (!getTargetStreamer().EmitHSAMetadataV2(HSAMetadataString))
5427       return Error(getLoc(), "invalid HSA metadata");
5428   }
5429 
5430   return false;
5431 }
5432 
5433 /// Common code to parse out a block of text (typically YAML) between start and
5434 /// end directives.
5435 bool AMDGPUAsmParser::ParseToEndDirective(const char *AssemblerDirectiveBegin,
5436                                           const char *AssemblerDirectiveEnd,
5437                                           std::string &CollectString) {
5438 
5439   raw_string_ostream CollectStream(CollectString);
5440 
5441   getLexer().setSkipSpace(false);
5442 
5443   bool FoundEnd = false;
5444   while (!isToken(AsmToken::Eof)) {
5445     while (isToken(AsmToken::Space)) {
5446       CollectStream << getTokenStr();
5447       Lex();
5448     }
5449 
5450     if (trySkipId(AssemblerDirectiveEnd)) {
5451       FoundEnd = true;
5452       break;
5453     }
5454 
5455     CollectStream << Parser.parseStringToEndOfStatement()
5456                   << getContext().getAsmInfo()->getSeparatorString();
5457 
5458     Parser.eatToEndOfStatement();
5459   }
5460 
5461   getLexer().setSkipSpace(true);
5462 
5463   if (isToken(AsmToken::Eof) && !FoundEnd) {
5464     return TokError(Twine("expected directive ") +
5465                     Twine(AssemblerDirectiveEnd) + Twine(" not found"));
5466   }
5467 
5468   CollectStream.flush();
5469   return false;
5470 }
5471 
5472 /// Parse the assembler directive for new MsgPack-format PAL metadata.
5473 bool AMDGPUAsmParser::ParseDirectivePALMetadataBegin() {
5474   std::string String;
5475   if (ParseToEndDirective(AMDGPU::PALMD::AssemblerDirectiveBegin,
5476                           AMDGPU::PALMD::AssemblerDirectiveEnd, String))
5477     return true;
5478 
5479   auto PALMetadata = getTargetStreamer().getPALMetadata();
5480   if (!PALMetadata->setFromString(String))
5481     return Error(getLoc(), "invalid PAL metadata");
5482   return false;
5483 }
5484 
5485 /// Parse the assembler directive for old linear-format PAL metadata.
5486 bool AMDGPUAsmParser::ParseDirectivePALMetadata() {
5487   if (getSTI().getTargetTriple().getOS() != Triple::AMDPAL) {
5488     return Error(getLoc(),
5489                  (Twine(PALMD::AssemblerDirective) + Twine(" directive is "
5490                  "not available on non-amdpal OSes")).str());
5491   }
5492 
5493   auto PALMetadata = getTargetStreamer().getPALMetadata();
5494   PALMetadata->setLegacy();
5495   for (;;) {
5496     uint32_t Key, Value;
5497     if (ParseAsAbsoluteExpression(Key)) {
5498       return TokError(Twine("invalid value in ") +
5499                       Twine(PALMD::AssemblerDirective));
5500     }
5501     if (!trySkipToken(AsmToken::Comma)) {
5502       return TokError(Twine("expected an even number of values in ") +
5503                       Twine(PALMD::AssemblerDirective));
5504     }
5505     if (ParseAsAbsoluteExpression(Value)) {
5506       return TokError(Twine("invalid value in ") +
5507                       Twine(PALMD::AssemblerDirective));
5508     }
5509     PALMetadata->setRegister(Key, Value);
5510     if (!trySkipToken(AsmToken::Comma))
5511       break;
5512   }
5513   return false;
5514 }
5515 
5516 /// ParseDirectiveAMDGPULDS
5517 ///  ::= .amdgpu_lds identifier ',' size_expression [',' align_expression]
5518 bool AMDGPUAsmParser::ParseDirectiveAMDGPULDS() {
5519   if (getParser().checkForValidSection())
5520     return true;
5521 
5522   StringRef Name;
5523   SMLoc NameLoc = getLoc();
5524   if (getParser().parseIdentifier(Name))
5525     return TokError("expected identifier in directive");
5526 
5527   MCSymbol *Symbol = getContext().getOrCreateSymbol(Name);
5528   if (parseToken(AsmToken::Comma, "expected ','"))
5529     return true;
5530 
5531   unsigned LocalMemorySize = AMDGPU::IsaInfo::getLocalMemorySize(&getSTI());
5532 
5533   int64_t Size;
5534   SMLoc SizeLoc = getLoc();
5535   if (getParser().parseAbsoluteExpression(Size))
5536     return true;
5537   if (Size < 0)
5538     return Error(SizeLoc, "size must be non-negative");
5539   if (Size > LocalMemorySize)
5540     return Error(SizeLoc, "size is too large");
5541 
5542   int64_t Alignment = 4;
5543   if (trySkipToken(AsmToken::Comma)) {
5544     SMLoc AlignLoc = getLoc();
5545     if (getParser().parseAbsoluteExpression(Alignment))
5546       return true;
5547     if (Alignment < 0 || !isPowerOf2_64(Alignment))
5548       return Error(AlignLoc, "alignment must be a power of two");
5549 
5550     // Alignment larger than the size of LDS is possible in theory, as long
5551     // as the linker manages to place to symbol at address 0, but we do want
5552     // to make sure the alignment fits nicely into a 32-bit integer.
5553     if (Alignment >= 1u << 31)
5554       return Error(AlignLoc, "alignment is too large");
5555   }
5556 
5557   if (parseEOL())
5558     return true;
5559 
5560   Symbol->redefineIfPossible();
5561   if (!Symbol->isUndefined())
5562     return Error(NameLoc, "invalid symbol redefinition");
5563 
5564   getTargetStreamer().emitAMDGPULDS(Symbol, Size, Align(Alignment));
5565   return false;
5566 }
5567 
5568 bool AMDGPUAsmParser::ParseDirective(AsmToken DirectiveID) {
5569   StringRef IDVal = DirectiveID.getString();
5570 
5571   if (isHsaAbiVersion3AndAbove(&getSTI())) {
5572     if (IDVal == ".amdhsa_kernel")
5573      return ParseDirectiveAMDHSAKernel();
5574 
5575     // TODO: Restructure/combine with PAL metadata directive.
5576     if (IDVal == AMDGPU::HSAMD::V3::AssemblerDirectiveBegin)
5577       return ParseDirectiveHSAMetadata();
5578   } else {
5579     if (IDVal == ".hsa_code_object_version")
5580       return ParseDirectiveHSACodeObjectVersion();
5581 
5582     if (IDVal == ".hsa_code_object_isa")
5583       return ParseDirectiveHSACodeObjectISA();
5584 
5585     if (IDVal == ".amd_kernel_code_t")
5586       return ParseDirectiveAMDKernelCodeT();
5587 
5588     if (IDVal == ".amdgpu_hsa_kernel")
5589       return ParseDirectiveAMDGPUHsaKernel();
5590 
5591     if (IDVal == ".amd_amdgpu_isa")
5592       return ParseDirectiveISAVersion();
5593 
5594     if (IDVal == AMDGPU::HSAMD::AssemblerDirectiveBegin)
5595       return ParseDirectiveHSAMetadata();
5596   }
5597 
5598   if (IDVal == ".amdgcn_target")
5599     return ParseDirectiveAMDGCNTarget();
5600 
5601   if (IDVal == ".amdgpu_lds")
5602     return ParseDirectiveAMDGPULDS();
5603 
5604   if (IDVal == PALMD::AssemblerDirectiveBegin)
5605     return ParseDirectivePALMetadataBegin();
5606 
5607   if (IDVal == PALMD::AssemblerDirective)
5608     return ParseDirectivePALMetadata();
5609 
5610   return true;
5611 }
5612 
5613 bool AMDGPUAsmParser::subtargetHasRegister(const MCRegisterInfo &MRI,
5614                                            unsigned RegNo) {
5615 
5616   if (MRI.regsOverlap(AMDGPU::TTMP12_TTMP13_TTMP14_TTMP15, RegNo))
5617     return isGFX9Plus();
5618 
5619   // GFX10+ has 2 more SGPRs 104 and 105.
5620   if (MRI.regsOverlap(AMDGPU::SGPR104_SGPR105, RegNo))
5621     return hasSGPR104_SGPR105();
5622 
5623   switch (RegNo) {
5624   case AMDGPU::SRC_SHARED_BASE:
5625   case AMDGPU::SRC_SHARED_LIMIT:
5626   case AMDGPU::SRC_PRIVATE_BASE:
5627   case AMDGPU::SRC_PRIVATE_LIMIT:
5628     return isGFX9Plus();
5629   case AMDGPU::SRC_POPS_EXITING_WAVE_ID:
5630     return isGFX9Plus() && !isGFX11Plus();
5631   case AMDGPU::TBA:
5632   case AMDGPU::TBA_LO:
5633   case AMDGPU::TBA_HI:
5634   case AMDGPU::TMA:
5635   case AMDGPU::TMA_LO:
5636   case AMDGPU::TMA_HI:
5637     return !isGFX9Plus();
5638   case AMDGPU::XNACK_MASK:
5639   case AMDGPU::XNACK_MASK_LO:
5640   case AMDGPU::XNACK_MASK_HI:
5641     return (isVI() || isGFX9()) && getTargetStreamer().getTargetID()->isXnackSupported();
5642   case AMDGPU::SGPR_NULL:
5643     return isGFX10Plus();
5644   default:
5645     break;
5646   }
5647 
5648   if (isCI())
5649     return true;
5650 
5651   if (isSI() || isGFX10Plus()) {
5652     // No flat_scr on SI.
5653     // On GFX10Plus flat scratch is not a valid register operand and can only be
5654     // accessed with s_setreg/s_getreg.
5655     switch (RegNo) {
5656     case AMDGPU::FLAT_SCR:
5657     case AMDGPU::FLAT_SCR_LO:
5658     case AMDGPU::FLAT_SCR_HI:
5659       return false;
5660     default:
5661       return true;
5662     }
5663   }
5664 
5665   // VI only has 102 SGPRs, so make sure we aren't trying to use the 2 more that
5666   // SI/CI have.
5667   if (MRI.regsOverlap(AMDGPU::SGPR102_SGPR103, RegNo))
5668     return hasSGPR102_SGPR103();
5669 
5670   return true;
5671 }
5672 
5673 OperandMatchResultTy
5674 AMDGPUAsmParser::parseOperand(OperandVector &Operands, StringRef Mnemonic,
5675                               OperandMode Mode) {
5676   // Try to parse with a custom parser
5677   OperandMatchResultTy ResTy = MatchOperandParserImpl(Operands, Mnemonic);
5678 
5679   // If we successfully parsed the operand or if there as an error parsing,
5680   // we are done.
5681   //
5682   // If we are parsing after we reach EndOfStatement then this means we
5683   // are appending default values to the Operands list.  This is only done
5684   // by custom parser, so we shouldn't continue on to the generic parsing.
5685   if (ResTy == MatchOperand_Success || ResTy == MatchOperand_ParseFail ||
5686       isToken(AsmToken::EndOfStatement))
5687     return ResTy;
5688 
5689   SMLoc RBraceLoc;
5690   SMLoc LBraceLoc = getLoc();
5691   if (Mode == OperandMode_NSA && trySkipToken(AsmToken::LBrac)) {
5692     unsigned Prefix = Operands.size();
5693 
5694     for (;;) {
5695       auto Loc = getLoc();
5696       ResTy = parseReg(Operands);
5697       if (ResTy == MatchOperand_NoMatch)
5698         Error(Loc, "expected a register");
5699       if (ResTy != MatchOperand_Success)
5700         return MatchOperand_ParseFail;
5701 
5702       RBraceLoc = getLoc();
5703       if (trySkipToken(AsmToken::RBrac))
5704         break;
5705 
5706       if (!skipToken(AsmToken::Comma,
5707                      "expected a comma or a closing square bracket")) {
5708         return MatchOperand_ParseFail;
5709       }
5710     }
5711 
5712     if (Operands.size() - Prefix > 1) {
5713       Operands.insert(Operands.begin() + Prefix,
5714                       AMDGPUOperand::CreateToken(this, "[", LBraceLoc));
5715       Operands.push_back(AMDGPUOperand::CreateToken(this, "]", RBraceLoc));
5716     }
5717 
5718     return MatchOperand_Success;
5719   }
5720 
5721   return parseRegOrImm(Operands);
5722 }
5723 
5724 StringRef AMDGPUAsmParser::parseMnemonicSuffix(StringRef Name) {
5725   // Clear any forced encodings from the previous instruction.
5726   setForcedEncodingSize(0);
5727   setForcedDPP(false);
5728   setForcedSDWA(false);
5729 
5730   if (Name.endswith("_e64_dpp")) {
5731     setForcedDPP(true);
5732     setForcedEncodingSize(64);
5733     return Name.substr(0, Name.size() - 8);
5734   } else if (Name.endswith("_e64")) {
5735     setForcedEncodingSize(64);
5736     return Name.substr(0, Name.size() - 4);
5737   } else if (Name.endswith("_e32")) {
5738     setForcedEncodingSize(32);
5739     return Name.substr(0, Name.size() - 4);
5740   } else if (Name.endswith("_dpp")) {
5741     setForcedDPP(true);
5742     return Name.substr(0, Name.size() - 4);
5743   } else if (Name.endswith("_sdwa")) {
5744     setForcedSDWA(true);
5745     return Name.substr(0, Name.size() - 5);
5746   }
5747   return Name;
5748 }
5749 
5750 static void applyMnemonicAliases(StringRef &Mnemonic,
5751                                  const FeatureBitset &Features,
5752                                  unsigned VariantID);
5753 
5754 bool AMDGPUAsmParser::ParseInstruction(ParseInstructionInfo &Info,
5755                                        StringRef Name,
5756                                        SMLoc NameLoc, OperandVector &Operands) {
5757   // Add the instruction mnemonic
5758   Name = parseMnemonicSuffix(Name);
5759 
5760   // If the target architecture uses MnemonicAlias, call it here to parse
5761   // operands correctly.
5762   applyMnemonicAliases(Name, getAvailableFeatures(), 0);
5763 
5764   Operands.push_back(AMDGPUOperand::CreateToken(this, Name, NameLoc));
5765 
5766   bool IsMIMG = Name.startswith("image_");
5767 
5768   while (!trySkipToken(AsmToken::EndOfStatement)) {
5769     OperandMode Mode = OperandMode_Default;
5770     if (IsMIMG && isGFX10Plus() && Operands.size() == 2)
5771       Mode = OperandMode_NSA;
5772     CPolSeen = 0;
5773     OperandMatchResultTy Res = parseOperand(Operands, Name, Mode);
5774 
5775     if (Res != MatchOperand_Success) {
5776       checkUnsupportedInstruction(Name, NameLoc);
5777       if (!Parser.hasPendingError()) {
5778         // FIXME: use real operand location rather than the current location.
5779         StringRef Msg =
5780           (Res == MatchOperand_ParseFail) ? "failed parsing operand." :
5781                                             "not a valid operand.";
5782         Error(getLoc(), Msg);
5783       }
5784       while (!trySkipToken(AsmToken::EndOfStatement)) {
5785         lex();
5786       }
5787       return true;
5788     }
5789 
5790     // Eat the comma or space if there is one.
5791     trySkipToken(AsmToken::Comma);
5792   }
5793 
5794   return false;
5795 }
5796 
5797 //===----------------------------------------------------------------------===//
5798 // Utility functions
5799 //===----------------------------------------------------------------------===//
5800 
5801 OperandMatchResultTy
5802 AMDGPUAsmParser::parseIntWithPrefix(const char *Prefix, int64_t &IntVal) {
5803 
5804   if (!trySkipId(Prefix, AsmToken::Colon))
5805     return MatchOperand_NoMatch;
5806 
5807   return parseExpr(IntVal) ? MatchOperand_Success : MatchOperand_ParseFail;
5808 }
5809 
5810 OperandMatchResultTy
5811 AMDGPUAsmParser::parseIntWithPrefix(const char *Prefix, OperandVector &Operands,
5812                                     AMDGPUOperand::ImmTy ImmTy,
5813                                     bool (*ConvertResult)(int64_t&)) {
5814   SMLoc S = getLoc();
5815   int64_t Value = 0;
5816 
5817   OperandMatchResultTy Res = parseIntWithPrefix(Prefix, Value);
5818   if (Res != MatchOperand_Success)
5819     return Res;
5820 
5821   if (ConvertResult && !ConvertResult(Value)) {
5822     Error(S, "invalid " + StringRef(Prefix) + " value.");
5823   }
5824 
5825   Operands.push_back(AMDGPUOperand::CreateImm(this, Value, S, ImmTy));
5826   return MatchOperand_Success;
5827 }
5828 
5829 OperandMatchResultTy
5830 AMDGPUAsmParser::parseOperandArrayWithPrefix(const char *Prefix,
5831                                              OperandVector &Operands,
5832                                              AMDGPUOperand::ImmTy ImmTy,
5833                                              bool (*ConvertResult)(int64_t&)) {
5834   SMLoc S = getLoc();
5835   if (!trySkipId(Prefix, AsmToken::Colon))
5836     return MatchOperand_NoMatch;
5837 
5838   if (!skipToken(AsmToken::LBrac, "expected a left square bracket"))
5839     return MatchOperand_ParseFail;
5840 
5841   unsigned Val = 0;
5842   const unsigned MaxSize = 4;
5843 
5844   // FIXME: How to verify the number of elements matches the number of src
5845   // operands?
5846   for (int I = 0; ; ++I) {
5847     int64_t Op;
5848     SMLoc Loc = getLoc();
5849     if (!parseExpr(Op))
5850       return MatchOperand_ParseFail;
5851 
5852     if (Op != 0 && Op != 1) {
5853       Error(Loc, "invalid " + StringRef(Prefix) + " value.");
5854       return MatchOperand_ParseFail;
5855     }
5856 
5857     Val |= (Op << I);
5858 
5859     if (trySkipToken(AsmToken::RBrac))
5860       break;
5861 
5862     if (I + 1 == MaxSize) {
5863       Error(getLoc(), "expected a closing square bracket");
5864       return MatchOperand_ParseFail;
5865     }
5866 
5867     if (!skipToken(AsmToken::Comma, "expected a comma"))
5868       return MatchOperand_ParseFail;
5869   }
5870 
5871   Operands.push_back(AMDGPUOperand::CreateImm(this, Val, S, ImmTy));
5872   return MatchOperand_Success;
5873 }
5874 
5875 OperandMatchResultTy
5876 AMDGPUAsmParser::parseNamedBit(StringRef Name, OperandVector &Operands,
5877                                AMDGPUOperand::ImmTy ImmTy) {
5878   int64_t Bit;
5879   SMLoc S = getLoc();
5880 
5881   if (trySkipId(Name)) {
5882     Bit = 1;
5883   } else if (trySkipId("no", Name)) {
5884     Bit = 0;
5885   } else {
5886     return MatchOperand_NoMatch;
5887   }
5888 
5889   if (Name == "r128" && !hasMIMG_R128()) {
5890     Error(S, "r128 modifier is not supported on this GPU");
5891     return MatchOperand_ParseFail;
5892   }
5893   if (Name == "a16" && !isGFX9() && !hasGFX10A16()) {
5894     Error(S, "a16 modifier is not supported on this GPU");
5895     return MatchOperand_ParseFail;
5896   }
5897 
5898   if (isGFX9() && ImmTy == AMDGPUOperand::ImmTyA16)
5899     ImmTy = AMDGPUOperand::ImmTyR128A16;
5900 
5901   Operands.push_back(AMDGPUOperand::CreateImm(this, Bit, S, ImmTy));
5902   return MatchOperand_Success;
5903 }
5904 
5905 OperandMatchResultTy
5906 AMDGPUAsmParser::parseCPol(OperandVector &Operands) {
5907   unsigned CPolOn = 0;
5908   unsigned CPolOff = 0;
5909   SMLoc S = getLoc();
5910 
5911   StringRef Mnemo = ((AMDGPUOperand &)*Operands[0]).getToken();
5912   if (isGFX940() && !Mnemo.startswith("s_")) {
5913     if (trySkipId("sc0"))
5914       CPolOn = AMDGPU::CPol::SC0;
5915     else if (trySkipId("nosc0"))
5916       CPolOff = AMDGPU::CPol::SC0;
5917     else if (trySkipId("nt"))
5918       CPolOn = AMDGPU::CPol::NT;
5919     else if (trySkipId("nont"))
5920       CPolOff = AMDGPU::CPol::NT;
5921     else if (trySkipId("sc1"))
5922       CPolOn = AMDGPU::CPol::SC1;
5923     else if (trySkipId("nosc1"))
5924       CPolOff = AMDGPU::CPol::SC1;
5925     else
5926       return MatchOperand_NoMatch;
5927   }
5928   else if (trySkipId("glc"))
5929     CPolOn = AMDGPU::CPol::GLC;
5930   else if (trySkipId("noglc"))
5931     CPolOff = AMDGPU::CPol::GLC;
5932   else if (trySkipId("slc"))
5933     CPolOn = AMDGPU::CPol::SLC;
5934   else if (trySkipId("noslc"))
5935     CPolOff = AMDGPU::CPol::SLC;
5936   else if (trySkipId("dlc"))
5937     CPolOn = AMDGPU::CPol::DLC;
5938   else if (trySkipId("nodlc"))
5939     CPolOff = AMDGPU::CPol::DLC;
5940   else if (trySkipId("scc"))
5941     CPolOn = AMDGPU::CPol::SCC;
5942   else if (trySkipId("noscc"))
5943     CPolOff = AMDGPU::CPol::SCC;
5944   else
5945     return MatchOperand_NoMatch;
5946 
5947   if (!isGFX10Plus() && ((CPolOn | CPolOff) & AMDGPU::CPol::DLC)) {
5948     Error(S, "dlc modifier is not supported on this GPU");
5949     return MatchOperand_ParseFail;
5950   }
5951 
5952   if (!isGFX90A() && ((CPolOn | CPolOff) & AMDGPU::CPol::SCC)) {
5953     Error(S, "scc modifier is not supported on this GPU");
5954     return MatchOperand_ParseFail;
5955   }
5956 
5957   if (CPolSeen & (CPolOn | CPolOff)) {
5958     Error(S, "duplicate cache policy modifier");
5959     return MatchOperand_ParseFail;
5960   }
5961 
5962   CPolSeen |= (CPolOn | CPolOff);
5963 
5964   for (unsigned I = 1; I != Operands.size(); ++I) {
5965     AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[I]);
5966     if (Op.isCPol()) {
5967       Op.setImm((Op.getImm() | CPolOn) & ~CPolOff);
5968       return MatchOperand_Success;
5969     }
5970   }
5971 
5972   Operands.push_back(AMDGPUOperand::CreateImm(this, CPolOn, S,
5973                                               AMDGPUOperand::ImmTyCPol));
5974 
5975   return MatchOperand_Success;
5976 }
5977 
5978 static void addOptionalImmOperand(
5979   MCInst& Inst, const OperandVector& Operands,
5980   AMDGPUAsmParser::OptionalImmIndexMap& OptionalIdx,
5981   AMDGPUOperand::ImmTy ImmT,
5982   int64_t Default = 0) {
5983   auto i = OptionalIdx.find(ImmT);
5984   if (i != OptionalIdx.end()) {
5985     unsigned Idx = i->second;
5986     ((AMDGPUOperand &)*Operands[Idx]).addImmOperands(Inst, 1);
5987   } else {
5988     Inst.addOperand(MCOperand::createImm(Default));
5989   }
5990 }
5991 
5992 OperandMatchResultTy
5993 AMDGPUAsmParser::parseStringWithPrefix(StringRef Prefix,
5994                                        StringRef &Value,
5995                                        SMLoc &StringLoc) {
5996   if (!trySkipId(Prefix, AsmToken::Colon))
5997     return MatchOperand_NoMatch;
5998 
5999   StringLoc = getLoc();
6000   return parseId(Value, "expected an identifier") ? MatchOperand_Success
6001                                                   : MatchOperand_ParseFail;
6002 }
6003 
6004 //===----------------------------------------------------------------------===//
6005 // MTBUF format
6006 //===----------------------------------------------------------------------===//
6007 
6008 bool AMDGPUAsmParser::tryParseFmt(const char *Pref,
6009                                   int64_t MaxVal,
6010                                   int64_t &Fmt) {
6011   int64_t Val;
6012   SMLoc Loc = getLoc();
6013 
6014   auto Res = parseIntWithPrefix(Pref, Val);
6015   if (Res == MatchOperand_ParseFail)
6016     return false;
6017   if (Res == MatchOperand_NoMatch)
6018     return true;
6019 
6020   if (Val < 0 || Val > MaxVal) {
6021     Error(Loc, Twine("out of range ", StringRef(Pref)));
6022     return false;
6023   }
6024 
6025   Fmt = Val;
6026   return true;
6027 }
6028 
6029 // dfmt and nfmt (in a tbuffer instruction) are parsed as one to allow their
6030 // values to live in a joint format operand in the MCInst encoding.
6031 OperandMatchResultTy
6032 AMDGPUAsmParser::parseDfmtNfmt(int64_t &Format) {
6033   using namespace llvm::AMDGPU::MTBUFFormat;
6034 
6035   int64_t Dfmt = DFMT_UNDEF;
6036   int64_t Nfmt = NFMT_UNDEF;
6037 
6038   // dfmt and nfmt can appear in either order, and each is optional.
6039   for (int I = 0; I < 2; ++I) {
6040     if (Dfmt == DFMT_UNDEF && !tryParseFmt("dfmt", DFMT_MAX, Dfmt))
6041       return MatchOperand_ParseFail;
6042 
6043     if (Nfmt == NFMT_UNDEF && !tryParseFmt("nfmt", NFMT_MAX, Nfmt)) {
6044       return MatchOperand_ParseFail;
6045     }
6046     // Skip optional comma between dfmt/nfmt
6047     // but guard against 2 commas following each other.
6048     if ((Dfmt == DFMT_UNDEF) != (Nfmt == NFMT_UNDEF) &&
6049         !peekToken().is(AsmToken::Comma)) {
6050       trySkipToken(AsmToken::Comma);
6051     }
6052   }
6053 
6054   if (Dfmt == DFMT_UNDEF && Nfmt == NFMT_UNDEF)
6055     return MatchOperand_NoMatch;
6056 
6057   Dfmt = (Dfmt == DFMT_UNDEF) ? DFMT_DEFAULT : Dfmt;
6058   Nfmt = (Nfmt == NFMT_UNDEF) ? NFMT_DEFAULT : Nfmt;
6059 
6060   Format = encodeDfmtNfmt(Dfmt, Nfmt);
6061   return MatchOperand_Success;
6062 }
6063 
6064 OperandMatchResultTy
6065 AMDGPUAsmParser::parseUfmt(int64_t &Format) {
6066   using namespace llvm::AMDGPU::MTBUFFormat;
6067 
6068   int64_t Fmt = UFMT_UNDEF;
6069 
6070   if (!tryParseFmt("format", UFMT_MAX, Fmt))
6071     return MatchOperand_ParseFail;
6072 
6073   if (Fmt == UFMT_UNDEF)
6074     return MatchOperand_NoMatch;
6075 
6076   Format = Fmt;
6077   return MatchOperand_Success;
6078 }
6079 
6080 bool AMDGPUAsmParser::matchDfmtNfmt(int64_t &Dfmt,
6081                                     int64_t &Nfmt,
6082                                     StringRef FormatStr,
6083                                     SMLoc Loc) {
6084   using namespace llvm::AMDGPU::MTBUFFormat;
6085   int64_t Format;
6086 
6087   Format = getDfmt(FormatStr);
6088   if (Format != DFMT_UNDEF) {
6089     Dfmt = Format;
6090     return true;
6091   }
6092 
6093   Format = getNfmt(FormatStr, getSTI());
6094   if (Format != NFMT_UNDEF) {
6095     Nfmt = Format;
6096     return true;
6097   }
6098 
6099   Error(Loc, "unsupported format");
6100   return false;
6101 }
6102 
6103 OperandMatchResultTy
6104 AMDGPUAsmParser::parseSymbolicSplitFormat(StringRef FormatStr,
6105                                           SMLoc FormatLoc,
6106                                           int64_t &Format) {
6107   using namespace llvm::AMDGPU::MTBUFFormat;
6108 
6109   int64_t Dfmt = DFMT_UNDEF;
6110   int64_t Nfmt = NFMT_UNDEF;
6111   if (!matchDfmtNfmt(Dfmt, Nfmt, FormatStr, FormatLoc))
6112     return MatchOperand_ParseFail;
6113 
6114   if (trySkipToken(AsmToken::Comma)) {
6115     StringRef Str;
6116     SMLoc Loc = getLoc();
6117     if (!parseId(Str, "expected a format string") ||
6118         !matchDfmtNfmt(Dfmt, Nfmt, Str, Loc)) {
6119       return MatchOperand_ParseFail;
6120     }
6121     if (Dfmt == DFMT_UNDEF) {
6122       Error(Loc, "duplicate numeric format");
6123       return MatchOperand_ParseFail;
6124     } else if (Nfmt == NFMT_UNDEF) {
6125       Error(Loc, "duplicate data format");
6126       return MatchOperand_ParseFail;
6127     }
6128   }
6129 
6130   Dfmt = (Dfmt == DFMT_UNDEF) ? DFMT_DEFAULT : Dfmt;
6131   Nfmt = (Nfmt == NFMT_UNDEF) ? NFMT_DEFAULT : Nfmt;
6132 
6133   if (isGFX10Plus()) {
6134     auto Ufmt = convertDfmtNfmt2Ufmt(Dfmt, Nfmt, getSTI());
6135     if (Ufmt == UFMT_UNDEF) {
6136       Error(FormatLoc, "unsupported format");
6137       return MatchOperand_ParseFail;
6138     }
6139     Format = Ufmt;
6140   } else {
6141     Format = encodeDfmtNfmt(Dfmt, Nfmt);
6142   }
6143 
6144   return MatchOperand_Success;
6145 }
6146 
6147 OperandMatchResultTy
6148 AMDGPUAsmParser::parseSymbolicUnifiedFormat(StringRef FormatStr,
6149                                             SMLoc Loc,
6150                                             int64_t &Format) {
6151   using namespace llvm::AMDGPU::MTBUFFormat;
6152 
6153   auto Id = getUnifiedFormat(FormatStr, getSTI());
6154   if (Id == UFMT_UNDEF)
6155     return MatchOperand_NoMatch;
6156 
6157   if (!isGFX10Plus()) {
6158     Error(Loc, "unified format is not supported on this GPU");
6159     return MatchOperand_ParseFail;
6160   }
6161 
6162   Format = Id;
6163   return MatchOperand_Success;
6164 }
6165 
6166 OperandMatchResultTy
6167 AMDGPUAsmParser::parseNumericFormat(int64_t &Format) {
6168   using namespace llvm::AMDGPU::MTBUFFormat;
6169   SMLoc Loc = getLoc();
6170 
6171   if (!parseExpr(Format))
6172     return MatchOperand_ParseFail;
6173   if (!isValidFormatEncoding(Format, getSTI())) {
6174     Error(Loc, "out of range format");
6175     return MatchOperand_ParseFail;
6176   }
6177 
6178   return MatchOperand_Success;
6179 }
6180 
6181 OperandMatchResultTy
6182 AMDGPUAsmParser::parseSymbolicOrNumericFormat(int64_t &Format) {
6183   using namespace llvm::AMDGPU::MTBUFFormat;
6184 
6185   if (!trySkipId("format", AsmToken::Colon))
6186     return MatchOperand_NoMatch;
6187 
6188   if (trySkipToken(AsmToken::LBrac)) {
6189     StringRef FormatStr;
6190     SMLoc Loc = getLoc();
6191     if (!parseId(FormatStr, "expected a format string"))
6192       return MatchOperand_ParseFail;
6193 
6194     auto Res = parseSymbolicUnifiedFormat(FormatStr, Loc, Format);
6195     if (Res == MatchOperand_NoMatch)
6196       Res = parseSymbolicSplitFormat(FormatStr, Loc, Format);
6197     if (Res != MatchOperand_Success)
6198       return Res;
6199 
6200     if (!skipToken(AsmToken::RBrac, "expected a closing square bracket"))
6201       return MatchOperand_ParseFail;
6202 
6203     return MatchOperand_Success;
6204   }
6205 
6206   return parseNumericFormat(Format);
6207 }
6208 
6209 OperandMatchResultTy
6210 AMDGPUAsmParser::parseFORMAT(OperandVector &Operands) {
6211   using namespace llvm::AMDGPU::MTBUFFormat;
6212 
6213   int64_t Format = getDefaultFormatEncoding(getSTI());
6214   OperandMatchResultTy Res;
6215   SMLoc Loc = getLoc();
6216 
6217   // Parse legacy format syntax.
6218   Res = isGFX10Plus() ? parseUfmt(Format) : parseDfmtNfmt(Format);
6219   if (Res == MatchOperand_ParseFail)
6220     return Res;
6221 
6222   bool FormatFound = (Res == MatchOperand_Success);
6223 
6224   Operands.push_back(
6225     AMDGPUOperand::CreateImm(this, Format, Loc, AMDGPUOperand::ImmTyFORMAT));
6226 
6227   if (FormatFound)
6228     trySkipToken(AsmToken::Comma);
6229 
6230   if (isToken(AsmToken::EndOfStatement)) {
6231     // We are expecting an soffset operand,
6232     // but let matcher handle the error.
6233     return MatchOperand_Success;
6234   }
6235 
6236   // Parse soffset.
6237   Res = parseRegOrImm(Operands);
6238   if (Res != MatchOperand_Success)
6239     return Res;
6240 
6241   trySkipToken(AsmToken::Comma);
6242 
6243   if (!FormatFound) {
6244     Res = parseSymbolicOrNumericFormat(Format);
6245     if (Res == MatchOperand_ParseFail)
6246       return Res;
6247     if (Res == MatchOperand_Success) {
6248       auto Size = Operands.size();
6249       AMDGPUOperand &Op = static_cast<AMDGPUOperand &>(*Operands[Size - 2]);
6250       assert(Op.isImm() && Op.getImmTy() == AMDGPUOperand::ImmTyFORMAT);
6251       Op.setImm(Format);
6252     }
6253     return MatchOperand_Success;
6254   }
6255 
6256   if (isId("format") && peekToken().is(AsmToken::Colon)) {
6257     Error(getLoc(), "duplicate format");
6258     return MatchOperand_ParseFail;
6259   }
6260   return MatchOperand_Success;
6261 }
6262 
6263 //===----------------------------------------------------------------------===//
6264 // ds
6265 //===----------------------------------------------------------------------===//
6266 
6267 void AMDGPUAsmParser::cvtDSOffset01(MCInst &Inst,
6268                                     const OperandVector &Operands) {
6269   OptionalImmIndexMap OptionalIdx;
6270 
6271   for (unsigned i = 1, e = Operands.size(); i != e; ++i) {
6272     AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[i]);
6273 
6274     // Add the register arguments
6275     if (Op.isReg()) {
6276       Op.addRegOperands(Inst, 1);
6277       continue;
6278     }
6279 
6280     // Handle optional arguments
6281     OptionalIdx[Op.getImmTy()] = i;
6282   }
6283 
6284   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyOffset0);
6285   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyOffset1);
6286   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyGDS);
6287 
6288   Inst.addOperand(MCOperand::createReg(AMDGPU::M0)); // m0
6289 }
6290 
6291 void AMDGPUAsmParser::cvtDSImpl(MCInst &Inst, const OperandVector &Operands,
6292                                 bool IsGdsHardcoded) {
6293   OptionalImmIndexMap OptionalIdx;
6294   AMDGPUOperand::ImmTy OffsetType = AMDGPUOperand::ImmTyOffset;
6295 
6296   for (unsigned i = 1, e = Operands.size(); i != e; ++i) {
6297     AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[i]);
6298 
6299     // Add the register arguments
6300     if (Op.isReg()) {
6301       Op.addRegOperands(Inst, 1);
6302       continue;
6303     }
6304 
6305     if (Op.isToken() && Op.getToken() == "gds") {
6306       IsGdsHardcoded = true;
6307       continue;
6308     }
6309 
6310     // Handle optional arguments
6311     OptionalIdx[Op.getImmTy()] = i;
6312 
6313     if (Op.getImmTy() == AMDGPUOperand::ImmTySwizzle)
6314       OffsetType = AMDGPUOperand::ImmTySwizzle;
6315   }
6316 
6317   addOptionalImmOperand(Inst, Operands, OptionalIdx, OffsetType);
6318 
6319   if (!IsGdsHardcoded) {
6320     addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyGDS);
6321   }
6322   Inst.addOperand(MCOperand::createReg(AMDGPU::M0)); // m0
6323 }
6324 
6325 void AMDGPUAsmParser::cvtExp(MCInst &Inst, const OperandVector &Operands) {
6326   OptionalImmIndexMap OptionalIdx;
6327 
6328   unsigned OperandIdx[4];
6329   unsigned EnMask = 0;
6330   int SrcIdx = 0;
6331 
6332   for (unsigned i = 1, e = Operands.size(); i != e; ++i) {
6333     AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[i]);
6334 
6335     // Add the register arguments
6336     if (Op.isReg()) {
6337       assert(SrcIdx < 4);
6338       OperandIdx[SrcIdx] = Inst.size();
6339       Op.addRegOperands(Inst, 1);
6340       ++SrcIdx;
6341       continue;
6342     }
6343 
6344     if (Op.isOff()) {
6345       assert(SrcIdx < 4);
6346       OperandIdx[SrcIdx] = Inst.size();
6347       Inst.addOperand(MCOperand::createReg(AMDGPU::NoRegister));
6348       ++SrcIdx;
6349       continue;
6350     }
6351 
6352     if (Op.isImm() && Op.getImmTy() == AMDGPUOperand::ImmTyExpTgt) {
6353       Op.addImmOperands(Inst, 1);
6354       continue;
6355     }
6356 
6357     if (Op.isToken() && (Op.getToken() == "done" || Op.getToken() == "row_en"))
6358       continue;
6359 
6360     // Handle optional arguments
6361     OptionalIdx[Op.getImmTy()] = i;
6362   }
6363 
6364   assert(SrcIdx == 4);
6365 
6366   bool Compr = false;
6367   if (OptionalIdx.find(AMDGPUOperand::ImmTyExpCompr) != OptionalIdx.end()) {
6368     Compr = true;
6369     Inst.getOperand(OperandIdx[1]) = Inst.getOperand(OperandIdx[2]);
6370     Inst.getOperand(OperandIdx[2]).setReg(AMDGPU::NoRegister);
6371     Inst.getOperand(OperandIdx[3]).setReg(AMDGPU::NoRegister);
6372   }
6373 
6374   for (auto i = 0; i < SrcIdx; ++i) {
6375     if (Inst.getOperand(OperandIdx[i]).getReg() != AMDGPU::NoRegister) {
6376       EnMask |= Compr? (0x3 << i * 2) : (0x1 << i);
6377     }
6378   }
6379 
6380   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyExpVM);
6381   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyExpCompr);
6382 
6383   Inst.addOperand(MCOperand::createImm(EnMask));
6384 }
6385 
6386 //===----------------------------------------------------------------------===//
6387 // s_waitcnt
6388 //===----------------------------------------------------------------------===//
6389 
6390 static bool
6391 encodeCnt(
6392   const AMDGPU::IsaVersion ISA,
6393   int64_t &IntVal,
6394   int64_t CntVal,
6395   bool Saturate,
6396   unsigned (*encode)(const IsaVersion &Version, unsigned, unsigned),
6397   unsigned (*decode)(const IsaVersion &Version, unsigned))
6398 {
6399   bool Failed = false;
6400 
6401   IntVal = encode(ISA, IntVal, CntVal);
6402   if (CntVal != decode(ISA, IntVal)) {
6403     if (Saturate) {
6404       IntVal = encode(ISA, IntVal, -1);
6405     } else {
6406       Failed = true;
6407     }
6408   }
6409   return Failed;
6410 }
6411 
6412 bool AMDGPUAsmParser::parseCnt(int64_t &IntVal) {
6413 
6414   SMLoc CntLoc = getLoc();
6415   StringRef CntName = getTokenStr();
6416 
6417   if (!skipToken(AsmToken::Identifier, "expected a counter name") ||
6418       !skipToken(AsmToken::LParen, "expected a left parenthesis"))
6419     return false;
6420 
6421   int64_t CntVal;
6422   SMLoc ValLoc = getLoc();
6423   if (!parseExpr(CntVal))
6424     return false;
6425 
6426   AMDGPU::IsaVersion ISA = AMDGPU::getIsaVersion(getSTI().getCPU());
6427 
6428   bool Failed = true;
6429   bool Sat = CntName.endswith("_sat");
6430 
6431   if (CntName == "vmcnt" || CntName == "vmcnt_sat") {
6432     Failed = encodeCnt(ISA, IntVal, CntVal, Sat, encodeVmcnt, decodeVmcnt);
6433   } else if (CntName == "expcnt" || CntName == "expcnt_sat") {
6434     Failed = encodeCnt(ISA, IntVal, CntVal, Sat, encodeExpcnt, decodeExpcnt);
6435   } else if (CntName == "lgkmcnt" || CntName == "lgkmcnt_sat") {
6436     Failed = encodeCnt(ISA, IntVal, CntVal, Sat, encodeLgkmcnt, decodeLgkmcnt);
6437   } else {
6438     Error(CntLoc, "invalid counter name " + CntName);
6439     return false;
6440   }
6441 
6442   if (Failed) {
6443     Error(ValLoc, "too large value for " + CntName);
6444     return false;
6445   }
6446 
6447   if (!skipToken(AsmToken::RParen, "expected a closing parenthesis"))
6448     return false;
6449 
6450   if (trySkipToken(AsmToken::Amp) || trySkipToken(AsmToken::Comma)) {
6451     if (isToken(AsmToken::EndOfStatement)) {
6452       Error(getLoc(), "expected a counter name");
6453       return false;
6454     }
6455   }
6456 
6457   return true;
6458 }
6459 
6460 OperandMatchResultTy
6461 AMDGPUAsmParser::parseSWaitCntOps(OperandVector &Operands) {
6462   AMDGPU::IsaVersion ISA = AMDGPU::getIsaVersion(getSTI().getCPU());
6463   int64_t Waitcnt = getWaitcntBitMask(ISA);
6464   SMLoc S = getLoc();
6465 
6466   if (isToken(AsmToken::Identifier) && peekToken().is(AsmToken::LParen)) {
6467     while (!isToken(AsmToken::EndOfStatement)) {
6468       if (!parseCnt(Waitcnt))
6469         return MatchOperand_ParseFail;
6470     }
6471   } else {
6472     if (!parseExpr(Waitcnt))
6473       return MatchOperand_ParseFail;
6474   }
6475 
6476   Operands.push_back(AMDGPUOperand::CreateImm(this, Waitcnt, S));
6477   return MatchOperand_Success;
6478 }
6479 
6480 bool AMDGPUAsmParser::parseDelay(int64_t &Delay) {
6481   SMLoc FieldLoc = getLoc();
6482   StringRef FieldName = getTokenStr();
6483   if (!skipToken(AsmToken::Identifier, "expected a field name") ||
6484       !skipToken(AsmToken::LParen, "expected a left parenthesis"))
6485     return false;
6486 
6487   SMLoc ValueLoc = getLoc();
6488   StringRef ValueName = getTokenStr();
6489   if (!skipToken(AsmToken::Identifier, "expected a value name") ||
6490       !skipToken(AsmToken::RParen, "expected a right parenthesis"))
6491     return false;
6492 
6493   unsigned Shift;
6494   if (FieldName == "instid0") {
6495     Shift = 0;
6496   } else if (FieldName == "instskip") {
6497     Shift = 4;
6498   } else if (FieldName == "instid1") {
6499     Shift = 7;
6500   } else {
6501     Error(FieldLoc, "invalid field name " + FieldName);
6502     return false;
6503   }
6504 
6505   int Value;
6506   if (Shift == 4) {
6507     // Parse values for instskip.
6508     Value = StringSwitch<int>(ValueName)
6509                 .Case("SAME", 0)
6510                 .Case("NEXT", 1)
6511                 .Case("SKIP_1", 2)
6512                 .Case("SKIP_2", 3)
6513                 .Case("SKIP_3", 4)
6514                 .Case("SKIP_4", 5)
6515                 .Default(-1);
6516   } else {
6517     // Parse values for instid0 and instid1.
6518     Value = StringSwitch<int>(ValueName)
6519                 .Case("NO_DEP", 0)
6520                 .Case("VALU_DEP_1", 1)
6521                 .Case("VALU_DEP_2", 2)
6522                 .Case("VALU_DEP_3", 3)
6523                 .Case("VALU_DEP_4", 4)
6524                 .Case("TRANS32_DEP_1", 5)
6525                 .Case("TRANS32_DEP_2", 6)
6526                 .Case("TRANS32_DEP_3", 7)
6527                 .Case("FMA_ACCUM_CYCLE_1", 8)
6528                 .Case("SALU_CYCLE_1", 9)
6529                 .Case("SALU_CYCLE_2", 10)
6530                 .Case("SALU_CYCLE_3", 11)
6531                 .Default(-1);
6532   }
6533   if (Value < 0) {
6534     Error(ValueLoc, "invalid value name " + ValueName);
6535     return false;
6536   }
6537 
6538   Delay |= Value << Shift;
6539   return true;
6540 }
6541 
6542 OperandMatchResultTy
6543 AMDGPUAsmParser::parseSDelayAluOps(OperandVector &Operands) {
6544   int64_t Delay = 0;
6545   SMLoc S = getLoc();
6546 
6547   if (isToken(AsmToken::Identifier) && peekToken().is(AsmToken::LParen)) {
6548     do {
6549       if (!parseDelay(Delay))
6550         return MatchOperand_ParseFail;
6551     } while (trySkipToken(AsmToken::Pipe));
6552   } else {
6553     if (!parseExpr(Delay))
6554       return MatchOperand_ParseFail;
6555   }
6556 
6557   Operands.push_back(AMDGPUOperand::CreateImm(this, Delay, S));
6558   return MatchOperand_Success;
6559 }
6560 
6561 bool
6562 AMDGPUOperand::isSWaitCnt() const {
6563   return isImm();
6564 }
6565 
6566 bool AMDGPUOperand::isSDelayAlu() const { return isImm(); }
6567 
6568 //===----------------------------------------------------------------------===//
6569 // DepCtr
6570 //===----------------------------------------------------------------------===//
6571 
6572 void AMDGPUAsmParser::depCtrError(SMLoc Loc, int ErrorId,
6573                                   StringRef DepCtrName) {
6574   switch (ErrorId) {
6575   case OPR_ID_UNKNOWN:
6576     Error(Loc, Twine("invalid counter name ", DepCtrName));
6577     return;
6578   case OPR_ID_UNSUPPORTED:
6579     Error(Loc, Twine(DepCtrName, " is not supported on this GPU"));
6580     return;
6581   case OPR_ID_DUPLICATE:
6582     Error(Loc, Twine("duplicate counter name ", DepCtrName));
6583     return;
6584   case OPR_VAL_INVALID:
6585     Error(Loc, Twine("invalid value for ", DepCtrName));
6586     return;
6587   default:
6588     assert(false);
6589   }
6590 }
6591 
6592 bool AMDGPUAsmParser::parseDepCtr(int64_t &DepCtr, unsigned &UsedOprMask) {
6593 
6594   using namespace llvm::AMDGPU::DepCtr;
6595 
6596   SMLoc DepCtrLoc = getLoc();
6597   StringRef DepCtrName = getTokenStr();
6598 
6599   if (!skipToken(AsmToken::Identifier, "expected a counter name") ||
6600       !skipToken(AsmToken::LParen, "expected a left parenthesis"))
6601     return false;
6602 
6603   int64_t ExprVal;
6604   if (!parseExpr(ExprVal))
6605     return false;
6606 
6607   unsigned PrevOprMask = UsedOprMask;
6608   int CntVal = encodeDepCtr(DepCtrName, ExprVal, UsedOprMask, getSTI());
6609 
6610   if (CntVal < 0) {
6611     depCtrError(DepCtrLoc, CntVal, DepCtrName);
6612     return false;
6613   }
6614 
6615   if (!skipToken(AsmToken::RParen, "expected a closing parenthesis"))
6616     return false;
6617 
6618   if (trySkipToken(AsmToken::Amp) || trySkipToken(AsmToken::Comma)) {
6619     if (isToken(AsmToken::EndOfStatement)) {
6620       Error(getLoc(), "expected a counter name");
6621       return false;
6622     }
6623   }
6624 
6625   unsigned CntValMask = PrevOprMask ^ UsedOprMask;
6626   DepCtr = (DepCtr & ~CntValMask) | CntVal;
6627   return true;
6628 }
6629 
6630 OperandMatchResultTy AMDGPUAsmParser::parseDepCtrOps(OperandVector &Operands) {
6631   using namespace llvm::AMDGPU::DepCtr;
6632 
6633   int64_t DepCtr = getDefaultDepCtrEncoding(getSTI());
6634   SMLoc Loc = getLoc();
6635 
6636   if (isToken(AsmToken::Identifier) && peekToken().is(AsmToken::LParen)) {
6637     unsigned UsedOprMask = 0;
6638     while (!isToken(AsmToken::EndOfStatement)) {
6639       if (!parseDepCtr(DepCtr, UsedOprMask))
6640         return MatchOperand_ParseFail;
6641     }
6642   } else {
6643     if (!parseExpr(DepCtr))
6644       return MatchOperand_ParseFail;
6645   }
6646 
6647   Operands.push_back(AMDGPUOperand::CreateImm(this, DepCtr, Loc));
6648   return MatchOperand_Success;
6649 }
6650 
6651 bool AMDGPUOperand::isDepCtr() const { return isS16Imm(); }
6652 
6653 //===----------------------------------------------------------------------===//
6654 // hwreg
6655 //===----------------------------------------------------------------------===//
6656 
6657 bool
6658 AMDGPUAsmParser::parseHwregBody(OperandInfoTy &HwReg,
6659                                 OperandInfoTy &Offset,
6660                                 OperandInfoTy &Width) {
6661   using namespace llvm::AMDGPU::Hwreg;
6662 
6663   // The register may be specified by name or using a numeric code
6664   HwReg.Loc = getLoc();
6665   if (isToken(AsmToken::Identifier) &&
6666       (HwReg.Id = getHwregId(getTokenStr(), getSTI())) != OPR_ID_UNKNOWN) {
6667     HwReg.IsSymbolic = true;
6668     lex(); // skip register name
6669   } else if (!parseExpr(HwReg.Id, "a register name")) {
6670     return false;
6671   }
6672 
6673   if (trySkipToken(AsmToken::RParen))
6674     return true;
6675 
6676   // parse optional params
6677   if (!skipToken(AsmToken::Comma, "expected a comma or a closing parenthesis"))
6678     return false;
6679 
6680   Offset.Loc = getLoc();
6681   if (!parseExpr(Offset.Id))
6682     return false;
6683 
6684   if (!skipToken(AsmToken::Comma, "expected a comma"))
6685     return false;
6686 
6687   Width.Loc = getLoc();
6688   return parseExpr(Width.Id) &&
6689          skipToken(AsmToken::RParen, "expected a closing parenthesis");
6690 }
6691 
6692 bool
6693 AMDGPUAsmParser::validateHwreg(const OperandInfoTy &HwReg,
6694                                const OperandInfoTy &Offset,
6695                                const OperandInfoTy &Width) {
6696 
6697   using namespace llvm::AMDGPU::Hwreg;
6698 
6699   if (HwReg.IsSymbolic) {
6700     if (HwReg.Id == OPR_ID_UNSUPPORTED) {
6701       Error(HwReg.Loc,
6702             "specified hardware register is not supported on this GPU");
6703       return false;
6704     }
6705   } else {
6706     if (!isValidHwreg(HwReg.Id)) {
6707       Error(HwReg.Loc,
6708             "invalid code of hardware register: only 6-bit values are legal");
6709       return false;
6710     }
6711   }
6712   if (!isValidHwregOffset(Offset.Id)) {
6713     Error(Offset.Loc, "invalid bit offset: only 5-bit values are legal");
6714     return false;
6715   }
6716   if (!isValidHwregWidth(Width.Id)) {
6717     Error(Width.Loc,
6718           "invalid bitfield width: only values from 1 to 32 are legal");
6719     return false;
6720   }
6721   return true;
6722 }
6723 
6724 OperandMatchResultTy
6725 AMDGPUAsmParser::parseHwreg(OperandVector &Operands) {
6726   using namespace llvm::AMDGPU::Hwreg;
6727 
6728   int64_t ImmVal = 0;
6729   SMLoc Loc = getLoc();
6730 
6731   if (trySkipId("hwreg", AsmToken::LParen)) {
6732     OperandInfoTy HwReg(OPR_ID_UNKNOWN);
6733     OperandInfoTy Offset(OFFSET_DEFAULT_);
6734     OperandInfoTy Width(WIDTH_DEFAULT_);
6735     if (parseHwregBody(HwReg, Offset, Width) &&
6736         validateHwreg(HwReg, Offset, Width)) {
6737       ImmVal = encodeHwreg(HwReg.Id, Offset.Id, Width.Id);
6738     } else {
6739       return MatchOperand_ParseFail;
6740     }
6741   } else if (parseExpr(ImmVal, "a hwreg macro")) {
6742     if (ImmVal < 0 || !isUInt<16>(ImmVal)) {
6743       Error(Loc, "invalid immediate: only 16-bit values are legal");
6744       return MatchOperand_ParseFail;
6745     }
6746   } else {
6747     return MatchOperand_ParseFail;
6748   }
6749 
6750   Operands.push_back(AMDGPUOperand::CreateImm(this, ImmVal, Loc, AMDGPUOperand::ImmTyHwreg));
6751   return MatchOperand_Success;
6752 }
6753 
6754 bool AMDGPUOperand::isHwreg() const {
6755   return isImmTy(ImmTyHwreg);
6756 }
6757 
6758 //===----------------------------------------------------------------------===//
6759 // sendmsg
6760 //===----------------------------------------------------------------------===//
6761 
6762 bool
6763 AMDGPUAsmParser::parseSendMsgBody(OperandInfoTy &Msg,
6764                                   OperandInfoTy &Op,
6765                                   OperandInfoTy &Stream) {
6766   using namespace llvm::AMDGPU::SendMsg;
6767 
6768   Msg.Loc = getLoc();
6769   if (isToken(AsmToken::Identifier) &&
6770       (Msg.Id = getMsgId(getTokenStr(), getSTI())) != OPR_ID_UNKNOWN) {
6771     Msg.IsSymbolic = true;
6772     lex(); // skip message name
6773   } else if (!parseExpr(Msg.Id, "a message name")) {
6774     return false;
6775   }
6776 
6777   if (trySkipToken(AsmToken::Comma)) {
6778     Op.IsDefined = true;
6779     Op.Loc = getLoc();
6780     if (isToken(AsmToken::Identifier) &&
6781         (Op.Id = getMsgOpId(Msg.Id, getTokenStr())) >= 0) {
6782       lex(); // skip operation name
6783     } else if (!parseExpr(Op.Id, "an operation name")) {
6784       return false;
6785     }
6786 
6787     if (trySkipToken(AsmToken::Comma)) {
6788       Stream.IsDefined = true;
6789       Stream.Loc = getLoc();
6790       if (!parseExpr(Stream.Id))
6791         return false;
6792     }
6793   }
6794 
6795   return skipToken(AsmToken::RParen, "expected a closing parenthesis");
6796 }
6797 
6798 bool
6799 AMDGPUAsmParser::validateSendMsg(const OperandInfoTy &Msg,
6800                                  const OperandInfoTy &Op,
6801                                  const OperandInfoTy &Stream) {
6802   using namespace llvm::AMDGPU::SendMsg;
6803 
6804   // Validation strictness depends on whether message is specified
6805   // in a symbolic or in a numeric form. In the latter case
6806   // only encoding possibility is checked.
6807   bool Strict = Msg.IsSymbolic;
6808 
6809   if (Strict) {
6810     if (Msg.Id == OPR_ID_UNSUPPORTED) {
6811       Error(Msg.Loc, "specified message id is not supported on this GPU");
6812       return false;
6813     }
6814   } else {
6815     if (!isValidMsgId(Msg.Id, getSTI())) {
6816       Error(Msg.Loc, "invalid message id");
6817       return false;
6818     }
6819   }
6820   if (Strict && (msgRequiresOp(Msg.Id, getSTI()) != Op.IsDefined)) {
6821     if (Op.IsDefined) {
6822       Error(Op.Loc, "message does not support operations");
6823     } else {
6824       Error(Msg.Loc, "missing message operation");
6825     }
6826     return false;
6827   }
6828   if (!isValidMsgOp(Msg.Id, Op.Id, getSTI(), Strict)) {
6829     Error(Op.Loc, "invalid operation id");
6830     return false;
6831   }
6832   if (Strict && !msgSupportsStream(Msg.Id, Op.Id, getSTI()) &&
6833       Stream.IsDefined) {
6834     Error(Stream.Loc, "message operation does not support streams");
6835     return false;
6836   }
6837   if (!isValidMsgStream(Msg.Id, Op.Id, Stream.Id, getSTI(), Strict)) {
6838     Error(Stream.Loc, "invalid message stream id");
6839     return false;
6840   }
6841   return true;
6842 }
6843 
6844 OperandMatchResultTy
6845 AMDGPUAsmParser::parseSendMsgOp(OperandVector &Operands) {
6846   using namespace llvm::AMDGPU::SendMsg;
6847 
6848   int64_t ImmVal = 0;
6849   SMLoc Loc = getLoc();
6850 
6851   if (trySkipId("sendmsg", AsmToken::LParen)) {
6852     OperandInfoTy Msg(OPR_ID_UNKNOWN);
6853     OperandInfoTy Op(OP_NONE_);
6854     OperandInfoTy Stream(STREAM_ID_NONE_);
6855     if (parseSendMsgBody(Msg, Op, Stream) &&
6856         validateSendMsg(Msg, Op, Stream)) {
6857       ImmVal = encodeMsg(Msg.Id, Op.Id, Stream.Id);
6858     } else {
6859       return MatchOperand_ParseFail;
6860     }
6861   } else if (parseExpr(ImmVal, "a sendmsg macro")) {
6862     if (ImmVal < 0 || !isUInt<16>(ImmVal)) {
6863       Error(Loc, "invalid immediate: only 16-bit values are legal");
6864       return MatchOperand_ParseFail;
6865     }
6866   } else {
6867     return MatchOperand_ParseFail;
6868   }
6869 
6870   Operands.push_back(AMDGPUOperand::CreateImm(this, ImmVal, Loc, AMDGPUOperand::ImmTySendMsg));
6871   return MatchOperand_Success;
6872 }
6873 
6874 bool AMDGPUOperand::isSendMsg() const {
6875   return isImmTy(ImmTySendMsg);
6876 }
6877 
6878 //===----------------------------------------------------------------------===//
6879 // v_interp
6880 //===----------------------------------------------------------------------===//
6881 
6882 OperandMatchResultTy AMDGPUAsmParser::parseInterpSlot(OperandVector &Operands) {
6883   StringRef Str;
6884   SMLoc S = getLoc();
6885 
6886   if (!parseId(Str))
6887     return MatchOperand_NoMatch;
6888 
6889   int Slot = StringSwitch<int>(Str)
6890     .Case("p10", 0)
6891     .Case("p20", 1)
6892     .Case("p0", 2)
6893     .Default(-1);
6894 
6895   if (Slot == -1) {
6896     Error(S, "invalid interpolation slot");
6897     return MatchOperand_ParseFail;
6898   }
6899 
6900   Operands.push_back(AMDGPUOperand::CreateImm(this, Slot, S,
6901                                               AMDGPUOperand::ImmTyInterpSlot));
6902   return MatchOperand_Success;
6903 }
6904 
6905 OperandMatchResultTy AMDGPUAsmParser::parseInterpAttr(OperandVector &Operands) {
6906   StringRef Str;
6907   SMLoc S = getLoc();
6908 
6909   if (!parseId(Str))
6910     return MatchOperand_NoMatch;
6911 
6912   if (!Str.startswith("attr")) {
6913     Error(S, "invalid interpolation attribute");
6914     return MatchOperand_ParseFail;
6915   }
6916 
6917   StringRef Chan = Str.take_back(2);
6918   int AttrChan = StringSwitch<int>(Chan)
6919     .Case(".x", 0)
6920     .Case(".y", 1)
6921     .Case(".z", 2)
6922     .Case(".w", 3)
6923     .Default(-1);
6924   if (AttrChan == -1) {
6925     Error(S, "invalid or missing interpolation attribute channel");
6926     return MatchOperand_ParseFail;
6927   }
6928 
6929   Str = Str.drop_back(2).drop_front(4);
6930 
6931   uint8_t Attr;
6932   if (Str.getAsInteger(10, Attr)) {
6933     Error(S, "invalid or missing interpolation attribute number");
6934     return MatchOperand_ParseFail;
6935   }
6936 
6937   if (Attr > 63) {
6938     Error(S, "out of bounds interpolation attribute number");
6939     return MatchOperand_ParseFail;
6940   }
6941 
6942   SMLoc SChan = SMLoc::getFromPointer(Chan.data());
6943 
6944   Operands.push_back(AMDGPUOperand::CreateImm(this, Attr, S,
6945                                               AMDGPUOperand::ImmTyInterpAttr));
6946   Operands.push_back(AMDGPUOperand::CreateImm(this, AttrChan, SChan,
6947                                               AMDGPUOperand::ImmTyAttrChan));
6948   return MatchOperand_Success;
6949 }
6950 
6951 //===----------------------------------------------------------------------===//
6952 // exp
6953 //===----------------------------------------------------------------------===//
6954 
6955 OperandMatchResultTy AMDGPUAsmParser::parseExpTgt(OperandVector &Operands) {
6956   using namespace llvm::AMDGPU::Exp;
6957 
6958   StringRef Str;
6959   SMLoc S = getLoc();
6960 
6961   if (!parseId(Str))
6962     return MatchOperand_NoMatch;
6963 
6964   unsigned Id = getTgtId(Str);
6965   if (Id == ET_INVALID || !isSupportedTgtId(Id, getSTI())) {
6966     Error(S, (Id == ET_INVALID) ?
6967                 "invalid exp target" :
6968                 "exp target is not supported on this GPU");
6969     return MatchOperand_ParseFail;
6970   }
6971 
6972   Operands.push_back(AMDGPUOperand::CreateImm(this, Id, S,
6973                                               AMDGPUOperand::ImmTyExpTgt));
6974   return MatchOperand_Success;
6975 }
6976 
6977 //===----------------------------------------------------------------------===//
6978 // parser helpers
6979 //===----------------------------------------------------------------------===//
6980 
6981 bool
6982 AMDGPUAsmParser::isId(const AsmToken &Token, const StringRef Id) const {
6983   return Token.is(AsmToken::Identifier) && Token.getString() == Id;
6984 }
6985 
6986 bool
6987 AMDGPUAsmParser::isId(const StringRef Id) const {
6988   return isId(getToken(), Id);
6989 }
6990 
6991 bool
6992 AMDGPUAsmParser::isToken(const AsmToken::TokenKind Kind) const {
6993   return getTokenKind() == Kind;
6994 }
6995 
6996 bool
6997 AMDGPUAsmParser::trySkipId(const StringRef Id) {
6998   if (isId(Id)) {
6999     lex();
7000     return true;
7001   }
7002   return false;
7003 }
7004 
7005 bool
7006 AMDGPUAsmParser::trySkipId(const StringRef Pref, const StringRef Id) {
7007   if (isToken(AsmToken::Identifier)) {
7008     StringRef Tok = getTokenStr();
7009     if (Tok.startswith(Pref) && Tok.drop_front(Pref.size()) == Id) {
7010       lex();
7011       return true;
7012     }
7013   }
7014   return false;
7015 }
7016 
7017 bool
7018 AMDGPUAsmParser::trySkipId(const StringRef Id, const AsmToken::TokenKind Kind) {
7019   if (isId(Id) && peekToken().is(Kind)) {
7020     lex();
7021     lex();
7022     return true;
7023   }
7024   return false;
7025 }
7026 
7027 bool
7028 AMDGPUAsmParser::trySkipToken(const AsmToken::TokenKind Kind) {
7029   if (isToken(Kind)) {
7030     lex();
7031     return true;
7032   }
7033   return false;
7034 }
7035 
7036 bool
7037 AMDGPUAsmParser::skipToken(const AsmToken::TokenKind Kind,
7038                            const StringRef ErrMsg) {
7039   if (!trySkipToken(Kind)) {
7040     Error(getLoc(), ErrMsg);
7041     return false;
7042   }
7043   return true;
7044 }
7045 
7046 bool
7047 AMDGPUAsmParser::parseExpr(int64_t &Imm, StringRef Expected) {
7048   SMLoc S = getLoc();
7049 
7050   const MCExpr *Expr;
7051   if (Parser.parseExpression(Expr))
7052     return false;
7053 
7054   if (Expr->evaluateAsAbsolute(Imm))
7055     return true;
7056 
7057   if (Expected.empty()) {
7058     Error(S, "expected absolute expression");
7059   } else {
7060     Error(S, Twine("expected ", Expected) +
7061              Twine(" or an absolute expression"));
7062   }
7063   return false;
7064 }
7065 
7066 bool
7067 AMDGPUAsmParser::parseExpr(OperandVector &Operands) {
7068   SMLoc S = getLoc();
7069 
7070   const MCExpr *Expr;
7071   if (Parser.parseExpression(Expr))
7072     return false;
7073 
7074   int64_t IntVal;
7075   if (Expr->evaluateAsAbsolute(IntVal)) {
7076     Operands.push_back(AMDGPUOperand::CreateImm(this, IntVal, S));
7077   } else {
7078     Operands.push_back(AMDGPUOperand::CreateExpr(this, Expr, S));
7079   }
7080   return true;
7081 }
7082 
7083 bool
7084 AMDGPUAsmParser::parseString(StringRef &Val, const StringRef ErrMsg) {
7085   if (isToken(AsmToken::String)) {
7086     Val = getToken().getStringContents();
7087     lex();
7088     return true;
7089   } else {
7090     Error(getLoc(), ErrMsg);
7091     return false;
7092   }
7093 }
7094 
7095 bool
7096 AMDGPUAsmParser::parseId(StringRef &Val, const StringRef ErrMsg) {
7097   if (isToken(AsmToken::Identifier)) {
7098     Val = getTokenStr();
7099     lex();
7100     return true;
7101   } else {
7102     if (!ErrMsg.empty())
7103       Error(getLoc(), ErrMsg);
7104     return false;
7105   }
7106 }
7107 
7108 AsmToken
7109 AMDGPUAsmParser::getToken() const {
7110   return Parser.getTok();
7111 }
7112 
7113 AsmToken
7114 AMDGPUAsmParser::peekToken() {
7115   return isToken(AsmToken::EndOfStatement) ? getToken() : getLexer().peekTok();
7116 }
7117 
7118 void
7119 AMDGPUAsmParser::peekTokens(MutableArrayRef<AsmToken> Tokens) {
7120   auto TokCount = getLexer().peekTokens(Tokens);
7121 
7122   for (auto Idx = TokCount; Idx < Tokens.size(); ++Idx)
7123     Tokens[Idx] = AsmToken(AsmToken::Error, "");
7124 }
7125 
7126 AsmToken::TokenKind
7127 AMDGPUAsmParser::getTokenKind() const {
7128   return getLexer().getKind();
7129 }
7130 
7131 SMLoc
7132 AMDGPUAsmParser::getLoc() const {
7133   return getToken().getLoc();
7134 }
7135 
7136 StringRef
7137 AMDGPUAsmParser::getTokenStr() const {
7138   return getToken().getString();
7139 }
7140 
7141 void
7142 AMDGPUAsmParser::lex() {
7143   Parser.Lex();
7144 }
7145 
7146 SMLoc
7147 AMDGPUAsmParser::getOperandLoc(std::function<bool(const AMDGPUOperand&)> Test,
7148                                const OperandVector &Operands) const {
7149   for (unsigned i = Operands.size() - 1; i > 0; --i) {
7150     AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[i]);
7151     if (Test(Op))
7152       return Op.getStartLoc();
7153   }
7154   return ((AMDGPUOperand &)*Operands[0]).getStartLoc();
7155 }
7156 
7157 SMLoc
7158 AMDGPUAsmParser::getImmLoc(AMDGPUOperand::ImmTy Type,
7159                            const OperandVector &Operands) const {
7160   auto Test = [=](const AMDGPUOperand& Op) { return Op.isImmTy(Type); };
7161   return getOperandLoc(Test, Operands);
7162 }
7163 
7164 SMLoc
7165 AMDGPUAsmParser::getRegLoc(unsigned Reg,
7166                            const OperandVector &Operands) const {
7167   auto Test = [=](const AMDGPUOperand& Op) {
7168     return Op.isRegKind() && Op.getReg() == Reg;
7169   };
7170   return getOperandLoc(Test, Operands);
7171 }
7172 
7173 SMLoc
7174 AMDGPUAsmParser::getLitLoc(const OperandVector &Operands) const {
7175   auto Test = [](const AMDGPUOperand& Op) {
7176     return Op.IsImmKindLiteral() || Op.isExpr();
7177   };
7178   return getOperandLoc(Test, Operands);
7179 }
7180 
7181 SMLoc
7182 AMDGPUAsmParser::getConstLoc(const OperandVector &Operands) const {
7183   auto Test = [](const AMDGPUOperand& Op) {
7184     return Op.isImmKindConst();
7185   };
7186   return getOperandLoc(Test, Operands);
7187 }
7188 
7189 //===----------------------------------------------------------------------===//
7190 // swizzle
7191 //===----------------------------------------------------------------------===//
7192 
7193 LLVM_READNONE
7194 static unsigned
7195 encodeBitmaskPerm(const unsigned AndMask,
7196                   const unsigned OrMask,
7197                   const unsigned XorMask) {
7198   using namespace llvm::AMDGPU::Swizzle;
7199 
7200   return BITMASK_PERM_ENC |
7201          (AndMask << BITMASK_AND_SHIFT) |
7202          (OrMask  << BITMASK_OR_SHIFT)  |
7203          (XorMask << BITMASK_XOR_SHIFT);
7204 }
7205 
7206 bool
7207 AMDGPUAsmParser::parseSwizzleOperand(int64_t &Op,
7208                                      const unsigned MinVal,
7209                                      const unsigned MaxVal,
7210                                      const StringRef ErrMsg,
7211                                      SMLoc &Loc) {
7212   if (!skipToken(AsmToken::Comma, "expected a comma")) {
7213     return false;
7214   }
7215   Loc = getLoc();
7216   if (!parseExpr(Op)) {
7217     return false;
7218   }
7219   if (Op < MinVal || Op > MaxVal) {
7220     Error(Loc, ErrMsg);
7221     return false;
7222   }
7223 
7224   return true;
7225 }
7226 
7227 bool
7228 AMDGPUAsmParser::parseSwizzleOperands(const unsigned OpNum, int64_t* Op,
7229                                       const unsigned MinVal,
7230                                       const unsigned MaxVal,
7231                                       const StringRef ErrMsg) {
7232   SMLoc Loc;
7233   for (unsigned i = 0; i < OpNum; ++i) {
7234     if (!parseSwizzleOperand(Op[i], MinVal, MaxVal, ErrMsg, Loc))
7235       return false;
7236   }
7237 
7238   return true;
7239 }
7240 
7241 bool
7242 AMDGPUAsmParser::parseSwizzleQuadPerm(int64_t &Imm) {
7243   using namespace llvm::AMDGPU::Swizzle;
7244 
7245   int64_t Lane[LANE_NUM];
7246   if (parseSwizzleOperands(LANE_NUM, Lane, 0, LANE_MAX,
7247                            "expected a 2-bit lane id")) {
7248     Imm = QUAD_PERM_ENC;
7249     for (unsigned I = 0; I < LANE_NUM; ++I) {
7250       Imm |= Lane[I] << (LANE_SHIFT * I);
7251     }
7252     return true;
7253   }
7254   return false;
7255 }
7256 
7257 bool
7258 AMDGPUAsmParser::parseSwizzleBroadcast(int64_t &Imm) {
7259   using namespace llvm::AMDGPU::Swizzle;
7260 
7261   SMLoc Loc;
7262   int64_t GroupSize;
7263   int64_t LaneIdx;
7264 
7265   if (!parseSwizzleOperand(GroupSize,
7266                            2, 32,
7267                            "group size must be in the interval [2,32]",
7268                            Loc)) {
7269     return false;
7270   }
7271   if (!isPowerOf2_64(GroupSize)) {
7272     Error(Loc, "group size must be a power of two");
7273     return false;
7274   }
7275   if (parseSwizzleOperand(LaneIdx,
7276                           0, GroupSize - 1,
7277                           "lane id must be in the interval [0,group size - 1]",
7278                           Loc)) {
7279     Imm = encodeBitmaskPerm(BITMASK_MAX - GroupSize + 1, LaneIdx, 0);
7280     return true;
7281   }
7282   return false;
7283 }
7284 
7285 bool
7286 AMDGPUAsmParser::parseSwizzleReverse(int64_t &Imm) {
7287   using namespace llvm::AMDGPU::Swizzle;
7288 
7289   SMLoc Loc;
7290   int64_t GroupSize;
7291 
7292   if (!parseSwizzleOperand(GroupSize,
7293                            2, 32,
7294                            "group size must be in the interval [2,32]",
7295                            Loc)) {
7296     return false;
7297   }
7298   if (!isPowerOf2_64(GroupSize)) {
7299     Error(Loc, "group size must be a power of two");
7300     return false;
7301   }
7302 
7303   Imm = encodeBitmaskPerm(BITMASK_MAX, 0, GroupSize - 1);
7304   return true;
7305 }
7306 
7307 bool
7308 AMDGPUAsmParser::parseSwizzleSwap(int64_t &Imm) {
7309   using namespace llvm::AMDGPU::Swizzle;
7310 
7311   SMLoc Loc;
7312   int64_t GroupSize;
7313 
7314   if (!parseSwizzleOperand(GroupSize,
7315                            1, 16,
7316                            "group size must be in the interval [1,16]",
7317                            Loc)) {
7318     return false;
7319   }
7320   if (!isPowerOf2_64(GroupSize)) {
7321     Error(Loc, "group size must be a power of two");
7322     return false;
7323   }
7324 
7325   Imm = encodeBitmaskPerm(BITMASK_MAX, 0, GroupSize);
7326   return true;
7327 }
7328 
7329 bool
7330 AMDGPUAsmParser::parseSwizzleBitmaskPerm(int64_t &Imm) {
7331   using namespace llvm::AMDGPU::Swizzle;
7332 
7333   if (!skipToken(AsmToken::Comma, "expected a comma")) {
7334     return false;
7335   }
7336 
7337   StringRef Ctl;
7338   SMLoc StrLoc = getLoc();
7339   if (!parseString(Ctl)) {
7340     return false;
7341   }
7342   if (Ctl.size() != BITMASK_WIDTH) {
7343     Error(StrLoc, "expected a 5-character mask");
7344     return false;
7345   }
7346 
7347   unsigned AndMask = 0;
7348   unsigned OrMask = 0;
7349   unsigned XorMask = 0;
7350 
7351   for (size_t i = 0; i < Ctl.size(); ++i) {
7352     unsigned Mask = 1 << (BITMASK_WIDTH - 1 - i);
7353     switch(Ctl[i]) {
7354     default:
7355       Error(StrLoc, "invalid mask");
7356       return false;
7357     case '0':
7358       break;
7359     case '1':
7360       OrMask |= Mask;
7361       break;
7362     case 'p':
7363       AndMask |= Mask;
7364       break;
7365     case 'i':
7366       AndMask |= Mask;
7367       XorMask |= Mask;
7368       break;
7369     }
7370   }
7371 
7372   Imm = encodeBitmaskPerm(AndMask, OrMask, XorMask);
7373   return true;
7374 }
7375 
7376 bool
7377 AMDGPUAsmParser::parseSwizzleOffset(int64_t &Imm) {
7378 
7379   SMLoc OffsetLoc = getLoc();
7380 
7381   if (!parseExpr(Imm, "a swizzle macro")) {
7382     return false;
7383   }
7384   if (!isUInt<16>(Imm)) {
7385     Error(OffsetLoc, "expected a 16-bit offset");
7386     return false;
7387   }
7388   return true;
7389 }
7390 
7391 bool
7392 AMDGPUAsmParser::parseSwizzleMacro(int64_t &Imm) {
7393   using namespace llvm::AMDGPU::Swizzle;
7394 
7395   if (skipToken(AsmToken::LParen, "expected a left parentheses")) {
7396 
7397     SMLoc ModeLoc = getLoc();
7398     bool Ok = false;
7399 
7400     if (trySkipId(IdSymbolic[ID_QUAD_PERM])) {
7401       Ok = parseSwizzleQuadPerm(Imm);
7402     } else if (trySkipId(IdSymbolic[ID_BITMASK_PERM])) {
7403       Ok = parseSwizzleBitmaskPerm(Imm);
7404     } else if (trySkipId(IdSymbolic[ID_BROADCAST])) {
7405       Ok = parseSwizzleBroadcast(Imm);
7406     } else if (trySkipId(IdSymbolic[ID_SWAP])) {
7407       Ok = parseSwizzleSwap(Imm);
7408     } else if (trySkipId(IdSymbolic[ID_REVERSE])) {
7409       Ok = parseSwizzleReverse(Imm);
7410     } else {
7411       Error(ModeLoc, "expected a swizzle mode");
7412     }
7413 
7414     return Ok && skipToken(AsmToken::RParen, "expected a closing parentheses");
7415   }
7416 
7417   return false;
7418 }
7419 
7420 OperandMatchResultTy
7421 AMDGPUAsmParser::parseSwizzleOp(OperandVector &Operands) {
7422   SMLoc S = getLoc();
7423   int64_t Imm = 0;
7424 
7425   if (trySkipId("offset")) {
7426 
7427     bool Ok = false;
7428     if (skipToken(AsmToken::Colon, "expected a colon")) {
7429       if (trySkipId("swizzle")) {
7430         Ok = parseSwizzleMacro(Imm);
7431       } else {
7432         Ok = parseSwizzleOffset(Imm);
7433       }
7434     }
7435 
7436     Operands.push_back(AMDGPUOperand::CreateImm(this, Imm, S, AMDGPUOperand::ImmTySwizzle));
7437 
7438     return Ok? MatchOperand_Success : MatchOperand_ParseFail;
7439   } else {
7440     // Swizzle "offset" operand is optional.
7441     // If it is omitted, try parsing other optional operands.
7442     return parseOptionalOpr(Operands);
7443   }
7444 }
7445 
7446 bool
7447 AMDGPUOperand::isSwizzle() const {
7448   return isImmTy(ImmTySwizzle);
7449 }
7450 
7451 //===----------------------------------------------------------------------===//
7452 // VGPR Index Mode
7453 //===----------------------------------------------------------------------===//
7454 
7455 int64_t AMDGPUAsmParser::parseGPRIdxMacro() {
7456 
7457   using namespace llvm::AMDGPU::VGPRIndexMode;
7458 
7459   if (trySkipToken(AsmToken::RParen)) {
7460     return OFF;
7461   }
7462 
7463   int64_t Imm = 0;
7464 
7465   while (true) {
7466     unsigned Mode = 0;
7467     SMLoc S = getLoc();
7468 
7469     for (unsigned ModeId = ID_MIN; ModeId <= ID_MAX; ++ModeId) {
7470       if (trySkipId(IdSymbolic[ModeId])) {
7471         Mode = 1 << ModeId;
7472         break;
7473       }
7474     }
7475 
7476     if (Mode == 0) {
7477       Error(S, (Imm == 0)?
7478                "expected a VGPR index mode or a closing parenthesis" :
7479                "expected a VGPR index mode");
7480       return UNDEF;
7481     }
7482 
7483     if (Imm & Mode) {
7484       Error(S, "duplicate VGPR index mode");
7485       return UNDEF;
7486     }
7487     Imm |= Mode;
7488 
7489     if (trySkipToken(AsmToken::RParen))
7490       break;
7491     if (!skipToken(AsmToken::Comma,
7492                    "expected a comma or a closing parenthesis"))
7493       return UNDEF;
7494   }
7495 
7496   return Imm;
7497 }
7498 
7499 OperandMatchResultTy
7500 AMDGPUAsmParser::parseGPRIdxMode(OperandVector &Operands) {
7501 
7502   using namespace llvm::AMDGPU::VGPRIndexMode;
7503 
7504   int64_t Imm = 0;
7505   SMLoc S = getLoc();
7506 
7507   if (trySkipId("gpr_idx", AsmToken::LParen)) {
7508     Imm = parseGPRIdxMacro();
7509     if (Imm == UNDEF)
7510       return MatchOperand_ParseFail;
7511   } else {
7512     if (getParser().parseAbsoluteExpression(Imm))
7513       return MatchOperand_ParseFail;
7514     if (Imm < 0 || !isUInt<4>(Imm)) {
7515       Error(S, "invalid immediate: only 4-bit values are legal");
7516       return MatchOperand_ParseFail;
7517     }
7518   }
7519 
7520   Operands.push_back(
7521       AMDGPUOperand::CreateImm(this, Imm, S, AMDGPUOperand::ImmTyGprIdxMode));
7522   return MatchOperand_Success;
7523 }
7524 
7525 bool AMDGPUOperand::isGPRIdxMode() const {
7526   return isImmTy(ImmTyGprIdxMode);
7527 }
7528 
7529 //===----------------------------------------------------------------------===//
7530 // sopp branch targets
7531 //===----------------------------------------------------------------------===//
7532 
7533 OperandMatchResultTy
7534 AMDGPUAsmParser::parseSOppBrTarget(OperandVector &Operands) {
7535 
7536   // Make sure we are not parsing something
7537   // that looks like a label or an expression but is not.
7538   // This will improve error messages.
7539   if (isRegister() || isModifier())
7540     return MatchOperand_NoMatch;
7541 
7542   if (!parseExpr(Operands))
7543     return MatchOperand_ParseFail;
7544 
7545   AMDGPUOperand &Opr = ((AMDGPUOperand &)*Operands[Operands.size() - 1]);
7546   assert(Opr.isImm() || Opr.isExpr());
7547   SMLoc Loc = Opr.getStartLoc();
7548 
7549   // Currently we do not support arbitrary expressions as branch targets.
7550   // Only labels and absolute expressions are accepted.
7551   if (Opr.isExpr() && !Opr.isSymbolRefExpr()) {
7552     Error(Loc, "expected an absolute expression or a label");
7553   } else if (Opr.isImm() && !Opr.isS16Imm()) {
7554     Error(Loc, "expected a 16-bit signed jump offset");
7555   }
7556 
7557   return MatchOperand_Success;
7558 }
7559 
7560 //===----------------------------------------------------------------------===//
7561 // Boolean holding registers
7562 //===----------------------------------------------------------------------===//
7563 
7564 OperandMatchResultTy
7565 AMDGPUAsmParser::parseBoolReg(OperandVector &Operands) {
7566   return parseReg(Operands);
7567 }
7568 
7569 //===----------------------------------------------------------------------===//
7570 // mubuf
7571 //===----------------------------------------------------------------------===//
7572 
7573 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultCPol() const {
7574   return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTyCPol);
7575 }
7576 
7577 void AMDGPUAsmParser::cvtMubufImpl(MCInst &Inst,
7578                                    const OperandVector &Operands,
7579                                    bool IsAtomic,
7580                                    bool IsLds) {
7581   OptionalImmIndexMap OptionalIdx;
7582   unsigned FirstOperandIdx = 1;
7583   bool IsAtomicReturn = false;
7584 
7585   if (IsAtomic) {
7586     for (unsigned i = FirstOperandIdx, e = Operands.size(); i != e; ++i) {
7587       AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[i]);
7588       if (!Op.isCPol())
7589         continue;
7590       IsAtomicReturn = Op.getImm() & AMDGPU::CPol::GLC;
7591       break;
7592     }
7593 
7594     if (!IsAtomicReturn) {
7595       int NewOpc = AMDGPU::getAtomicNoRetOp(Inst.getOpcode());
7596       if (NewOpc != -1)
7597         Inst.setOpcode(NewOpc);
7598     }
7599 
7600     IsAtomicReturn =  MII.get(Inst.getOpcode()).TSFlags &
7601                       SIInstrFlags::IsAtomicRet;
7602   }
7603 
7604   for (unsigned i = FirstOperandIdx, e = Operands.size(); i != e; ++i) {
7605     AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[i]);
7606 
7607     // Add the register arguments
7608     if (Op.isReg()) {
7609       Op.addRegOperands(Inst, 1);
7610       // Insert a tied src for atomic return dst.
7611       // This cannot be postponed as subsequent calls to
7612       // addImmOperands rely on correct number of MC operands.
7613       if (IsAtomicReturn && i == FirstOperandIdx)
7614         Op.addRegOperands(Inst, 1);
7615       continue;
7616     }
7617 
7618     // Handle the case where soffset is an immediate
7619     if (Op.isImm() && Op.getImmTy() == AMDGPUOperand::ImmTyNone) {
7620       Op.addImmOperands(Inst, 1);
7621       continue;
7622     }
7623 
7624     // Handle tokens like 'offen' which are sometimes hard-coded into the
7625     // asm string.  There are no MCInst operands for these.
7626     if (Op.isToken()) {
7627       continue;
7628     }
7629     assert(Op.isImm());
7630 
7631     // Handle optional arguments
7632     OptionalIdx[Op.getImmTy()] = i;
7633   }
7634 
7635   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyOffset);
7636   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyCPol, 0);
7637 
7638   if (!IsLds) { // tfe is not legal with lds opcodes
7639     addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyTFE);
7640   }
7641   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySWZ);
7642 }
7643 
7644 void AMDGPUAsmParser::cvtMtbuf(MCInst &Inst, const OperandVector &Operands) {
7645   OptionalImmIndexMap OptionalIdx;
7646 
7647   for (unsigned i = 1, e = Operands.size(); i != e; ++i) {
7648     AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[i]);
7649 
7650     // Add the register arguments
7651     if (Op.isReg()) {
7652       Op.addRegOperands(Inst, 1);
7653       continue;
7654     }
7655 
7656     // Handle the case where soffset is an immediate
7657     if (Op.isImm() && Op.getImmTy() == AMDGPUOperand::ImmTyNone) {
7658       Op.addImmOperands(Inst, 1);
7659       continue;
7660     }
7661 
7662     // Handle tokens like 'offen' which are sometimes hard-coded into the
7663     // asm string.  There are no MCInst operands for these.
7664     if (Op.isToken()) {
7665       continue;
7666     }
7667     assert(Op.isImm());
7668 
7669     // Handle optional arguments
7670     OptionalIdx[Op.getImmTy()] = i;
7671   }
7672 
7673   addOptionalImmOperand(Inst, Operands, OptionalIdx,
7674                         AMDGPUOperand::ImmTyOffset);
7675   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyFORMAT);
7676   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyCPol, 0);
7677   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyTFE);
7678   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySWZ);
7679 }
7680 
7681 //===----------------------------------------------------------------------===//
7682 // mimg
7683 //===----------------------------------------------------------------------===//
7684 
7685 void AMDGPUAsmParser::cvtMIMG(MCInst &Inst, const OperandVector &Operands,
7686                               bool IsAtomic) {
7687   unsigned I = 1;
7688   const MCInstrDesc &Desc = MII.get(Inst.getOpcode());
7689   for (unsigned J = 0; J < Desc.getNumDefs(); ++J) {
7690     ((AMDGPUOperand &)*Operands[I++]).addRegOperands(Inst, 1);
7691   }
7692 
7693   if (IsAtomic) {
7694     // Add src, same as dst
7695     assert(Desc.getNumDefs() == 1);
7696     ((AMDGPUOperand &)*Operands[I - 1]).addRegOperands(Inst, 1);
7697   }
7698 
7699   OptionalImmIndexMap OptionalIdx;
7700 
7701   for (unsigned E = Operands.size(); I != E; ++I) {
7702     AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[I]);
7703 
7704     // Add the register arguments
7705     if (Op.isReg()) {
7706       Op.addRegOperands(Inst, 1);
7707     } else if (Op.isImmModifier()) {
7708       OptionalIdx[Op.getImmTy()] = I;
7709     } else if (!Op.isToken()) {
7710       llvm_unreachable("unexpected operand type");
7711     }
7712   }
7713 
7714   bool IsGFX10Plus = isGFX10Plus();
7715 
7716   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyDMask);
7717   if (IsGFX10Plus)
7718     addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyDim, -1);
7719   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyUNorm);
7720   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyCPol);
7721   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyR128A16);
7722   if (IsGFX10Plus)
7723     addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyA16);
7724   if (AMDGPU::getNamedOperandIdx(Inst.getOpcode(), AMDGPU::OpName::tfe) != -1)
7725     addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyTFE);
7726   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyLWE);
7727   if (!IsGFX10Plus)
7728     addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyDA);
7729   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyD16);
7730 }
7731 
7732 void AMDGPUAsmParser::cvtMIMGAtomic(MCInst &Inst, const OperandVector &Operands) {
7733   cvtMIMG(Inst, Operands, true);
7734 }
7735 
7736 void AMDGPUAsmParser::cvtSMEMAtomic(MCInst &Inst, const OperandVector &Operands) {
7737   OptionalImmIndexMap OptionalIdx;
7738   bool IsAtomicReturn = false;
7739 
7740   for (unsigned i = 1, e = Operands.size(); i != e; ++i) {
7741     AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[i]);
7742     if (!Op.isCPol())
7743       continue;
7744     IsAtomicReturn = Op.getImm() & AMDGPU::CPol::GLC;
7745     break;
7746   }
7747 
7748   if (!IsAtomicReturn) {
7749     int NewOpc = AMDGPU::getAtomicNoRetOp(Inst.getOpcode());
7750     if (NewOpc != -1)
7751       Inst.setOpcode(NewOpc);
7752   }
7753 
7754   IsAtomicReturn =  MII.get(Inst.getOpcode()).TSFlags &
7755                     SIInstrFlags::IsAtomicRet;
7756 
7757   for (unsigned i = 1, e = Operands.size(); i != e; ++i) {
7758     AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[i]);
7759 
7760     // Add the register arguments
7761     if (Op.isReg()) {
7762       Op.addRegOperands(Inst, 1);
7763       if (IsAtomicReturn && i == 1)
7764         Op.addRegOperands(Inst, 1);
7765       continue;
7766     }
7767 
7768     // Handle the case where soffset is an immediate
7769     if (Op.isImm() && Op.getImmTy() == AMDGPUOperand::ImmTyNone) {
7770       Op.addImmOperands(Inst, 1);
7771       continue;
7772     }
7773 
7774     // Handle tokens like 'offen' which are sometimes hard-coded into the
7775     // asm string.  There are no MCInst operands for these.
7776     if (Op.isToken()) {
7777       continue;
7778     }
7779     assert(Op.isImm());
7780 
7781     // Handle optional arguments
7782     OptionalIdx[Op.getImmTy()] = i;
7783   }
7784 
7785   if ((int)Inst.getNumOperands() <=
7786       AMDGPU::getNamedOperandIdx(Inst.getOpcode(), AMDGPU::OpName::offset))
7787     addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyOffset);
7788   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyCPol, 0);
7789 }
7790 
7791 void AMDGPUAsmParser::cvtIntersectRay(MCInst &Inst,
7792                                       const OperandVector &Operands) {
7793   for (unsigned I = 1; I < Operands.size(); ++I) {
7794     auto &Operand = (AMDGPUOperand &)*Operands[I];
7795     if (Operand.isReg())
7796       Operand.addRegOperands(Inst, 1);
7797   }
7798 
7799   Inst.addOperand(MCOperand::createImm(1)); // a16
7800 }
7801 
7802 //===----------------------------------------------------------------------===//
7803 // smrd
7804 //===----------------------------------------------------------------------===//
7805 
7806 bool AMDGPUOperand::isSMRDOffset8() const {
7807   return isImm() && isUInt<8>(getImm());
7808 }
7809 
7810 bool AMDGPUOperand::isSMEMOffset() const {
7811   return isImmTy(ImmTyNone) ||
7812          isImmTy(ImmTyOffset); // Offset range is checked later by validator.
7813 }
7814 
7815 bool AMDGPUOperand::isSMRDLiteralOffset() const {
7816   // 32-bit literals are only supported on CI and we only want to use them
7817   // when the offset is > 8-bits.
7818   return isImm() && !isUInt<8>(getImm()) && isUInt<32>(getImm());
7819 }
7820 
7821 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultSMRDOffset8() const {
7822   return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTyOffset);
7823 }
7824 
7825 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultSMEMOffset() const {
7826   return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTyOffset);
7827 }
7828 
7829 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultSMRDLiteralOffset() const {
7830   return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTyOffset);
7831 }
7832 
7833 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultFlatOffset() const {
7834   return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTyOffset);
7835 }
7836 
7837 //===----------------------------------------------------------------------===//
7838 // vop3
7839 //===----------------------------------------------------------------------===//
7840 
7841 static bool ConvertOmodMul(int64_t &Mul) {
7842   if (Mul != 1 && Mul != 2 && Mul != 4)
7843     return false;
7844 
7845   Mul >>= 1;
7846   return true;
7847 }
7848 
7849 static bool ConvertOmodDiv(int64_t &Div) {
7850   if (Div == 1) {
7851     Div = 0;
7852     return true;
7853   }
7854 
7855   if (Div == 2) {
7856     Div = 3;
7857     return true;
7858   }
7859 
7860   return false;
7861 }
7862 
7863 // Both bound_ctrl:0 and bound_ctrl:1 are encoded as 1.
7864 // This is intentional and ensures compatibility with sp3.
7865 // See bug 35397 for details.
7866 static bool ConvertBoundCtrl(int64_t &BoundCtrl) {
7867   if (BoundCtrl == 0 || BoundCtrl == 1) {
7868     BoundCtrl = 1;
7869     return true;
7870   }
7871   return false;
7872 }
7873 
7874 // Note: the order in this table matches the order of operands in AsmString.
7875 static const OptionalOperand AMDGPUOptionalOperandTable[] = {
7876   {"offen",   AMDGPUOperand::ImmTyOffen, true, nullptr},
7877   {"idxen",   AMDGPUOperand::ImmTyIdxen, true, nullptr},
7878   {"addr64",  AMDGPUOperand::ImmTyAddr64, true, nullptr},
7879   {"offset0", AMDGPUOperand::ImmTyOffset0, false, nullptr},
7880   {"offset1", AMDGPUOperand::ImmTyOffset1, false, nullptr},
7881   {"gds",     AMDGPUOperand::ImmTyGDS, true, nullptr},
7882   {"lds",     AMDGPUOperand::ImmTyLDS, true, nullptr},
7883   {"offset",  AMDGPUOperand::ImmTyOffset, false, nullptr},
7884   {"inst_offset", AMDGPUOperand::ImmTyInstOffset, false, nullptr},
7885   {"",        AMDGPUOperand::ImmTyCPol, false, nullptr},
7886   {"swz",     AMDGPUOperand::ImmTySWZ, true, nullptr},
7887   {"tfe",     AMDGPUOperand::ImmTyTFE, true, nullptr},
7888   {"d16",     AMDGPUOperand::ImmTyD16, true, nullptr},
7889   {"high",    AMDGPUOperand::ImmTyHigh, true, nullptr},
7890   {"clamp",   AMDGPUOperand::ImmTyClampSI, true, nullptr},
7891   {"omod",    AMDGPUOperand::ImmTyOModSI, false, ConvertOmodMul},
7892   {"unorm",   AMDGPUOperand::ImmTyUNorm, true, nullptr},
7893   {"da",      AMDGPUOperand::ImmTyDA,    true, nullptr},
7894   {"r128",    AMDGPUOperand::ImmTyR128A16,  true, nullptr},
7895   {"a16",     AMDGPUOperand::ImmTyA16,  true, nullptr},
7896   {"lwe",     AMDGPUOperand::ImmTyLWE,   true, nullptr},
7897   {"d16",     AMDGPUOperand::ImmTyD16,   true, nullptr},
7898   {"dmask",   AMDGPUOperand::ImmTyDMask, false, nullptr},
7899   {"dim",     AMDGPUOperand::ImmTyDim,   false, nullptr},
7900   {"dst_sel",    AMDGPUOperand::ImmTySdwaDstSel, false, nullptr},
7901   {"src0_sel",   AMDGPUOperand::ImmTySdwaSrc0Sel, false, nullptr},
7902   {"src1_sel",   AMDGPUOperand::ImmTySdwaSrc1Sel, false, nullptr},
7903   {"dst_unused", AMDGPUOperand::ImmTySdwaDstUnused, false, nullptr},
7904   {"compr", AMDGPUOperand::ImmTyExpCompr, true, nullptr },
7905   {"vm", AMDGPUOperand::ImmTyExpVM, true, nullptr},
7906   {"op_sel", AMDGPUOperand::ImmTyOpSel, false, nullptr},
7907   {"op_sel_hi", AMDGPUOperand::ImmTyOpSelHi, false, nullptr},
7908   {"neg_lo", AMDGPUOperand::ImmTyNegLo, false, nullptr},
7909   {"neg_hi", AMDGPUOperand::ImmTyNegHi, false, nullptr},
7910   {"dpp8",     AMDGPUOperand::ImmTyDPP8, false, nullptr},
7911   {"dpp_ctrl", AMDGPUOperand::ImmTyDppCtrl, false, nullptr},
7912   {"row_mask",   AMDGPUOperand::ImmTyDppRowMask, false, nullptr},
7913   {"bank_mask",  AMDGPUOperand::ImmTyDppBankMask, false, nullptr},
7914   {"bound_ctrl", AMDGPUOperand::ImmTyDppBoundCtrl, false, ConvertBoundCtrl},
7915   {"fi",   AMDGPUOperand::ImmTyDppFi, false, nullptr},
7916   {"blgp", AMDGPUOperand::ImmTyBLGP, false, nullptr},
7917   {"cbsz", AMDGPUOperand::ImmTyCBSZ, false, nullptr},
7918   {"abid", AMDGPUOperand::ImmTyABID, false, nullptr},
7919   {"wait_vdst", AMDGPUOperand::ImmTyWaitVDST, false, nullptr},
7920   {"wait_exp", AMDGPUOperand::ImmTyWaitEXP, false, nullptr}
7921 };
7922 
7923 void AMDGPUAsmParser::onBeginOfFile() {
7924   if (!getParser().getStreamer().getTargetStreamer() ||
7925       getSTI().getTargetTriple().getArch() == Triple::r600)
7926     return;
7927 
7928   if (!getTargetStreamer().getTargetID())
7929     getTargetStreamer().initializeTargetID(getSTI(), getSTI().getFeatureString());
7930 
7931   if (isHsaAbiVersion3AndAbove(&getSTI()))
7932     getTargetStreamer().EmitDirectiveAMDGCNTarget();
7933 }
7934 
7935 OperandMatchResultTy AMDGPUAsmParser::parseOptionalOperand(OperandVector &Operands) {
7936 
7937   OperandMatchResultTy res = parseOptionalOpr(Operands);
7938 
7939   // This is a hack to enable hardcoded mandatory operands which follow
7940   // optional operands.
7941   //
7942   // Current design assumes that all operands after the first optional operand
7943   // are also optional. However implementation of some instructions violates
7944   // this rule (see e.g. flat/global atomic which have hardcoded 'glc' operands).
7945   //
7946   // To alleviate this problem, we have to (implicitly) parse extra operands
7947   // to make sure autogenerated parser of custom operands never hit hardcoded
7948   // mandatory operands.
7949 
7950   for (unsigned i = 0; i < MAX_OPR_LOOKAHEAD; ++i) {
7951     if (res != MatchOperand_Success ||
7952         isToken(AsmToken::EndOfStatement))
7953       break;
7954 
7955     trySkipToken(AsmToken::Comma);
7956     res = parseOptionalOpr(Operands);
7957   }
7958 
7959   return res;
7960 }
7961 
7962 OperandMatchResultTy AMDGPUAsmParser::parseOptionalOpr(OperandVector &Operands) {
7963   OperandMatchResultTy res;
7964   for (const OptionalOperand &Op : AMDGPUOptionalOperandTable) {
7965     // try to parse any optional operand here
7966     if (Op.IsBit) {
7967       res = parseNamedBit(Op.Name, Operands, Op.Type);
7968     } else if (Op.Type == AMDGPUOperand::ImmTyOModSI) {
7969       res = parseOModOperand(Operands);
7970     } else if (Op.Type == AMDGPUOperand::ImmTySdwaDstSel ||
7971                Op.Type == AMDGPUOperand::ImmTySdwaSrc0Sel ||
7972                Op.Type == AMDGPUOperand::ImmTySdwaSrc1Sel) {
7973       res = parseSDWASel(Operands, Op.Name, Op.Type);
7974     } else if (Op.Type == AMDGPUOperand::ImmTySdwaDstUnused) {
7975       res = parseSDWADstUnused(Operands);
7976     } else if (Op.Type == AMDGPUOperand::ImmTyOpSel ||
7977                Op.Type == AMDGPUOperand::ImmTyOpSelHi ||
7978                Op.Type == AMDGPUOperand::ImmTyNegLo ||
7979                Op.Type == AMDGPUOperand::ImmTyNegHi) {
7980       res = parseOperandArrayWithPrefix(Op.Name, Operands, Op.Type,
7981                                         Op.ConvertResult);
7982     } else if (Op.Type == AMDGPUOperand::ImmTyDim) {
7983       res = parseDim(Operands);
7984     } else if (Op.Type == AMDGPUOperand::ImmTyCPol) {
7985       res = parseCPol(Operands);
7986     } else if (Op.Type == AMDGPUOperand::ImmTyDPP8) {
7987       res = parseDPP8(Operands);
7988     } else if (Op.Type == AMDGPUOperand::ImmTyDppCtrl) {
7989       res = parseDPPCtrl(Operands);
7990     } else {
7991       res = parseIntWithPrefix(Op.Name, Operands, Op.Type, Op.ConvertResult);
7992       if (Op.Type == AMDGPUOperand::ImmTyBLGP && res == MatchOperand_NoMatch) {
7993         res = parseOperandArrayWithPrefix("neg", Operands,
7994                                           AMDGPUOperand::ImmTyBLGP,
7995                                           nullptr);
7996       }
7997     }
7998     if (res != MatchOperand_NoMatch) {
7999       return res;
8000     }
8001   }
8002   return MatchOperand_NoMatch;
8003 }
8004 
8005 OperandMatchResultTy AMDGPUAsmParser::parseOModOperand(OperandVector &Operands) {
8006   StringRef Name = getTokenStr();
8007   if (Name == "mul") {
8008     return parseIntWithPrefix("mul", Operands,
8009                               AMDGPUOperand::ImmTyOModSI, ConvertOmodMul);
8010   }
8011 
8012   if (Name == "div") {
8013     return parseIntWithPrefix("div", Operands,
8014                               AMDGPUOperand::ImmTyOModSI, ConvertOmodDiv);
8015   }
8016 
8017   return MatchOperand_NoMatch;
8018 }
8019 
8020 void AMDGPUAsmParser::cvtVOP3OpSel(MCInst &Inst, const OperandVector &Operands) {
8021   cvtVOP3P(Inst, Operands);
8022 
8023   int Opc = Inst.getOpcode();
8024 
8025   int SrcNum;
8026   const int Ops[] = { AMDGPU::OpName::src0,
8027                       AMDGPU::OpName::src1,
8028                       AMDGPU::OpName::src2 };
8029   for (SrcNum = 0;
8030        SrcNum < 3 && AMDGPU::getNamedOperandIdx(Opc, Ops[SrcNum]) != -1;
8031        ++SrcNum);
8032   assert(SrcNum > 0);
8033 
8034   int OpSelIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::op_sel);
8035   unsigned OpSel = Inst.getOperand(OpSelIdx).getImm();
8036 
8037   if ((OpSel & (1 << SrcNum)) != 0) {
8038     int ModIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src0_modifiers);
8039     uint32_t ModVal = Inst.getOperand(ModIdx).getImm();
8040     Inst.getOperand(ModIdx).setImm(ModVal | SISrcMods::DST_OP_SEL);
8041   }
8042 }
8043 
8044 static bool isRegOrImmWithInputMods(const MCInstrDesc &Desc, unsigned OpNum) {
8045       // 1. This operand is input modifiers
8046   return Desc.OpInfo[OpNum].OperandType == AMDGPU::OPERAND_INPUT_MODS
8047       // 2. This is not last operand
8048       && Desc.NumOperands > (OpNum + 1)
8049       // 3. Next operand is register class
8050       && Desc.OpInfo[OpNum + 1].RegClass != -1
8051       // 4. Next register is not tied to any other operand
8052       && Desc.getOperandConstraint(OpNum + 1, MCOI::OperandConstraint::TIED_TO) == -1;
8053 }
8054 
8055 void AMDGPUAsmParser::cvtVOP3Interp(MCInst &Inst, const OperandVector &Operands)
8056 {
8057   OptionalImmIndexMap OptionalIdx;
8058   unsigned Opc = Inst.getOpcode();
8059 
8060   unsigned I = 1;
8061   const MCInstrDesc &Desc = MII.get(Inst.getOpcode());
8062   for (unsigned J = 0; J < Desc.getNumDefs(); ++J) {
8063     ((AMDGPUOperand &)*Operands[I++]).addRegOperands(Inst, 1);
8064   }
8065 
8066   for (unsigned E = Operands.size(); I != E; ++I) {
8067     AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[I]);
8068     if (isRegOrImmWithInputMods(Desc, Inst.getNumOperands())) {
8069       Op.addRegOrImmWithFPInputModsOperands(Inst, 2);
8070     } else if (Op.isInterpSlot() ||
8071                Op.isInterpAttr() ||
8072                Op.isAttrChan()) {
8073       Inst.addOperand(MCOperand::createImm(Op.getImm()));
8074     } else if (Op.isImmModifier()) {
8075       OptionalIdx[Op.getImmTy()] = I;
8076     } else {
8077       llvm_unreachable("unhandled operand type");
8078     }
8079   }
8080 
8081   if (AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::high) != -1) {
8082     addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyHigh);
8083   }
8084 
8085   if (AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::clamp) != -1) {
8086     addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyClampSI);
8087   }
8088 
8089   if (AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::omod) != -1) {
8090     addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyOModSI);
8091   }
8092 }
8093 
8094 void AMDGPUAsmParser::cvtVINTERP(MCInst &Inst, const OperandVector &Operands)
8095 {
8096   OptionalImmIndexMap OptionalIdx;
8097   unsigned Opc = Inst.getOpcode();
8098 
8099   unsigned I = 1;
8100   const MCInstrDesc &Desc = MII.get(Inst.getOpcode());
8101   for (unsigned J = 0; J < Desc.getNumDefs(); ++J) {
8102     ((AMDGPUOperand &)*Operands[I++]).addRegOperands(Inst, 1);
8103   }
8104 
8105   for (unsigned E = Operands.size(); I != E; ++I) {
8106     AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[I]);
8107     if (isRegOrImmWithInputMods(Desc, Inst.getNumOperands())) {
8108       Op.addRegOrImmWithFPInputModsOperands(Inst, 2);
8109     } else if (Op.isImmModifier()) {
8110       OptionalIdx[Op.getImmTy()] = I;
8111     } else {
8112       llvm_unreachable("unhandled operand type");
8113     }
8114   }
8115 
8116   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyClampSI);
8117 
8118   int OpSelIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::op_sel);
8119   if (OpSelIdx != -1)
8120     addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyOpSel);
8121 
8122   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyWaitEXP);
8123 
8124   if (OpSelIdx == -1)
8125     return;
8126 
8127   const int Ops[] = { AMDGPU::OpName::src0,
8128                       AMDGPU::OpName::src1,
8129                       AMDGPU::OpName::src2 };
8130   const int ModOps[] = { AMDGPU::OpName::src0_modifiers,
8131                          AMDGPU::OpName::src1_modifiers,
8132                          AMDGPU::OpName::src2_modifiers };
8133 
8134   unsigned OpSel = Inst.getOperand(OpSelIdx).getImm();
8135 
8136   for (int J = 0; J < 3; ++J) {
8137     int OpIdx = AMDGPU::getNamedOperandIdx(Opc, Ops[J]);
8138     if (OpIdx == -1)
8139       break;
8140 
8141     int ModIdx = AMDGPU::getNamedOperandIdx(Opc, ModOps[J]);
8142     uint32_t ModVal = Inst.getOperand(ModIdx).getImm();
8143 
8144     if ((OpSel & (1 << J)) != 0)
8145       ModVal |= SISrcMods::OP_SEL_0;
8146     if (ModOps[J] == AMDGPU::OpName::src0_modifiers &&
8147         (OpSel & (1 << 3)) != 0)
8148       ModVal |= SISrcMods::DST_OP_SEL;
8149 
8150     Inst.getOperand(ModIdx).setImm(ModVal);
8151   }
8152 }
8153 
8154 void AMDGPUAsmParser::cvtVOP3(MCInst &Inst, const OperandVector &Operands,
8155                               OptionalImmIndexMap &OptionalIdx) {
8156   unsigned Opc = Inst.getOpcode();
8157 
8158   unsigned I = 1;
8159   const MCInstrDesc &Desc = MII.get(Inst.getOpcode());
8160   for (unsigned J = 0; J < Desc.getNumDefs(); ++J) {
8161     ((AMDGPUOperand &)*Operands[I++]).addRegOperands(Inst, 1);
8162   }
8163 
8164   if (AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src0_modifiers) != -1) {
8165     // This instruction has src modifiers
8166     for (unsigned E = Operands.size(); I != E; ++I) {
8167       AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[I]);
8168       if (isRegOrImmWithInputMods(Desc, Inst.getNumOperands())) {
8169         Op.addRegOrImmWithFPInputModsOperands(Inst, 2);
8170       } else if (Op.isImmModifier()) {
8171         OptionalIdx[Op.getImmTy()] = I;
8172       } else if (Op.isRegOrImm()) {
8173         Op.addRegOrImmOperands(Inst, 1);
8174       } else {
8175         llvm_unreachable("unhandled operand type");
8176       }
8177     }
8178   } else {
8179     // No src modifiers
8180     for (unsigned E = Operands.size(); I != E; ++I) {
8181       AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[I]);
8182       if (Op.isMod()) {
8183         OptionalIdx[Op.getImmTy()] = I;
8184       } else {
8185         Op.addRegOrImmOperands(Inst, 1);
8186       }
8187     }
8188   }
8189 
8190   if (AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::clamp) != -1) {
8191     addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyClampSI);
8192   }
8193 
8194   if (AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::omod) != -1) {
8195     addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyOModSI);
8196   }
8197 
8198   // Special case v_mac_{f16, f32} and v_fmac_{f16, f32} (gfx906/gfx10+):
8199   // it has src2 register operand that is tied to dst operand
8200   // we don't allow modifiers for this operand in assembler so src2_modifiers
8201   // should be 0.
8202   if (Opc == AMDGPU::V_MAC_F32_e64_gfx6_gfx7 ||
8203       Opc == AMDGPU::V_MAC_F32_e64_gfx10 ||
8204       Opc == AMDGPU::V_MAC_F32_e64_vi ||
8205       Opc == AMDGPU::V_MAC_LEGACY_F32_e64_gfx6_gfx7 ||
8206       Opc == AMDGPU::V_MAC_LEGACY_F32_e64_gfx10 ||
8207       Opc == AMDGPU::V_MAC_F16_e64_vi ||
8208       Opc == AMDGPU::V_FMAC_F64_e64_gfx90a ||
8209       Opc == AMDGPU::V_FMAC_F32_e64_gfx10 ||
8210       Opc == AMDGPU::V_FMAC_F32_e64_gfx11 ||
8211       Opc == AMDGPU::V_FMAC_F32_e64_vi ||
8212       Opc == AMDGPU::V_FMAC_LEGACY_F32_e64_gfx10 ||
8213       Opc == AMDGPU::V_FMAC_DX9_ZERO_F32_e64_gfx11 ||
8214       Opc == AMDGPU::V_FMAC_F16_e64_gfx10 ||
8215       Opc == AMDGPU::V_FMAC_F16_e64_gfx11) {
8216     auto it = Inst.begin();
8217     std::advance(it, AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src2_modifiers));
8218     it = Inst.insert(it, MCOperand::createImm(0)); // no modifiers for src2
8219     ++it;
8220     // Copy the operand to ensure it's not invalidated when Inst grows.
8221     Inst.insert(it, MCOperand(Inst.getOperand(0))); // src2 = dst
8222   }
8223 }
8224 
8225 void AMDGPUAsmParser::cvtVOP3(MCInst &Inst, const OperandVector &Operands) {
8226   OptionalImmIndexMap OptionalIdx;
8227   cvtVOP3(Inst, Operands, OptionalIdx);
8228 }
8229 
8230 void AMDGPUAsmParser::cvtVOP3P(MCInst &Inst, const OperandVector &Operands,
8231                                OptionalImmIndexMap &OptIdx) {
8232   const int Opc = Inst.getOpcode();
8233   const MCInstrDesc &Desc = MII.get(Opc);
8234 
8235   const bool IsPacked = (Desc.TSFlags & SIInstrFlags::IsPacked) != 0;
8236 
8237   if (AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::vdst_in) != -1) {
8238     assert(!IsPacked);
8239     Inst.addOperand(Inst.getOperand(0));
8240   }
8241 
8242   // FIXME: This is messy. Parse the modifiers as if it was a normal VOP3
8243   // instruction, and then figure out where to actually put the modifiers
8244 
8245   int OpSelIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::op_sel);
8246   if (OpSelIdx != -1) {
8247     addOptionalImmOperand(Inst, Operands, OptIdx, AMDGPUOperand::ImmTyOpSel);
8248   }
8249 
8250   int OpSelHiIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::op_sel_hi);
8251   if (OpSelHiIdx != -1) {
8252     int DefaultVal = IsPacked ? -1 : 0;
8253     addOptionalImmOperand(Inst, Operands, OptIdx, AMDGPUOperand::ImmTyOpSelHi,
8254                           DefaultVal);
8255   }
8256 
8257   int NegLoIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::neg_lo);
8258   if (NegLoIdx != -1) {
8259     addOptionalImmOperand(Inst, Operands, OptIdx, AMDGPUOperand::ImmTyNegLo);
8260     addOptionalImmOperand(Inst, Operands, OptIdx, AMDGPUOperand::ImmTyNegHi);
8261   }
8262 
8263   const int Ops[] = { AMDGPU::OpName::src0,
8264                       AMDGPU::OpName::src1,
8265                       AMDGPU::OpName::src2 };
8266   const int ModOps[] = { AMDGPU::OpName::src0_modifiers,
8267                          AMDGPU::OpName::src1_modifiers,
8268                          AMDGPU::OpName::src2_modifiers };
8269 
8270   unsigned OpSel = 0;
8271   unsigned OpSelHi = 0;
8272   unsigned NegLo = 0;
8273   unsigned NegHi = 0;
8274 
8275   if (OpSelIdx != -1)
8276     OpSel = Inst.getOperand(OpSelIdx).getImm();
8277 
8278   if (OpSelHiIdx != -1)
8279     OpSelHi = Inst.getOperand(OpSelHiIdx).getImm();
8280 
8281   if (NegLoIdx != -1) {
8282     int NegHiIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::neg_hi);
8283     NegLo = Inst.getOperand(NegLoIdx).getImm();
8284     NegHi = Inst.getOperand(NegHiIdx).getImm();
8285   }
8286 
8287   for (int J = 0; J < 3; ++J) {
8288     int OpIdx = AMDGPU::getNamedOperandIdx(Opc, Ops[J]);
8289     if (OpIdx == -1)
8290       break;
8291 
8292     int ModIdx = AMDGPU::getNamedOperandIdx(Opc, ModOps[J]);
8293 
8294     if (ModIdx == -1)
8295       continue;
8296 
8297     uint32_t ModVal = 0;
8298 
8299     if ((OpSel & (1 << J)) != 0)
8300       ModVal |= SISrcMods::OP_SEL_0;
8301 
8302     if ((OpSelHi & (1 << J)) != 0)
8303       ModVal |= SISrcMods::OP_SEL_1;
8304 
8305     if ((NegLo & (1 << J)) != 0)
8306       ModVal |= SISrcMods::NEG;
8307 
8308     if ((NegHi & (1 << J)) != 0)
8309       ModVal |= SISrcMods::NEG_HI;
8310 
8311     Inst.getOperand(ModIdx).setImm(Inst.getOperand(ModIdx).getImm() | ModVal);
8312   }
8313 }
8314 
8315 void AMDGPUAsmParser::cvtVOP3P(MCInst &Inst, const OperandVector &Operands) {
8316   OptionalImmIndexMap OptIdx;
8317   cvtVOP3(Inst, Operands, OptIdx);
8318   cvtVOP3P(Inst, Operands, OptIdx);
8319 }
8320 
8321 //===----------------------------------------------------------------------===//
8322 // dpp
8323 //===----------------------------------------------------------------------===//
8324 
8325 bool AMDGPUOperand::isDPP8() const {
8326   return isImmTy(ImmTyDPP8);
8327 }
8328 
8329 bool AMDGPUOperand::isDPPCtrl() const {
8330   using namespace AMDGPU::DPP;
8331 
8332   bool result = isImm() && getImmTy() == ImmTyDppCtrl && isUInt<9>(getImm());
8333   if (result) {
8334     int64_t Imm = getImm();
8335     return (Imm >= DppCtrl::QUAD_PERM_FIRST && Imm <= DppCtrl::QUAD_PERM_LAST) ||
8336            (Imm >= DppCtrl::ROW_SHL_FIRST && Imm <= DppCtrl::ROW_SHL_LAST) ||
8337            (Imm >= DppCtrl::ROW_SHR_FIRST && Imm <= DppCtrl::ROW_SHR_LAST) ||
8338            (Imm >= DppCtrl::ROW_ROR_FIRST && Imm <= DppCtrl::ROW_ROR_LAST) ||
8339            (Imm == DppCtrl::WAVE_SHL1) ||
8340            (Imm == DppCtrl::WAVE_ROL1) ||
8341            (Imm == DppCtrl::WAVE_SHR1) ||
8342            (Imm == DppCtrl::WAVE_ROR1) ||
8343            (Imm == DppCtrl::ROW_MIRROR) ||
8344            (Imm == DppCtrl::ROW_HALF_MIRROR) ||
8345            (Imm == DppCtrl::BCAST15) ||
8346            (Imm == DppCtrl::BCAST31) ||
8347            (Imm >= DppCtrl::ROW_SHARE_FIRST && Imm <= DppCtrl::ROW_SHARE_LAST) ||
8348            (Imm >= DppCtrl::ROW_XMASK_FIRST && Imm <= DppCtrl::ROW_XMASK_LAST);
8349   }
8350   return false;
8351 }
8352 
8353 //===----------------------------------------------------------------------===//
8354 // mAI
8355 //===----------------------------------------------------------------------===//
8356 
8357 bool AMDGPUOperand::isBLGP() const {
8358   return isImm() && getImmTy() == ImmTyBLGP && isUInt<3>(getImm());
8359 }
8360 
8361 bool AMDGPUOperand::isCBSZ() const {
8362   return isImm() && getImmTy() == ImmTyCBSZ && isUInt<3>(getImm());
8363 }
8364 
8365 bool AMDGPUOperand::isABID() const {
8366   return isImm() && getImmTy() == ImmTyABID && isUInt<4>(getImm());
8367 }
8368 
8369 bool AMDGPUOperand::isS16Imm() const {
8370   return isImm() && (isInt<16>(getImm()) || isUInt<16>(getImm()));
8371 }
8372 
8373 bool AMDGPUOperand::isU16Imm() const {
8374   return isImm() && isUInt<16>(getImm());
8375 }
8376 
8377 //===----------------------------------------------------------------------===//
8378 // dim
8379 //===----------------------------------------------------------------------===//
8380 
8381 bool AMDGPUAsmParser::parseDimId(unsigned &Encoding) {
8382   // We want to allow "dim:1D" etc.,
8383   // but the initial 1 is tokenized as an integer.
8384   std::string Token;
8385   if (isToken(AsmToken::Integer)) {
8386     SMLoc Loc = getToken().getEndLoc();
8387     Token = std::string(getTokenStr());
8388     lex();
8389     if (getLoc() != Loc)
8390       return false;
8391   }
8392 
8393   StringRef Suffix;
8394   if (!parseId(Suffix))
8395     return false;
8396   Token += Suffix;
8397 
8398   StringRef DimId = Token;
8399   if (DimId.startswith("SQ_RSRC_IMG_"))
8400     DimId = DimId.drop_front(12);
8401 
8402   const AMDGPU::MIMGDimInfo *DimInfo = AMDGPU::getMIMGDimInfoByAsmSuffix(DimId);
8403   if (!DimInfo)
8404     return false;
8405 
8406   Encoding = DimInfo->Encoding;
8407   return true;
8408 }
8409 
8410 OperandMatchResultTy AMDGPUAsmParser::parseDim(OperandVector &Operands) {
8411   if (!isGFX10Plus())
8412     return MatchOperand_NoMatch;
8413 
8414   SMLoc S = getLoc();
8415 
8416   if (!trySkipId("dim", AsmToken::Colon))
8417     return MatchOperand_NoMatch;
8418 
8419   unsigned Encoding;
8420   SMLoc Loc = getLoc();
8421   if (!parseDimId(Encoding)) {
8422     Error(Loc, "invalid dim value");
8423     return MatchOperand_ParseFail;
8424   }
8425 
8426   Operands.push_back(AMDGPUOperand::CreateImm(this, Encoding, S,
8427                                               AMDGPUOperand::ImmTyDim));
8428   return MatchOperand_Success;
8429 }
8430 
8431 //===----------------------------------------------------------------------===//
8432 // dpp
8433 //===----------------------------------------------------------------------===//
8434 
8435 OperandMatchResultTy AMDGPUAsmParser::parseDPP8(OperandVector &Operands) {
8436   SMLoc S = getLoc();
8437 
8438   if (!isGFX10Plus() || !trySkipId("dpp8", AsmToken::Colon))
8439     return MatchOperand_NoMatch;
8440 
8441   // dpp8:[%d,%d,%d,%d,%d,%d,%d,%d]
8442 
8443   int64_t Sels[8];
8444 
8445   if (!skipToken(AsmToken::LBrac, "expected an opening square bracket"))
8446     return MatchOperand_ParseFail;
8447 
8448   for (size_t i = 0; i < 8; ++i) {
8449     if (i > 0 && !skipToken(AsmToken::Comma, "expected a comma"))
8450       return MatchOperand_ParseFail;
8451 
8452     SMLoc Loc = getLoc();
8453     if (getParser().parseAbsoluteExpression(Sels[i]))
8454       return MatchOperand_ParseFail;
8455     if (0 > Sels[i] || 7 < Sels[i]) {
8456       Error(Loc, "expected a 3-bit value");
8457       return MatchOperand_ParseFail;
8458     }
8459   }
8460 
8461   if (!skipToken(AsmToken::RBrac, "expected a closing square bracket"))
8462     return MatchOperand_ParseFail;
8463 
8464   unsigned DPP8 = 0;
8465   for (size_t i = 0; i < 8; ++i)
8466     DPP8 |= (Sels[i] << (i * 3));
8467 
8468   Operands.push_back(AMDGPUOperand::CreateImm(this, DPP8, S, AMDGPUOperand::ImmTyDPP8));
8469   return MatchOperand_Success;
8470 }
8471 
8472 bool
8473 AMDGPUAsmParser::isSupportedDPPCtrl(StringRef Ctrl,
8474                                     const OperandVector &Operands) {
8475   if (Ctrl == "row_newbcast")
8476     return isGFX90A();
8477 
8478   if (Ctrl == "row_share" ||
8479       Ctrl == "row_xmask")
8480     return isGFX10Plus();
8481 
8482   if (Ctrl == "wave_shl" ||
8483       Ctrl == "wave_shr" ||
8484       Ctrl == "wave_rol" ||
8485       Ctrl == "wave_ror" ||
8486       Ctrl == "row_bcast")
8487     return isVI() || isGFX9();
8488 
8489   return Ctrl == "row_mirror" ||
8490          Ctrl == "row_half_mirror" ||
8491          Ctrl == "quad_perm" ||
8492          Ctrl == "row_shl" ||
8493          Ctrl == "row_shr" ||
8494          Ctrl == "row_ror";
8495 }
8496 
8497 int64_t
8498 AMDGPUAsmParser::parseDPPCtrlPerm() {
8499   // quad_perm:[%d,%d,%d,%d]
8500 
8501   if (!skipToken(AsmToken::LBrac, "expected an opening square bracket"))
8502     return -1;
8503 
8504   int64_t Val = 0;
8505   for (int i = 0; i < 4; ++i) {
8506     if (i > 0 && !skipToken(AsmToken::Comma, "expected a comma"))
8507       return -1;
8508 
8509     int64_t Temp;
8510     SMLoc Loc = getLoc();
8511     if (getParser().parseAbsoluteExpression(Temp))
8512       return -1;
8513     if (Temp < 0 || Temp > 3) {
8514       Error(Loc, "expected a 2-bit value");
8515       return -1;
8516     }
8517 
8518     Val += (Temp << i * 2);
8519   }
8520 
8521   if (!skipToken(AsmToken::RBrac, "expected a closing square bracket"))
8522     return -1;
8523 
8524   return Val;
8525 }
8526 
8527 int64_t
8528 AMDGPUAsmParser::parseDPPCtrlSel(StringRef Ctrl) {
8529   using namespace AMDGPU::DPP;
8530 
8531   // sel:%d
8532 
8533   int64_t Val;
8534   SMLoc Loc = getLoc();
8535 
8536   if (getParser().parseAbsoluteExpression(Val))
8537     return -1;
8538 
8539   struct DppCtrlCheck {
8540     int64_t Ctrl;
8541     int Lo;
8542     int Hi;
8543   };
8544 
8545   DppCtrlCheck Check = StringSwitch<DppCtrlCheck>(Ctrl)
8546     .Case("wave_shl",  {DppCtrl::WAVE_SHL1,       1,  1})
8547     .Case("wave_rol",  {DppCtrl::WAVE_ROL1,       1,  1})
8548     .Case("wave_shr",  {DppCtrl::WAVE_SHR1,       1,  1})
8549     .Case("wave_ror",  {DppCtrl::WAVE_ROR1,       1,  1})
8550     .Case("row_shl",   {DppCtrl::ROW_SHL0,        1, 15})
8551     .Case("row_shr",   {DppCtrl::ROW_SHR0,        1, 15})
8552     .Case("row_ror",   {DppCtrl::ROW_ROR0,        1, 15})
8553     .Case("row_share", {DppCtrl::ROW_SHARE_FIRST, 0, 15})
8554     .Case("row_xmask", {DppCtrl::ROW_XMASK_FIRST, 0, 15})
8555     .Case("row_newbcast", {DppCtrl::ROW_NEWBCAST_FIRST, 0, 15})
8556     .Default({-1, 0, 0});
8557 
8558   bool Valid;
8559   if (Check.Ctrl == -1) {
8560     Valid = (Ctrl == "row_bcast" && (Val == 15 || Val == 31));
8561     Val = (Val == 15)? DppCtrl::BCAST15 : DppCtrl::BCAST31;
8562   } else {
8563     Valid = Check.Lo <= Val && Val <= Check.Hi;
8564     Val = (Check.Lo == Check.Hi) ? Check.Ctrl : (Check.Ctrl | Val);
8565   }
8566 
8567   if (!Valid) {
8568     Error(Loc, Twine("invalid ", Ctrl) + Twine(" value"));
8569     return -1;
8570   }
8571 
8572   return Val;
8573 }
8574 
8575 OperandMatchResultTy
8576 AMDGPUAsmParser::parseDPPCtrl(OperandVector &Operands) {
8577   using namespace AMDGPU::DPP;
8578 
8579   if (!isToken(AsmToken::Identifier) ||
8580       !isSupportedDPPCtrl(getTokenStr(), Operands))
8581     return MatchOperand_NoMatch;
8582 
8583   SMLoc S = getLoc();
8584   int64_t Val = -1;
8585   StringRef Ctrl;
8586 
8587   parseId(Ctrl);
8588 
8589   if (Ctrl == "row_mirror") {
8590     Val = DppCtrl::ROW_MIRROR;
8591   } else if (Ctrl == "row_half_mirror") {
8592     Val = DppCtrl::ROW_HALF_MIRROR;
8593   } else {
8594     if (skipToken(AsmToken::Colon, "expected a colon")) {
8595       if (Ctrl == "quad_perm") {
8596         Val = parseDPPCtrlPerm();
8597       } else {
8598         Val = parseDPPCtrlSel(Ctrl);
8599       }
8600     }
8601   }
8602 
8603   if (Val == -1)
8604     return MatchOperand_ParseFail;
8605 
8606   Operands.push_back(
8607     AMDGPUOperand::CreateImm(this, Val, S, AMDGPUOperand::ImmTyDppCtrl));
8608   return MatchOperand_Success;
8609 }
8610 
8611 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultRowMask() const {
8612   return AMDGPUOperand::CreateImm(this, 0xf, SMLoc(), AMDGPUOperand::ImmTyDppRowMask);
8613 }
8614 
8615 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultEndpgmImmOperands() const {
8616   return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTyEndpgm);
8617 }
8618 
8619 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultBankMask() const {
8620   return AMDGPUOperand::CreateImm(this, 0xf, SMLoc(), AMDGPUOperand::ImmTyDppBankMask);
8621 }
8622 
8623 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultBoundCtrl() const {
8624   return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTyDppBoundCtrl);
8625 }
8626 
8627 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultFI() const {
8628   return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTyDppFi);
8629 }
8630 
8631 // Add dummy $old operand
8632 void AMDGPUAsmParser::cvtVOPC64NoDstDPP(MCInst &Inst,
8633                                         const OperandVector &Operands,
8634                                         bool IsDPP8) {
8635   Inst.addOperand(MCOperand::createReg(0));
8636   cvtVOP3DPP(Inst, Operands, IsDPP8);
8637 }
8638 
8639 void AMDGPUAsmParser::cvtVOP3DPP(MCInst &Inst, const OperandVector &Operands, bool IsDPP8) {
8640   OptionalImmIndexMap OptionalIdx;
8641   unsigned Opc = Inst.getOpcode();
8642   bool HasModifiers = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src0_modifiers) != -1;
8643   unsigned I = 1;
8644   const MCInstrDesc &Desc = MII.get(Inst.getOpcode());
8645   for (unsigned J = 0; J < Desc.getNumDefs(); ++J) {
8646     ((AMDGPUOperand &)*Operands[I++]).addRegOperands(Inst, 1);
8647   }
8648 
8649   int Fi = 0;
8650   for (unsigned E = Operands.size(); I != E; ++I) {
8651     auto TiedTo = Desc.getOperandConstraint(Inst.getNumOperands(),
8652                                             MCOI::TIED_TO);
8653     if (TiedTo != -1) {
8654       assert((unsigned)TiedTo < Inst.getNumOperands());
8655       // handle tied old or src2 for MAC instructions
8656       Inst.addOperand(Inst.getOperand(TiedTo));
8657     }
8658     AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[I]);
8659     // Add the register arguments
8660     if (IsDPP8 && Op.isFI()) {
8661       Fi = Op.getImm();
8662     } else if (HasModifiers &&
8663                isRegOrImmWithInputMods(Desc, Inst.getNumOperands())) {
8664       Op.addRegOrImmWithFPInputModsOperands(Inst, 2);
8665     } else if (Op.isReg()) {
8666       Op.addRegOperands(Inst, 1);
8667     } else if (Op.isImm() &&
8668                Desc.OpInfo[Inst.getNumOperands()].RegClass != -1) {
8669       assert(!HasModifiers && "Case should be unreachable with modifiers");
8670       assert(!Op.IsImmKindLiteral() && "Cannot use literal with DPP");
8671       Op.addImmOperands(Inst, 1);
8672     } else if (Op.isImm()) {
8673       OptionalIdx[Op.getImmTy()] = I;
8674     } else {
8675       llvm_unreachable("unhandled operand type");
8676     }
8677   }
8678   if (AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::clamp) != -1) {
8679     addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyClampSI);
8680   }
8681   if (AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::omod) != -1) {
8682     addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyOModSI);
8683   }
8684   if (Desc.TSFlags & SIInstrFlags::VOP3P)
8685     cvtVOP3P(Inst, Operands, OptionalIdx);
8686   else if (AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::op_sel) != -1) {
8687     addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyOpSel);
8688   }
8689 
8690   if (IsDPP8) {
8691     addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyDPP8);
8692     using namespace llvm::AMDGPU::DPP;
8693     Inst.addOperand(MCOperand::createImm(Fi? DPP8_FI_1 : DPP8_FI_0));
8694   } else {
8695     addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyDppCtrl, 0xe4);
8696     addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyDppRowMask, 0xf);
8697     addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyDppBankMask, 0xf);
8698     addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyDppBoundCtrl);
8699     if (AMDGPU::getNamedOperandIdx(Inst.getOpcode(), AMDGPU::OpName::fi) != -1) {
8700       addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyDppFi);
8701     }
8702   }
8703 }
8704 
8705 // Add dummy $old operand
8706 void AMDGPUAsmParser::cvtVOPCNoDstDPP(MCInst &Inst,
8707                                       const OperandVector &Operands,
8708                                       bool IsDPP8) {
8709   Inst.addOperand(MCOperand::createReg(0));
8710   cvtDPP(Inst, Operands, IsDPP8);
8711 }
8712 
8713 void AMDGPUAsmParser::cvtDPP(MCInst &Inst, const OperandVector &Operands, bool IsDPP8) {
8714   OptionalImmIndexMap OptionalIdx;
8715 
8716   unsigned Opc = Inst.getOpcode();
8717   bool HasModifiers =
8718       AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src0_modifiers) != -1;
8719   unsigned I = 1;
8720   const MCInstrDesc &Desc = MII.get(Inst.getOpcode());
8721   for (unsigned J = 0; J < Desc.getNumDefs(); ++J) {
8722     ((AMDGPUOperand &)*Operands[I++]).addRegOperands(Inst, 1);
8723   }
8724 
8725   int Fi = 0;
8726   for (unsigned E = Operands.size(); I != E; ++I) {
8727     auto TiedTo = Desc.getOperandConstraint(Inst.getNumOperands(),
8728                                             MCOI::TIED_TO);
8729     if (TiedTo != -1) {
8730       assert((unsigned)TiedTo < Inst.getNumOperands());
8731       // handle tied old or src2 for MAC instructions
8732       Inst.addOperand(Inst.getOperand(TiedTo));
8733     }
8734     AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[I]);
8735     // Add the register arguments
8736     if (Op.isReg() && validateVccOperand(Op.getReg())) {
8737       // VOP2b (v_add_u32, v_sub_u32 ...) dpp use "vcc" token.
8738       // Skip it.
8739       continue;
8740     }
8741 
8742     if (IsDPP8) {
8743       if (Op.isDPP8()) {
8744         Op.addImmOperands(Inst, 1);
8745       } else if (HasModifiers &&
8746                  isRegOrImmWithInputMods(Desc, Inst.getNumOperands())) {
8747         Op.addRegWithFPInputModsOperands(Inst, 2);
8748       } else if (Op.isFI()) {
8749         Fi = Op.getImm();
8750       } else if (Op.isReg()) {
8751         Op.addRegOperands(Inst, 1);
8752       } else {
8753         llvm_unreachable("Invalid operand type");
8754       }
8755     } else {
8756       if (HasModifiers &&
8757           isRegOrImmWithInputMods(Desc, Inst.getNumOperands())) {
8758         Op.addRegWithFPInputModsOperands(Inst, 2);
8759       } else if (Op.isReg()) {
8760         Op.addRegOperands(Inst, 1);
8761       } else if (Op.isDPPCtrl()) {
8762         Op.addImmOperands(Inst, 1);
8763       } else if (Op.isImm()) {
8764         // Handle optional arguments
8765         OptionalIdx[Op.getImmTy()] = I;
8766       } else {
8767         llvm_unreachable("Invalid operand type");
8768       }
8769     }
8770   }
8771 
8772   if (IsDPP8) {
8773     using namespace llvm::AMDGPU::DPP;
8774     Inst.addOperand(MCOperand::createImm(Fi? DPP8_FI_1 : DPP8_FI_0));
8775   } else {
8776     addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyDppRowMask, 0xf);
8777     addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyDppBankMask, 0xf);
8778     addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyDppBoundCtrl);
8779     if (AMDGPU::getNamedOperandIdx(Inst.getOpcode(), AMDGPU::OpName::fi) != -1) {
8780       addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyDppFi);
8781     }
8782   }
8783 }
8784 
8785 //===----------------------------------------------------------------------===//
8786 // sdwa
8787 //===----------------------------------------------------------------------===//
8788 
8789 OperandMatchResultTy
8790 AMDGPUAsmParser::parseSDWASel(OperandVector &Operands, StringRef Prefix,
8791                               AMDGPUOperand::ImmTy Type) {
8792   using namespace llvm::AMDGPU::SDWA;
8793 
8794   SMLoc S = getLoc();
8795   StringRef Value;
8796   OperandMatchResultTy res;
8797 
8798   SMLoc StringLoc;
8799   res = parseStringWithPrefix(Prefix, Value, StringLoc);
8800   if (res != MatchOperand_Success) {
8801     return res;
8802   }
8803 
8804   int64_t Int;
8805   Int = StringSwitch<int64_t>(Value)
8806         .Case("BYTE_0", SdwaSel::BYTE_0)
8807         .Case("BYTE_1", SdwaSel::BYTE_1)
8808         .Case("BYTE_2", SdwaSel::BYTE_2)
8809         .Case("BYTE_3", SdwaSel::BYTE_3)
8810         .Case("WORD_0", SdwaSel::WORD_0)
8811         .Case("WORD_1", SdwaSel::WORD_1)
8812         .Case("DWORD", SdwaSel::DWORD)
8813         .Default(0xffffffff);
8814 
8815   if (Int == 0xffffffff) {
8816     Error(StringLoc, "invalid " + Twine(Prefix) + " value");
8817     return MatchOperand_ParseFail;
8818   }
8819 
8820   Operands.push_back(AMDGPUOperand::CreateImm(this, Int, S, Type));
8821   return MatchOperand_Success;
8822 }
8823 
8824 OperandMatchResultTy
8825 AMDGPUAsmParser::parseSDWADstUnused(OperandVector &Operands) {
8826   using namespace llvm::AMDGPU::SDWA;
8827 
8828   SMLoc S = getLoc();
8829   StringRef Value;
8830   OperandMatchResultTy res;
8831 
8832   SMLoc StringLoc;
8833   res = parseStringWithPrefix("dst_unused", Value, StringLoc);
8834   if (res != MatchOperand_Success) {
8835     return res;
8836   }
8837 
8838   int64_t Int;
8839   Int = StringSwitch<int64_t>(Value)
8840         .Case("UNUSED_PAD", DstUnused::UNUSED_PAD)
8841         .Case("UNUSED_SEXT", DstUnused::UNUSED_SEXT)
8842         .Case("UNUSED_PRESERVE", DstUnused::UNUSED_PRESERVE)
8843         .Default(0xffffffff);
8844 
8845   if (Int == 0xffffffff) {
8846     Error(StringLoc, "invalid dst_unused value");
8847     return MatchOperand_ParseFail;
8848   }
8849 
8850   Operands.push_back(AMDGPUOperand::CreateImm(this, Int, S, AMDGPUOperand::ImmTySdwaDstUnused));
8851   return MatchOperand_Success;
8852 }
8853 
8854 void AMDGPUAsmParser::cvtSdwaVOP1(MCInst &Inst, const OperandVector &Operands) {
8855   cvtSDWA(Inst, Operands, SIInstrFlags::VOP1);
8856 }
8857 
8858 void AMDGPUAsmParser::cvtSdwaVOP2(MCInst &Inst, const OperandVector &Operands) {
8859   cvtSDWA(Inst, Operands, SIInstrFlags::VOP2);
8860 }
8861 
8862 void AMDGPUAsmParser::cvtSdwaVOP2b(MCInst &Inst, const OperandVector &Operands) {
8863   cvtSDWA(Inst, Operands, SIInstrFlags::VOP2, true, true);
8864 }
8865 
8866 void AMDGPUAsmParser::cvtSdwaVOP2e(MCInst &Inst, const OperandVector &Operands) {
8867   cvtSDWA(Inst, Operands, SIInstrFlags::VOP2, false, true);
8868 }
8869 
8870 void AMDGPUAsmParser::cvtSdwaVOPC(MCInst &Inst, const OperandVector &Operands) {
8871   cvtSDWA(Inst, Operands, SIInstrFlags::VOPC, isVI());
8872 }
8873 
8874 void AMDGPUAsmParser::cvtSDWA(MCInst &Inst, const OperandVector &Operands,
8875                               uint64_t BasicInstType,
8876                               bool SkipDstVcc,
8877                               bool SkipSrcVcc) {
8878   using namespace llvm::AMDGPU::SDWA;
8879 
8880   OptionalImmIndexMap OptionalIdx;
8881   bool SkipVcc = SkipDstVcc || SkipSrcVcc;
8882   bool SkippedVcc = false;
8883 
8884   unsigned I = 1;
8885   const MCInstrDesc &Desc = MII.get(Inst.getOpcode());
8886   for (unsigned J = 0; J < Desc.getNumDefs(); ++J) {
8887     ((AMDGPUOperand &)*Operands[I++]).addRegOperands(Inst, 1);
8888   }
8889 
8890   for (unsigned E = Operands.size(); I != E; ++I) {
8891     AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[I]);
8892     if (SkipVcc && !SkippedVcc && Op.isReg() &&
8893         (Op.getReg() == AMDGPU::VCC || Op.getReg() == AMDGPU::VCC_LO)) {
8894       // VOP2b (v_add_u32, v_sub_u32 ...) sdwa use "vcc" token as dst.
8895       // Skip it if it's 2nd (e.g. v_add_i32_sdwa v1, vcc, v2, v3)
8896       // or 4th (v_addc_u32_sdwa v1, vcc, v2, v3, vcc) operand.
8897       // Skip VCC only if we didn't skip it on previous iteration.
8898       // Note that src0 and src1 occupy 2 slots each because of modifiers.
8899       if (BasicInstType == SIInstrFlags::VOP2 &&
8900           ((SkipDstVcc && Inst.getNumOperands() == 1) ||
8901            (SkipSrcVcc && Inst.getNumOperands() == 5))) {
8902         SkippedVcc = true;
8903         continue;
8904       } else if (BasicInstType == SIInstrFlags::VOPC &&
8905                  Inst.getNumOperands() == 0) {
8906         SkippedVcc = true;
8907         continue;
8908       }
8909     }
8910     if (isRegOrImmWithInputMods(Desc, Inst.getNumOperands())) {
8911       Op.addRegOrImmWithInputModsOperands(Inst, 2);
8912     } else if (Op.isImm()) {
8913       // Handle optional arguments
8914       OptionalIdx[Op.getImmTy()] = I;
8915     } else {
8916       llvm_unreachable("Invalid operand type");
8917     }
8918     SkippedVcc = false;
8919   }
8920 
8921   if (Inst.getOpcode() != AMDGPU::V_NOP_sdwa_gfx10 &&
8922       Inst.getOpcode() != AMDGPU::V_NOP_sdwa_gfx9 &&
8923       Inst.getOpcode() != AMDGPU::V_NOP_sdwa_vi) {
8924     // v_nop_sdwa_sdwa_vi/gfx9 has no optional sdwa arguments
8925     switch (BasicInstType) {
8926     case SIInstrFlags::VOP1:
8927       addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyClampSI, 0);
8928       if (AMDGPU::getNamedOperandIdx(Inst.getOpcode(), AMDGPU::OpName::omod) != -1) {
8929         addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyOModSI, 0);
8930       }
8931       addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySdwaDstSel, SdwaSel::DWORD);
8932       addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySdwaDstUnused, DstUnused::UNUSED_PRESERVE);
8933       addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySdwaSrc0Sel, SdwaSel::DWORD);
8934       break;
8935 
8936     case SIInstrFlags::VOP2:
8937       addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyClampSI, 0);
8938       if (AMDGPU::getNamedOperandIdx(Inst.getOpcode(), AMDGPU::OpName::omod) != -1) {
8939         addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyOModSI, 0);
8940       }
8941       addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySdwaDstSel, SdwaSel::DWORD);
8942       addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySdwaDstUnused, DstUnused::UNUSED_PRESERVE);
8943       addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySdwaSrc0Sel, SdwaSel::DWORD);
8944       addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySdwaSrc1Sel, SdwaSel::DWORD);
8945       break;
8946 
8947     case SIInstrFlags::VOPC:
8948       if (AMDGPU::getNamedOperandIdx(Inst.getOpcode(), AMDGPU::OpName::clamp) != -1)
8949         addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyClampSI, 0);
8950       addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySdwaSrc0Sel, SdwaSel::DWORD);
8951       addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySdwaSrc1Sel, SdwaSel::DWORD);
8952       break;
8953 
8954     default:
8955       llvm_unreachable("Invalid instruction type. Only VOP1, VOP2 and VOPC allowed");
8956     }
8957   }
8958 
8959   // special case v_mac_{f16, f32}:
8960   // it has src2 register operand that is tied to dst operand
8961   if (Inst.getOpcode() == AMDGPU::V_MAC_F32_sdwa_vi ||
8962       Inst.getOpcode() == AMDGPU::V_MAC_F16_sdwa_vi)  {
8963     auto it = Inst.begin();
8964     std::advance(
8965       it, AMDGPU::getNamedOperandIdx(Inst.getOpcode(), AMDGPU::OpName::src2));
8966     Inst.insert(it, Inst.getOperand(0)); // src2 = dst
8967   }
8968 }
8969 
8970 //===----------------------------------------------------------------------===//
8971 // mAI
8972 //===----------------------------------------------------------------------===//
8973 
8974 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultBLGP() const {
8975   return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTyBLGP);
8976 }
8977 
8978 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultCBSZ() const {
8979   return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTyCBSZ);
8980 }
8981 
8982 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultABID() const {
8983   return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTyABID);
8984 }
8985 
8986 /// Force static initialization.
8987 extern "C" LLVM_EXTERNAL_VISIBILITY void LLVMInitializeAMDGPUAsmParser() {
8988   RegisterMCAsmParser<AMDGPUAsmParser> A(getTheAMDGPUTarget());
8989   RegisterMCAsmParser<AMDGPUAsmParser> B(getTheGCNTarget());
8990 }
8991 
8992 #define GET_REGISTER_MATCHER
8993 #define GET_MATCHER_IMPLEMENTATION
8994 #define GET_MNEMONIC_SPELL_CHECKER
8995 #define GET_MNEMONIC_CHECKER
8996 #include "AMDGPUGenAsmMatcher.inc"
8997 
8998 // This function should be defined after auto-generated include so that we have
8999 // MatchClassKind enum defined
9000 unsigned AMDGPUAsmParser::validateTargetOperandClass(MCParsedAsmOperand &Op,
9001                                                      unsigned Kind) {
9002   // Tokens like "glc" would be parsed as immediate operands in ParseOperand().
9003   // But MatchInstructionImpl() expects to meet token and fails to validate
9004   // operand. This method checks if we are given immediate operand but expect to
9005   // get corresponding token.
9006   AMDGPUOperand &Operand = (AMDGPUOperand&)Op;
9007   switch (Kind) {
9008   case MCK_addr64:
9009     return Operand.isAddr64() ? Match_Success : Match_InvalidOperand;
9010   case MCK_gds:
9011     return Operand.isGDS() ? Match_Success : Match_InvalidOperand;
9012   case MCK_lds:
9013     return Operand.isLDS() ? Match_Success : Match_InvalidOperand;
9014   case MCK_idxen:
9015     return Operand.isIdxen() ? Match_Success : Match_InvalidOperand;
9016   case MCK_offen:
9017     return Operand.isOffen() ? Match_Success : Match_InvalidOperand;
9018   case MCK_SSrcB32:
9019     // When operands have expression values, they will return true for isToken,
9020     // because it is not possible to distinguish between a token and an
9021     // expression at parse time. MatchInstructionImpl() will always try to
9022     // match an operand as a token, when isToken returns true, and when the
9023     // name of the expression is not a valid token, the match will fail,
9024     // so we need to handle it here.
9025     return Operand.isSSrcB32() ? Match_Success : Match_InvalidOperand;
9026   case MCK_SSrcF32:
9027     return Operand.isSSrcF32() ? Match_Success : Match_InvalidOperand;
9028   case MCK_SoppBrTarget:
9029     return Operand.isSoppBrTarget() ? Match_Success : Match_InvalidOperand;
9030   case MCK_VReg32OrOff:
9031     return Operand.isVReg32OrOff() ? Match_Success : Match_InvalidOperand;
9032   case MCK_InterpSlot:
9033     return Operand.isInterpSlot() ? Match_Success : Match_InvalidOperand;
9034   case MCK_Attr:
9035     return Operand.isInterpAttr() ? Match_Success : Match_InvalidOperand;
9036   case MCK_AttrChan:
9037     return Operand.isAttrChan() ? Match_Success : Match_InvalidOperand;
9038   case MCK_ImmSMEMOffset:
9039     return Operand.isSMEMOffset() ? Match_Success : Match_InvalidOperand;
9040   case MCK_SReg_64:
9041   case MCK_SReg_64_XEXEC:
9042     // Null is defined as a 32-bit register but
9043     // it should also be enabled with 64-bit operands.
9044     // The following code enables it for SReg_64 operands
9045     // used as source and destination. Remaining source
9046     // operands are handled in isInlinableImm.
9047     return Operand.isNull() ? Match_Success : Match_InvalidOperand;
9048   default:
9049     return Match_InvalidOperand;
9050   }
9051 }
9052 
9053 //===----------------------------------------------------------------------===//
9054 // endpgm
9055 //===----------------------------------------------------------------------===//
9056 
9057 OperandMatchResultTy AMDGPUAsmParser::parseEndpgmOp(OperandVector &Operands) {
9058   SMLoc S = getLoc();
9059   int64_t Imm = 0;
9060 
9061   if (!parseExpr(Imm)) {
9062     // The operand is optional, if not present default to 0
9063     Imm = 0;
9064   }
9065 
9066   if (!isUInt<16>(Imm)) {
9067     Error(S, "expected a 16-bit value");
9068     return MatchOperand_ParseFail;
9069   }
9070 
9071   Operands.push_back(
9072       AMDGPUOperand::CreateImm(this, Imm, S, AMDGPUOperand::ImmTyEndpgm));
9073   return MatchOperand_Success;
9074 }
9075 
9076 bool AMDGPUOperand::isEndpgm() const { return isImmTy(ImmTyEndpgm); }
9077 
9078 //===----------------------------------------------------------------------===//
9079 // LDSDIR
9080 //===----------------------------------------------------------------------===//
9081 
9082 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultWaitVDST() const {
9083   return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTyWaitVDST);
9084 }
9085 
9086 bool AMDGPUOperand::isWaitVDST() const {
9087   return isImmTy(ImmTyWaitVDST) && isUInt<4>(getImm());
9088 }
9089 
9090 //===----------------------------------------------------------------------===//
9091 // VINTERP
9092 //===----------------------------------------------------------------------===//
9093 
9094 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultWaitEXP() const {
9095   return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTyWaitEXP);
9096 }
9097 
9098 bool AMDGPUOperand::isWaitEXP() const {
9099   return isImmTy(ImmTyWaitEXP) && isUInt<3>(getImm());
9100 }
9101