1 //===-- AMDGPUAsmParser.cpp - Parse SI asm to MCInst instructions ----------===//
2 //
3 //                     The LLVM Compiler Infrastructure
4 //
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
7 //
8 //===----------------------------------------------------------------------===//
9 
10 #include "MCTargetDesc/AMDGPUMCTargetDesc.h"
11 #include "MCTargetDesc/AMDGPUTargetStreamer.h"
12 #include "Utils/AMDGPUBaseInfo.h"
13 #include "AMDKernelCodeT.h"
14 #include "SIDefines.h"
15 #include "llvm/ADT/APFloat.h"
16 #include "llvm/ADT/SmallString.h"
17 #include "llvm/ADT/SmallVector.h"
18 #include "llvm/ADT/STLExtras.h"
19 #include "llvm/ADT/StringSwitch.h"
20 #include "llvm/ADT/Twine.h"
21 #include "llvm/MC/MCContext.h"
22 #include "llvm/MC/MCExpr.h"
23 #include "llvm/MC/MCInst.h"
24 #include "llvm/MC/MCInstrInfo.h"
25 #include "llvm/MC/MCParser/MCAsmLexer.h"
26 #include "llvm/MC/MCParser/MCAsmParser.h"
27 #include "llvm/MC/MCParser/MCParsedAsmOperand.h"
28 #include "llvm/MC/MCRegisterInfo.h"
29 #include "llvm/MC/MCStreamer.h"
30 #include "llvm/MC/MCSubtargetInfo.h"
31 #include "llvm/MC/MCTargetAsmParser.h"
32 #include "llvm/Support/SourceMgr.h"
33 #include "llvm/Support/TargetRegistry.h"
34 #include "llvm/Support/raw_ostream.h"
35 #include "llvm/Support/Debug.h"
36 
37 using namespace llvm;
38 
39 namespace {
40 
41 struct OptionalOperand;
42 
43 class AMDGPUOperand : public MCParsedAsmOperand {
44   enum KindTy {
45     Token,
46     Immediate,
47     Register,
48     Expression
49   } Kind;
50 
51   SMLoc StartLoc, EndLoc;
52 
53 public:
54   AMDGPUOperand(enum KindTy K) : MCParsedAsmOperand(), Kind(K) {}
55 
56   MCContext *Ctx;
57 
58   enum ImmTy {
59     ImmTyNone,
60     ImmTyDSOffset0,
61     ImmTyDSOffset1,
62     ImmTyGDS,
63     ImmTyOffset,
64     ImmTyGLC,
65     ImmTySLC,
66     ImmTyTFE,
67     ImmTyClamp,
68     ImmTyOMod
69   };
70 
71   struct TokOp {
72     const char *Data;
73     unsigned Length;
74   };
75 
76   struct ImmOp {
77     bool IsFPImm;
78     ImmTy Type;
79     int64_t Val;
80   };
81 
82   struct RegOp {
83     unsigned RegNo;
84     int Modifiers;
85     const MCRegisterInfo *TRI;
86     bool IsForcedVOP3;
87   };
88 
89   union {
90     TokOp Tok;
91     ImmOp Imm;
92     RegOp Reg;
93     const MCExpr *Expr;
94   };
95 
96   void addImmOperands(MCInst &Inst, unsigned N) const {
97     Inst.addOperand(MCOperand::createImm(getImm()));
98   }
99 
100   StringRef getToken() const {
101     return StringRef(Tok.Data, Tok.Length);
102   }
103 
104   void addRegOperands(MCInst &Inst, unsigned N) const {
105     Inst.addOperand(MCOperand::createReg(getReg()));
106   }
107 
108   void addRegOrImmOperands(MCInst &Inst, unsigned N) const {
109     if (isReg())
110       addRegOperands(Inst, N);
111     else
112       addImmOperands(Inst, N);
113   }
114 
115   void addRegWithInputModsOperands(MCInst &Inst, unsigned N) const {
116     Inst.addOperand(MCOperand::createImm(
117         Reg.Modifiers == -1 ? 0 : Reg.Modifiers));
118     addRegOperands(Inst, N);
119   }
120 
121   void addSoppBrTargetOperands(MCInst &Inst, unsigned N) const {
122     if (isImm())
123       addImmOperands(Inst, N);
124     else {
125       assert(isExpr());
126       Inst.addOperand(MCOperand::createExpr(Expr));
127     }
128   }
129 
130   bool defaultTokenHasSuffix() const {
131     StringRef Token(Tok.Data, Tok.Length);
132 
133     return Token.endswith("_e32") || Token.endswith("_e64");
134   }
135 
136   bool isToken() const override {
137     return Kind == Token;
138   }
139 
140   bool isImm() const override {
141     return Kind == Immediate;
142   }
143 
144   bool isInlineImm() const {
145     float F = BitsToFloat(Imm.Val);
146     // TODO: Add 0.5pi for VI
147     return isImm() && ((Imm.Val <= 64 && Imm.Val >= -16) ||
148            (F == 0.0 || F == 0.5 || F == -0.5 || F == 1.0 || F == -1.0 ||
149            F == 2.0 || F == -2.0 || F == 4.0 || F == -4.0));
150   }
151 
152   bool isDSOffset0() const {
153     assert(isImm());
154     return Imm.Type == ImmTyDSOffset0;
155   }
156 
157   bool isDSOffset1() const {
158     assert(isImm());
159     return Imm.Type == ImmTyDSOffset1;
160   }
161 
162   int64_t getImm() const {
163     return Imm.Val;
164   }
165 
166   enum ImmTy getImmTy() const {
167     assert(isImm());
168     return Imm.Type;
169   }
170 
171   bool isRegKind() const {
172     return Kind == Register;
173   }
174 
175   bool isReg() const override {
176     return Kind == Register && Reg.Modifiers == -1;
177   }
178 
179   bool isRegWithInputMods() const {
180     return Kind == Register && (Reg.IsForcedVOP3 || Reg.Modifiers != -1);
181   }
182 
183   void setModifiers(unsigned Mods) {
184     assert(isReg());
185     Reg.Modifiers = Mods;
186   }
187 
188   bool hasModifiers() const {
189     assert(isRegKind());
190     return Reg.Modifiers != -1;
191   }
192 
193   unsigned getReg() const override {
194     return Reg.RegNo;
195   }
196 
197   bool isRegOrImm() const {
198     return isReg() || isImm();
199   }
200 
201   bool isRegClass(unsigned RCID) const {
202     return Reg.TRI->getRegClass(RCID).contains(getReg());
203   }
204 
205   bool isSCSrc32() const {
206     return isInlineImm() || (isReg() && isRegClass(AMDGPU::SReg_32RegClassID));
207   }
208 
209   bool isSSrc32() const {
210     return isImm() || (isReg() && isRegClass(AMDGPU::SReg_32RegClassID));
211   }
212 
213   bool isSSrc64() const {
214     return isImm() || isInlineImm() ||
215            (isReg() && isRegClass(AMDGPU::SReg_64RegClassID));
216   }
217 
218   bool isSCSrc64() const {
219     return (isReg() && isRegClass(AMDGPU::SReg_64RegClassID)) || isInlineImm();
220   }
221 
222   bool isVCSrc32() const {
223     return isInlineImm() || (isReg() && isRegClass(AMDGPU::VS_32RegClassID));
224   }
225 
226   bool isVCSrc64() const {
227     return isInlineImm() || (isReg() && isRegClass(AMDGPU::VS_64RegClassID));
228   }
229 
230   bool isVSrc32() const {
231     return isImm() || (isReg() && isRegClass(AMDGPU::VS_32RegClassID));
232   }
233 
234   bool isVSrc64() const {
235     return isImm() || (isReg() && isRegClass(AMDGPU::VS_64RegClassID));
236   }
237 
238   bool isMem() const override {
239     return false;
240   }
241 
242   bool isExpr() const {
243     return Kind == Expression;
244   }
245 
246   bool isSoppBrTarget() const {
247     return isExpr() || isImm();
248   }
249 
250   SMLoc getStartLoc() const override {
251     return StartLoc;
252   }
253 
254   SMLoc getEndLoc() const override {
255     return EndLoc;
256   }
257 
258   void print(raw_ostream &OS) const override {
259     switch (Kind) {
260     case Register:
261       OS << "<register " << getReg() << " mods: " << Reg.Modifiers << '>';
262       break;
263     case Immediate:
264       OS << getImm();
265       break;
266     case Token:
267       OS << '\'' << getToken() << '\'';
268       break;
269     case Expression:
270       OS << "<expr " << *Expr << '>';
271       break;
272     }
273   }
274 
275   static std::unique_ptr<AMDGPUOperand> CreateImm(int64_t Val, SMLoc Loc,
276                                                   enum ImmTy Type = ImmTyNone,
277                                                   bool IsFPImm = false) {
278     auto Op = llvm::make_unique<AMDGPUOperand>(Immediate);
279     Op->Imm.Val = Val;
280     Op->Imm.IsFPImm = IsFPImm;
281     Op->Imm.Type = Type;
282     Op->StartLoc = Loc;
283     Op->EndLoc = Loc;
284     return Op;
285   }
286 
287   static std::unique_ptr<AMDGPUOperand> CreateToken(StringRef Str, SMLoc Loc,
288                                            bool HasExplicitEncodingSize = true) {
289     auto Res = llvm::make_unique<AMDGPUOperand>(Token);
290     Res->Tok.Data = Str.data();
291     Res->Tok.Length = Str.size();
292     Res->StartLoc = Loc;
293     Res->EndLoc = Loc;
294     return Res;
295   }
296 
297   static std::unique_ptr<AMDGPUOperand> CreateReg(unsigned RegNo, SMLoc S,
298                                                   SMLoc E,
299                                                   const MCRegisterInfo *TRI,
300                                                   bool ForceVOP3) {
301     auto Op = llvm::make_unique<AMDGPUOperand>(Register);
302     Op->Reg.RegNo = RegNo;
303     Op->Reg.TRI = TRI;
304     Op->Reg.Modifiers = -1;
305     Op->Reg.IsForcedVOP3 = ForceVOP3;
306     Op->StartLoc = S;
307     Op->EndLoc = E;
308     return Op;
309   }
310 
311   static std::unique_ptr<AMDGPUOperand> CreateExpr(const class MCExpr *Expr, SMLoc S) {
312     auto Op = llvm::make_unique<AMDGPUOperand>(Expression);
313     Op->Expr = Expr;
314     Op->StartLoc = S;
315     Op->EndLoc = S;
316     return Op;
317   }
318 
319   bool isDSOffset() const;
320   bool isDSOffset01() const;
321   bool isSWaitCnt() const;
322   bool isMubufOffset() const;
323   bool isSMRDOffset() const;
324   bool isSMRDLiteralOffset() const;
325 };
326 
327 class AMDGPUAsmParser : public MCTargetAsmParser {
328   MCSubtargetInfo &STI;
329   const MCInstrInfo &MII;
330   MCAsmParser &Parser;
331 
332   unsigned ForcedEncodingSize;
333   /// @name Auto-generated Match Functions
334   /// {
335 
336 #define GET_ASSEMBLER_HEADER
337 #include "AMDGPUGenAsmMatcher.inc"
338 
339   /// }
340 
341 private:
342   bool ParseDirectiveMajorMinor(uint32_t &Major, uint32_t &Minor);
343   bool ParseDirectiveHSACodeObjectVersion();
344   bool ParseDirectiveHSACodeObjectISA();
345   bool ParseAMDKernelCodeTValue(StringRef ID, amd_kernel_code_t &Header);
346   bool ParseDirectiveAMDKernelCodeT();
347   bool ParseSectionDirectiveHSAText();
348 
349 public:
350 public:
351   enum AMDGPUMatchResultTy {
352     Match_PreferE32 = FIRST_TARGET_MATCH_RESULT_TY
353   };
354 
355   AMDGPUAsmParser(MCSubtargetInfo &STI, MCAsmParser &_Parser,
356                const MCInstrInfo &MII,
357                const MCTargetOptions &Options)
358       : MCTargetAsmParser(Options), STI(STI), MII(MII), Parser(_Parser),
359         ForcedEncodingSize(0){
360 
361     if (STI.getFeatureBits().none()) {
362       // Set default features.
363       STI.ToggleFeature("SOUTHERN_ISLANDS");
364     }
365 
366     setAvailableFeatures(ComputeAvailableFeatures(STI.getFeatureBits()));
367   }
368 
369   AMDGPUTargetStreamer &getTargetStreamer() {
370     MCTargetStreamer &TS = *getParser().getStreamer().getTargetStreamer();
371     return static_cast<AMDGPUTargetStreamer &>(TS);
372   }
373 
374   unsigned getForcedEncodingSize() const {
375     return ForcedEncodingSize;
376   }
377 
378   void setForcedEncodingSize(unsigned Size) {
379     ForcedEncodingSize = Size;
380   }
381 
382   bool isForcedVOP3() const {
383     return ForcedEncodingSize == 64;
384   }
385 
386   bool ParseRegister(unsigned &RegNo, SMLoc &StartLoc, SMLoc &EndLoc) override;
387   unsigned checkTargetMatchPredicate(MCInst &Inst) override;
388   bool MatchAndEmitInstruction(SMLoc IDLoc, unsigned &Opcode,
389                                OperandVector &Operands, MCStreamer &Out,
390                                uint64_t &ErrorInfo,
391                                bool MatchingInlineAsm) override;
392   bool ParseDirective(AsmToken DirectiveID) override;
393   OperandMatchResultTy parseOperand(OperandVector &Operands, StringRef Mnemonic);
394   bool ParseInstruction(ParseInstructionInfo &Info, StringRef Name,
395                         SMLoc NameLoc, OperandVector &Operands) override;
396 
397   OperandMatchResultTy parseIntWithPrefix(const char *Prefix, int64_t &Int,
398                                           int64_t Default = 0);
399   OperandMatchResultTy parseIntWithPrefix(const char *Prefix,
400                                           OperandVector &Operands,
401                                           enum AMDGPUOperand::ImmTy ImmTy =
402                                                       AMDGPUOperand::ImmTyNone);
403   OperandMatchResultTy parseNamedBit(const char *Name, OperandVector &Operands,
404                                      enum AMDGPUOperand::ImmTy ImmTy =
405                                                       AMDGPUOperand::ImmTyNone);
406   OperandMatchResultTy parseOptionalOps(
407                                    const ArrayRef<OptionalOperand> &OptionalOps,
408                                    OperandVector &Operands);
409 
410 
411   void cvtDSOffset01(MCInst &Inst, const OperandVector &Operands);
412   void cvtDS(MCInst &Inst, const OperandVector &Operands);
413   OperandMatchResultTy parseDSOptionalOps(OperandVector &Operands);
414   OperandMatchResultTy parseDSOff01OptionalOps(OperandVector &Operands);
415   OperandMatchResultTy parseDSOffsetOptional(OperandVector &Operands);
416 
417   bool parseCnt(int64_t &IntVal);
418   OperandMatchResultTy parseSWaitCntOps(OperandVector &Operands);
419   OperandMatchResultTy parseSOppBrTarget(OperandVector &Operands);
420 
421   OperandMatchResultTy parseFlatOptionalOps(OperandVector &Operands);
422   OperandMatchResultTy parseFlatAtomicOptionalOps(OperandVector &Operands);
423   void cvtFlat(MCInst &Inst, const OperandVector &Operands);
424 
425   void cvtMubuf(MCInst &Inst, const OperandVector &Operands);
426   OperandMatchResultTy parseOffset(OperandVector &Operands);
427   OperandMatchResultTy parseMubufOptionalOps(OperandVector &Operands);
428   OperandMatchResultTy parseGLC(OperandVector &Operands);
429   OperandMatchResultTy parseSLC(OperandVector &Operands);
430   OperandMatchResultTy parseTFE(OperandVector &Operands);
431 
432   OperandMatchResultTy parseDMask(OperandVector &Operands);
433   OperandMatchResultTy parseUNorm(OperandVector &Operands);
434   OperandMatchResultTy parseR128(OperandVector &Operands);
435 
436   void cvtVOP3(MCInst &Inst, const OperandVector &Operands);
437   OperandMatchResultTy parseVOP3OptionalOps(OperandVector &Operands);
438 };
439 
440 struct OptionalOperand {
441   const char *Name;
442   AMDGPUOperand::ImmTy Type;
443   bool IsBit;
444   int64_t Default;
445   bool (*ConvertResult)(int64_t&);
446 };
447 
448 }
449 
450 static int getRegClass(bool IsVgpr, unsigned RegWidth) {
451   if (IsVgpr) {
452     switch (RegWidth) {
453       default: return -1;
454       case 1: return AMDGPU::VGPR_32RegClassID;
455       case 2: return AMDGPU::VReg_64RegClassID;
456       case 3: return AMDGPU::VReg_96RegClassID;
457       case 4: return AMDGPU::VReg_128RegClassID;
458       case 8: return AMDGPU::VReg_256RegClassID;
459       case 16: return AMDGPU::VReg_512RegClassID;
460     }
461   }
462 
463   switch (RegWidth) {
464     default: return -1;
465     case 1: return AMDGPU::SGPR_32RegClassID;
466     case 2: return AMDGPU::SGPR_64RegClassID;
467     case 4: return AMDGPU::SReg_128RegClassID;
468     case 8: return AMDGPU::SReg_256RegClassID;
469     case 16: return AMDGPU::SReg_512RegClassID;
470   }
471 }
472 
473 static unsigned getRegForName(StringRef RegName) {
474 
475   return StringSwitch<unsigned>(RegName)
476     .Case("exec", AMDGPU::EXEC)
477     .Case("vcc", AMDGPU::VCC)
478     .Case("flat_scratch", AMDGPU::FLAT_SCR)
479     .Case("m0", AMDGPU::M0)
480     .Case("scc", AMDGPU::SCC)
481     .Case("flat_scratch_lo", AMDGPU::FLAT_SCR_LO)
482     .Case("flat_scratch_hi", AMDGPU::FLAT_SCR_HI)
483     .Case("vcc_lo", AMDGPU::VCC_LO)
484     .Case("vcc_hi", AMDGPU::VCC_HI)
485     .Case("exec_lo", AMDGPU::EXEC_LO)
486     .Case("exec_hi", AMDGPU::EXEC_HI)
487     .Default(0);
488 }
489 
490 bool AMDGPUAsmParser::ParseRegister(unsigned &RegNo, SMLoc &StartLoc, SMLoc &EndLoc) {
491   const AsmToken Tok = Parser.getTok();
492   StartLoc = Tok.getLoc();
493   EndLoc = Tok.getEndLoc();
494   StringRef RegName = Tok.getString();
495   RegNo = getRegForName(RegName);
496 
497   if (RegNo) {
498     Parser.Lex();
499     return false;
500   }
501 
502   // Match vgprs and sgprs
503   if (RegName[0] != 's' && RegName[0] != 'v')
504     return true;
505 
506   bool IsVgpr = RegName[0] == 'v';
507   unsigned RegWidth;
508   unsigned RegIndexInClass;
509   if (RegName.size() > 1) {
510     // We have a 32-bit register
511     RegWidth = 1;
512     if (RegName.substr(1).getAsInteger(10, RegIndexInClass))
513       return true;
514     Parser.Lex();
515   } else {
516     // We have a register greater than 32-bits.
517 
518     int64_t RegLo, RegHi;
519     Parser.Lex();
520     if (getLexer().isNot(AsmToken::LBrac))
521       return true;
522 
523     Parser.Lex();
524     if (getParser().parseAbsoluteExpression(RegLo))
525       return true;
526 
527     if (getLexer().isNot(AsmToken::Colon))
528       return true;
529 
530     Parser.Lex();
531     if (getParser().parseAbsoluteExpression(RegHi))
532       return true;
533 
534     if (getLexer().isNot(AsmToken::RBrac))
535       return true;
536 
537     Parser.Lex();
538     RegWidth = (RegHi - RegLo) + 1;
539     if (IsVgpr) {
540       // VGPR registers aren't aligned.
541       RegIndexInClass = RegLo;
542     } else {
543       // SGPR registers are aligned.  Max alignment is 4 dwords.
544       unsigned Size = std::min(RegWidth, 4u);
545       if (RegLo % Size != 0)
546         return true;
547 
548       RegIndexInClass = RegLo / Size;
549     }
550   }
551 
552   const MCRegisterInfo *TRI = getContext().getRegisterInfo();
553   int RCID = getRegClass(IsVgpr, RegWidth);
554   if (RCID == -1)
555     return true;
556 
557   const MCRegisterClass RC = TRI->getRegClass(RCID);
558   if (RegIndexInClass >= RC.getNumRegs())
559     return true;
560 
561   RegNo = RC.getRegister(RegIndexInClass);
562   return false;
563 }
564 
565 unsigned AMDGPUAsmParser::checkTargetMatchPredicate(MCInst &Inst) {
566 
567   uint64_t TSFlags = MII.get(Inst.getOpcode()).TSFlags;
568 
569   if ((getForcedEncodingSize() == 32 && (TSFlags & SIInstrFlags::VOP3)) ||
570       (getForcedEncodingSize() == 64 && !(TSFlags & SIInstrFlags::VOP3)))
571     return Match_InvalidOperand;
572 
573   if ((TSFlags & SIInstrFlags::VOP3) &&
574       (TSFlags & SIInstrFlags::VOPAsmPrefer32Bit) &&
575       getForcedEncodingSize() != 64)
576     return Match_PreferE32;
577 
578   return Match_Success;
579 }
580 
581 
582 bool AMDGPUAsmParser::MatchAndEmitInstruction(SMLoc IDLoc, unsigned &Opcode,
583                                               OperandVector &Operands,
584                                               MCStreamer &Out,
585                                               uint64_t &ErrorInfo,
586                                               bool MatchingInlineAsm) {
587   MCInst Inst;
588 
589   switch (MatchInstructionImpl(Operands, Inst, ErrorInfo, MatchingInlineAsm)) {
590     default: break;
591     case Match_Success:
592       Inst.setLoc(IDLoc);
593       Out.EmitInstruction(Inst, STI);
594       return false;
595     case Match_MissingFeature:
596       return Error(IDLoc, "instruction not supported on this GPU");
597 
598     case Match_MnemonicFail:
599       return Error(IDLoc, "unrecognized instruction mnemonic");
600 
601     case Match_InvalidOperand: {
602       SMLoc ErrorLoc = IDLoc;
603       if (ErrorInfo != ~0ULL) {
604         if (ErrorInfo >= Operands.size()) {
605           if (isForcedVOP3()) {
606             // If 64-bit encoding has been forced we can end up with no
607             // clamp or omod operands if none of the registers have modifiers,
608             // so we need to add these to the operand list.
609             AMDGPUOperand &LastOp =
610                 ((AMDGPUOperand &)*Operands[Operands.size() - 1]);
611             if (LastOp.isRegKind() ||
612                (LastOp.isImm() &&
613                 LastOp.getImmTy() != AMDGPUOperand::ImmTyNone)) {
614               SMLoc S = Parser.getTok().getLoc();
615               Operands.push_back(AMDGPUOperand::CreateImm(0, S,
616                                  AMDGPUOperand::ImmTyClamp));
617               Operands.push_back(AMDGPUOperand::CreateImm(0, S,
618                                  AMDGPUOperand::ImmTyOMod));
619               bool Res = MatchAndEmitInstruction(IDLoc, Opcode, Operands,
620                                                  Out, ErrorInfo,
621                                                  MatchingInlineAsm);
622               if (!Res)
623                 return Res;
624             }
625 
626           }
627           return Error(IDLoc, "too few operands for instruction");
628         }
629 
630         ErrorLoc = ((AMDGPUOperand &)*Operands[ErrorInfo]).getStartLoc();
631         if (ErrorLoc == SMLoc())
632           ErrorLoc = IDLoc;
633       }
634       return Error(ErrorLoc, "invalid operand for instruction");
635     }
636     case Match_PreferE32:
637       return Error(IDLoc, "internal error: instruction without _e64 suffix "
638                           "should be encoded as e32");
639   }
640   llvm_unreachable("Implement any new match types added!");
641 }
642 
643 bool AMDGPUAsmParser::ParseDirectiveMajorMinor(uint32_t &Major,
644                                                uint32_t &Minor) {
645   if (getLexer().isNot(AsmToken::Integer))
646     return TokError("invalid major version");
647 
648   Major = getLexer().getTok().getIntVal();
649   Lex();
650 
651   if (getLexer().isNot(AsmToken::Comma))
652     return TokError("minor version number required, comma expected");
653   Lex();
654 
655   if (getLexer().isNot(AsmToken::Integer))
656     return TokError("invalid minor version");
657 
658   Minor = getLexer().getTok().getIntVal();
659   Lex();
660 
661   return false;
662 }
663 
664 bool AMDGPUAsmParser::ParseDirectiveHSACodeObjectVersion() {
665 
666   uint32_t Major;
667   uint32_t Minor;
668 
669   if (ParseDirectiveMajorMinor(Major, Minor))
670     return true;
671 
672   getTargetStreamer().EmitDirectiveHSACodeObjectVersion(Major, Minor);
673   return false;
674 }
675 
676 bool AMDGPUAsmParser::ParseDirectiveHSACodeObjectISA() {
677 
678   uint32_t Major;
679   uint32_t Minor;
680   uint32_t Stepping;
681   StringRef VendorName;
682   StringRef ArchName;
683 
684   // If this directive has no arguments, then use the ISA version for the
685   // targeted GPU.
686   if (getLexer().is(AsmToken::EndOfStatement)) {
687     AMDGPU::IsaVersion Isa = AMDGPU::getIsaVersion(STI.getFeatureBits());
688     getTargetStreamer().EmitDirectiveHSACodeObjectISA(Isa.Major, Isa.Minor,
689                                                       Isa.Stepping,
690                                                       "AMD", "AMDGPU");
691     return false;
692   }
693 
694 
695   if (ParseDirectiveMajorMinor(Major, Minor))
696     return true;
697 
698   if (getLexer().isNot(AsmToken::Comma))
699     return TokError("stepping version number required, comma expected");
700   Lex();
701 
702   if (getLexer().isNot(AsmToken::Integer))
703     return TokError("invalid stepping version");
704 
705   Stepping = getLexer().getTok().getIntVal();
706   Lex();
707 
708   if (getLexer().isNot(AsmToken::Comma))
709     return TokError("vendor name required, comma expected");
710   Lex();
711 
712   if (getLexer().isNot(AsmToken::String))
713     return TokError("invalid vendor name");
714 
715   VendorName = getLexer().getTok().getStringContents();
716   Lex();
717 
718   if (getLexer().isNot(AsmToken::Comma))
719     return TokError("arch name required, comma expected");
720   Lex();
721 
722   if (getLexer().isNot(AsmToken::String))
723     return TokError("invalid arch name");
724 
725   ArchName = getLexer().getTok().getStringContents();
726   Lex();
727 
728   getTargetStreamer().EmitDirectiveHSACodeObjectISA(Major, Minor, Stepping,
729                                                     VendorName, ArchName);
730   return false;
731 }
732 
733 bool AMDGPUAsmParser::ParseAMDKernelCodeTValue(StringRef ID,
734                                                amd_kernel_code_t &Header) {
735 
736   if (getLexer().isNot(AsmToken::Equal))
737     return TokError("expected '='");
738   Lex();
739 
740   if (getLexer().isNot(AsmToken::Integer))
741     return TokError("amd_kernel_code_t values must be integers");
742 
743   uint64_t Value = getLexer().getTok().getIntVal();
744   Lex();
745 
746   if (ID == "kernel_code_version_major")
747     Header.amd_kernel_code_version_major = Value;
748   else if (ID == "kernel_code_version_minor")
749     Header.amd_kernel_code_version_minor = Value;
750   else if (ID == "machine_kind")
751     Header.amd_machine_kind = Value;
752   else if (ID == "machine_version_major")
753     Header.amd_machine_version_major = Value;
754   else if (ID == "machine_version_minor")
755     Header.amd_machine_version_minor = Value;
756   else if (ID == "machine_version_stepping")
757     Header.amd_machine_version_stepping = Value;
758   else if (ID == "kernel_code_entry_byte_offset")
759     Header.kernel_code_entry_byte_offset = Value;
760   else if (ID == "kernel_code_prefetch_byte_size")
761     Header.kernel_code_prefetch_byte_size = Value;
762   else if (ID == "max_scratch_backing_memory_byte_size")
763     Header.max_scratch_backing_memory_byte_size = Value;
764   else if (ID == "compute_pgm_rsrc1_vgprs")
765     Header.compute_pgm_resource_registers |= S_00B848_VGPRS(Value);
766   else if (ID == "compute_pgm_rsrc1_sgprs")
767     Header.compute_pgm_resource_registers |= S_00B848_SGPRS(Value);
768   else if (ID == "compute_pgm_rsrc1_priority")
769     Header.compute_pgm_resource_registers |= S_00B848_PRIORITY(Value);
770   else if (ID == "compute_pgm_rsrc1_float_mode")
771     Header.compute_pgm_resource_registers |= S_00B848_FLOAT_MODE(Value);
772   else if (ID == "compute_pgm_rsrc1_priv")
773     Header.compute_pgm_resource_registers |= S_00B848_PRIV(Value);
774   else if (ID == "compute_pgm_rsrc1_dx10_clamp")
775     Header.compute_pgm_resource_registers |= S_00B848_DX10_CLAMP(Value);
776   else if (ID == "compute_pgm_rsrc1_debug_mode")
777     Header.compute_pgm_resource_registers |= S_00B848_DEBUG_MODE(Value);
778   else if (ID == "compute_pgm_rsrc1_ieee_mode")
779     Header.compute_pgm_resource_registers |= S_00B848_IEEE_MODE(Value);
780   else if (ID == "compute_pgm_rsrc2_scratch_en")
781     Header.compute_pgm_resource_registers |= (S_00B84C_SCRATCH_EN(Value) << 32);
782   else if (ID == "compute_pgm_rsrc2_user_sgpr")
783     Header.compute_pgm_resource_registers |= (S_00B84C_USER_SGPR(Value) << 32);
784   else if (ID == "compute_pgm_rsrc2_tgid_x_en")
785     Header.compute_pgm_resource_registers |= (S_00B84C_TGID_X_EN(Value) << 32);
786   else if (ID == "compute_pgm_rsrc2_tgid_y_en")
787     Header.compute_pgm_resource_registers |= (S_00B84C_TGID_Y_EN(Value) << 32);
788   else if (ID == "compute_pgm_rsrc2_tgid_z_en")
789     Header.compute_pgm_resource_registers |= (S_00B84C_TGID_Z_EN(Value) << 32);
790   else if (ID == "compute_pgm_rsrc2_tg_size_en")
791     Header.compute_pgm_resource_registers |= (S_00B84C_TG_SIZE_EN(Value) << 32);
792   else if (ID == "compute_pgm_rsrc2_tidig_comp_cnt")
793     Header.compute_pgm_resource_registers |=
794         (S_00B84C_TIDIG_COMP_CNT(Value) << 32);
795   else if (ID == "compute_pgm_rsrc2_excp_en_msb")
796     Header.compute_pgm_resource_registers |=
797         (S_00B84C_EXCP_EN_MSB(Value) << 32);
798   else if (ID == "compute_pgm_rsrc2_lds_size")
799     Header.compute_pgm_resource_registers |= (S_00B84C_LDS_SIZE(Value) << 32);
800   else if (ID == "compute_pgm_rsrc2_excp_en")
801     Header.compute_pgm_resource_registers |= (S_00B84C_EXCP_EN(Value) << 32);
802   else if (ID == "compute_pgm_resource_registers")
803     Header.compute_pgm_resource_registers = Value;
804   else if (ID == "enable_sgpr_private_segment_buffer")
805     Header.code_properties |=
806         (Value << AMD_CODE_PROPERTY_ENABLE_SGPR_PRIVATE_SEGMENT_BUFFER_SHIFT);
807   else if (ID == "enable_sgpr_dispatch_ptr")
808     Header.code_properties |=
809         (Value << AMD_CODE_PROPERTY_ENABLE_SGPR_DISPATCH_PTR_SHIFT);
810   else if (ID == "enable_sgpr_queue_ptr")
811     Header.code_properties |=
812         (Value << AMD_CODE_PROPERTY_ENABLE_SGPR_QUEUE_PTR_SHIFT);
813   else if (ID == "enable_sgpr_kernarg_segment_ptr")
814     Header.code_properties |=
815         (Value << AMD_CODE_PROPERTY_ENABLE_SGPR_KERNARG_SEGMENT_PTR_SHIFT);
816   else if (ID == "enable_sgpr_dispatch_id")
817     Header.code_properties |=
818         (Value << AMD_CODE_PROPERTY_ENABLE_SGPR_DISPATCH_ID_SHIFT);
819   else if (ID == "enable_sgpr_flat_scratch_init")
820     Header.code_properties |=
821         (Value << AMD_CODE_PROPERTY_ENABLE_SGPR_FLAT_SCRATCH_INIT_SHIFT);
822   else if (ID == "enable_sgpr_private_segment_size")
823     Header.code_properties |=
824         (Value << AMD_CODE_PROPERTY_ENABLE_SGPR_PRIVATE_SEGMENT_SIZE_SHIFT);
825   else if (ID == "enable_sgpr_grid_workgroup_count_x")
826     Header.code_properties |=
827         (Value << AMD_CODE_PROPERTY_ENABLE_SGPR_GRID_WORKGROUP_COUNT_X_SHIFT);
828   else if (ID == "enable_sgpr_grid_workgroup_count_y")
829     Header.code_properties |=
830         (Value << AMD_CODE_PROPERTY_ENABLE_SGPR_GRID_WORKGROUP_COUNT_Y_SHIFT);
831   else if (ID == "enable_sgpr_grid_workgroup_count_z")
832     Header.code_properties |=
833         (Value << AMD_CODE_PROPERTY_ENABLE_SGPR_GRID_WORKGROUP_COUNT_Z_SHIFT);
834   else if (ID == "enable_ordered_append_gds")
835     Header.code_properties |=
836         (Value << AMD_CODE_PROPERTY_ENABLE_ORDERED_APPEND_GDS_SHIFT);
837   else if (ID == "private_element_size")
838     Header.code_properties |=
839         (Value << AMD_CODE_PROPERTY_PRIVATE_ELEMENT_SIZE_SHIFT);
840   else if (ID == "is_ptr64")
841     Header.code_properties |=
842         (Value << AMD_CODE_PROPERTY_IS_PTR64_SHIFT);
843   else if (ID == "is_dynamic_callstack")
844     Header.code_properties |=
845         (Value << AMD_CODE_PROPERTY_IS_DYNAMIC_CALLSTACK_SHIFT);
846   else if (ID == "is_debug_enabled")
847     Header.code_properties |=
848         (Value << AMD_CODE_PROPERTY_IS_DEBUG_SUPPORTED_SHIFT);
849   else if (ID == "is_xnack_enabled")
850     Header.code_properties |=
851         (Value << AMD_CODE_PROPERTY_IS_XNACK_SUPPORTED_SHIFT);
852   else if (ID == "workitem_private_segment_byte_size")
853     Header.workitem_private_segment_byte_size = Value;
854   else if (ID == "workgroup_group_segment_byte_size")
855     Header.workgroup_group_segment_byte_size = Value;
856   else if (ID == "gds_segment_byte_size")
857     Header.gds_segment_byte_size = Value;
858   else if (ID == "kernarg_segment_byte_size")
859     Header.kernarg_segment_byte_size = Value;
860   else if (ID == "workgroup_fbarrier_count")
861     Header.workgroup_fbarrier_count = Value;
862   else if (ID == "wavefront_sgpr_count")
863     Header.wavefront_sgpr_count = Value;
864   else if (ID == "workitem_vgpr_count")
865     Header.workitem_vgpr_count = Value;
866   else if (ID == "reserved_vgpr_first")
867     Header.reserved_vgpr_first = Value;
868   else if (ID == "reserved_vgpr_count")
869     Header.reserved_vgpr_count = Value;
870   else if (ID == "reserved_sgpr_first")
871     Header.reserved_sgpr_first = Value;
872   else if (ID == "reserved_sgpr_count")
873     Header.reserved_sgpr_count = Value;
874   else if (ID == "debug_wavefront_private_segment_offset_sgpr")
875     Header.debug_wavefront_private_segment_offset_sgpr = Value;
876   else if (ID == "debug_private_segment_buffer_sgpr")
877     Header.debug_private_segment_buffer_sgpr = Value;
878   else if (ID == "kernarg_segment_alignment")
879     Header.kernarg_segment_alignment = Value;
880   else if (ID == "group_segment_alignment")
881     Header.group_segment_alignment = Value;
882   else if (ID == "private_segment_alignment")
883     Header.private_segment_alignment = Value;
884   else if (ID == "wavefront_size")
885     Header.wavefront_size = Value;
886   else if (ID == "call_convention")
887     Header.call_convention = Value;
888   else if (ID == "runtime_loader_kernel_symbol")
889     Header.runtime_loader_kernel_symbol = Value;
890   else
891     return TokError("amd_kernel_code_t value not recognized.");
892 
893   return false;
894 }
895 
896 bool AMDGPUAsmParser::ParseDirectiveAMDKernelCodeT() {
897 
898   amd_kernel_code_t Header;
899   AMDGPU::initDefaultAMDKernelCodeT(Header, STI.getFeatureBits());
900 
901   while (true) {
902 
903     if (getLexer().isNot(AsmToken::EndOfStatement))
904       return TokError("amd_kernel_code_t values must begin on a new line");
905 
906     // Lex EndOfStatement.  This is in a while loop, because lexing a comment
907     // will set the current token to EndOfStatement.
908     while(getLexer().is(AsmToken::EndOfStatement))
909       Lex();
910 
911     if (getLexer().isNot(AsmToken::Identifier))
912       return TokError("expected value identifier or .end_amd_kernel_code_t");
913 
914     StringRef ID = getLexer().getTok().getIdentifier();
915     Lex();
916 
917     if (ID == ".end_amd_kernel_code_t")
918       break;
919 
920     if (ParseAMDKernelCodeTValue(ID, Header))
921       return true;
922   }
923 
924   getTargetStreamer().EmitAMDKernelCodeT(Header);
925 
926   return false;
927 }
928 
929 bool AMDGPUAsmParser::ParseSectionDirectiveHSAText() {
930   getParser().getStreamer().SwitchSection(
931       AMDGPU::getHSATextSection(getContext()));
932   return false;
933 }
934 
935 bool AMDGPUAsmParser::ParseDirective(AsmToken DirectiveID) {
936   StringRef IDVal = DirectiveID.getString();
937 
938   if (IDVal == ".hsa_code_object_version")
939     return ParseDirectiveHSACodeObjectVersion();
940 
941   if (IDVal == ".hsa_code_object_isa")
942     return ParseDirectiveHSACodeObjectISA();
943 
944   if (IDVal == ".amd_kernel_code_t")
945     return ParseDirectiveAMDKernelCodeT();
946 
947   if (IDVal == ".hsatext" || IDVal == ".text")
948     return ParseSectionDirectiveHSAText();
949 
950   return true;
951 }
952 
953 static bool operandsHaveModifiers(const OperandVector &Operands) {
954 
955   for (unsigned i = 0, e = Operands.size(); i != e; ++i) {
956     const AMDGPUOperand &Op = ((AMDGPUOperand&)*Operands[i]);
957     if (Op.isRegKind() && Op.hasModifiers())
958       return true;
959     if (Op.isImm() && (Op.getImmTy() == AMDGPUOperand::ImmTyOMod ||
960                        Op.getImmTy() == AMDGPUOperand::ImmTyClamp))
961       return true;
962   }
963   return false;
964 }
965 
966 AMDGPUAsmParser::OperandMatchResultTy
967 AMDGPUAsmParser::parseOperand(OperandVector &Operands, StringRef Mnemonic) {
968 
969   // Try to parse with a custom parser
970   OperandMatchResultTy ResTy = MatchOperandParserImpl(Operands, Mnemonic);
971 
972   // If we successfully parsed the operand or if there as an error parsing,
973   // we are done.
974   //
975   // If we are parsing after we reach EndOfStatement then this means we
976   // are appending default values to the Operands list.  This is only done
977   // by custom parser, so we shouldn't continue on to the generic parsing.
978   if (ResTy == MatchOperand_Success || ResTy == MatchOperand_ParseFail ||
979       getLexer().is(AsmToken::EndOfStatement))
980     return ResTy;
981 
982   bool Negate = false, Abs = false;
983   if (getLexer().getKind()== AsmToken::Minus) {
984     Parser.Lex();
985     Negate = true;
986   }
987 
988   if (getLexer().getKind() == AsmToken::Pipe) {
989     Parser.Lex();
990     Abs = true;
991   }
992 
993   switch(getLexer().getKind()) {
994     case AsmToken::Integer: {
995       SMLoc S = Parser.getTok().getLoc();
996       int64_t IntVal;
997       if (getParser().parseAbsoluteExpression(IntVal))
998         return MatchOperand_ParseFail;
999       if (!isInt<32>(IntVal) && !isUInt<32>(IntVal)) {
1000         Error(S, "invalid immediate: only 32-bit values are legal");
1001         return MatchOperand_ParseFail;
1002       }
1003 
1004       if (Negate)
1005         IntVal *= -1;
1006       Operands.push_back(AMDGPUOperand::CreateImm(IntVal, S));
1007       return MatchOperand_Success;
1008     }
1009     case AsmToken::Real: {
1010       // FIXME: We should emit an error if a double precisions floating-point
1011       // value is used.  I'm not sure the best way to detect this.
1012       SMLoc S = Parser.getTok().getLoc();
1013       int64_t IntVal;
1014       if (getParser().parseAbsoluteExpression(IntVal))
1015         return MatchOperand_ParseFail;
1016 
1017       APFloat F((float)BitsToDouble(IntVal));
1018       if (Negate)
1019         F.changeSign();
1020       Operands.push_back(
1021           AMDGPUOperand::CreateImm(F.bitcastToAPInt().getZExtValue(), S));
1022       return MatchOperand_Success;
1023     }
1024     case AsmToken::Identifier: {
1025       SMLoc S, E;
1026       unsigned RegNo;
1027       if (!ParseRegister(RegNo, S, E)) {
1028 
1029         bool HasModifiers = operandsHaveModifiers(Operands);
1030         unsigned Modifiers = 0;
1031 
1032         if (Negate)
1033           Modifiers |= 0x1;
1034 
1035         if (Abs) {
1036           if (getLexer().getKind() != AsmToken::Pipe)
1037             return MatchOperand_ParseFail;
1038           Parser.Lex();
1039           Modifiers |= 0x2;
1040         }
1041 
1042         if (Modifiers && !HasModifiers) {
1043           // We are adding a modifier to src1 or src2 and previous sources
1044           // don't have modifiers, so we need to go back and empty modifers
1045           // for each previous source.
1046           for (unsigned PrevRegIdx = Operands.size() - 1; PrevRegIdx > 1;
1047                --PrevRegIdx) {
1048 
1049             AMDGPUOperand &RegOp = ((AMDGPUOperand&)*Operands[PrevRegIdx]);
1050             RegOp.setModifiers(0);
1051           }
1052         }
1053 
1054 
1055         Operands.push_back(AMDGPUOperand::CreateReg(
1056             RegNo, S, E, getContext().getRegisterInfo(),
1057             isForcedVOP3()));
1058 
1059         if (HasModifiers || Modifiers) {
1060           AMDGPUOperand &RegOp = ((AMDGPUOperand&)*Operands[Operands.size() - 1]);
1061           RegOp.setModifiers(Modifiers);
1062 
1063         }
1064      }  else {
1065       Operands.push_back(AMDGPUOperand::CreateToken(Parser.getTok().getString(),
1066                                                     S));
1067       Parser.Lex();
1068      }
1069      return MatchOperand_Success;
1070     }
1071     default:
1072       return MatchOperand_NoMatch;
1073   }
1074 }
1075 
1076 bool AMDGPUAsmParser::ParseInstruction(ParseInstructionInfo &Info,
1077                                        StringRef Name,
1078                                        SMLoc NameLoc, OperandVector &Operands) {
1079 
1080   // Clear any forced encodings from the previous instruction.
1081   setForcedEncodingSize(0);
1082 
1083   if (Name.endswith("_e64"))
1084     setForcedEncodingSize(64);
1085   else if (Name.endswith("_e32"))
1086     setForcedEncodingSize(32);
1087 
1088   // Add the instruction mnemonic
1089   Operands.push_back(AMDGPUOperand::CreateToken(Name, NameLoc));
1090 
1091   while (!getLexer().is(AsmToken::EndOfStatement)) {
1092     AMDGPUAsmParser::OperandMatchResultTy Res = parseOperand(Operands, Name);
1093 
1094     // Eat the comma or space if there is one.
1095     if (getLexer().is(AsmToken::Comma))
1096       Parser.Lex();
1097 
1098     switch (Res) {
1099       case MatchOperand_Success: break;
1100       case MatchOperand_ParseFail: return Error(getLexer().getLoc(),
1101                                                 "failed parsing operand.");
1102       case MatchOperand_NoMatch: return Error(getLexer().getLoc(),
1103                                               "not a valid operand.");
1104     }
1105   }
1106 
1107   // Once we reach end of statement, continue parsing so we can add default
1108   // values for optional arguments.
1109   AMDGPUAsmParser::OperandMatchResultTy Res;
1110   while ((Res = parseOperand(Operands, Name)) != MatchOperand_NoMatch) {
1111     if (Res != MatchOperand_Success)
1112       return Error(getLexer().getLoc(), "failed parsing operand.");
1113   }
1114   return false;
1115 }
1116 
1117 //===----------------------------------------------------------------------===//
1118 // Utility functions
1119 //===----------------------------------------------------------------------===//
1120 
1121 AMDGPUAsmParser::OperandMatchResultTy
1122 AMDGPUAsmParser::parseIntWithPrefix(const char *Prefix, int64_t &Int,
1123                                     int64_t Default) {
1124 
1125   // We are at the end of the statement, and this is a default argument, so
1126   // use a default value.
1127   if (getLexer().is(AsmToken::EndOfStatement)) {
1128     Int = Default;
1129     return MatchOperand_Success;
1130   }
1131 
1132   switch(getLexer().getKind()) {
1133     default: return MatchOperand_NoMatch;
1134     case AsmToken::Identifier: {
1135       StringRef OffsetName = Parser.getTok().getString();
1136       if (!OffsetName.equals(Prefix))
1137         return MatchOperand_NoMatch;
1138 
1139       Parser.Lex();
1140       if (getLexer().isNot(AsmToken::Colon))
1141         return MatchOperand_ParseFail;
1142 
1143       Parser.Lex();
1144       if (getLexer().isNot(AsmToken::Integer))
1145         return MatchOperand_ParseFail;
1146 
1147       if (getParser().parseAbsoluteExpression(Int))
1148         return MatchOperand_ParseFail;
1149       break;
1150     }
1151   }
1152   return MatchOperand_Success;
1153 }
1154 
1155 AMDGPUAsmParser::OperandMatchResultTy
1156 AMDGPUAsmParser::parseIntWithPrefix(const char *Prefix, OperandVector &Operands,
1157                                     enum AMDGPUOperand::ImmTy ImmTy) {
1158 
1159   SMLoc S = Parser.getTok().getLoc();
1160   int64_t Offset = 0;
1161 
1162   AMDGPUAsmParser::OperandMatchResultTy Res = parseIntWithPrefix(Prefix, Offset);
1163   if (Res != MatchOperand_Success)
1164     return Res;
1165 
1166   Operands.push_back(AMDGPUOperand::CreateImm(Offset, S, ImmTy));
1167   return MatchOperand_Success;
1168 }
1169 
1170 AMDGPUAsmParser::OperandMatchResultTy
1171 AMDGPUAsmParser::parseNamedBit(const char *Name, OperandVector &Operands,
1172                                enum AMDGPUOperand::ImmTy ImmTy) {
1173   int64_t Bit = 0;
1174   SMLoc S = Parser.getTok().getLoc();
1175 
1176   // We are at the end of the statement, and this is a default argument, so
1177   // use a default value.
1178   if (getLexer().isNot(AsmToken::EndOfStatement)) {
1179     switch(getLexer().getKind()) {
1180       case AsmToken::Identifier: {
1181         StringRef Tok = Parser.getTok().getString();
1182         if (Tok == Name) {
1183           Bit = 1;
1184           Parser.Lex();
1185         } else if (Tok.startswith("no") && Tok.endswith(Name)) {
1186           Bit = 0;
1187           Parser.Lex();
1188         } else {
1189           return MatchOperand_NoMatch;
1190         }
1191         break;
1192       }
1193       default:
1194         return MatchOperand_NoMatch;
1195     }
1196   }
1197 
1198   Operands.push_back(AMDGPUOperand::CreateImm(Bit, S, ImmTy));
1199   return MatchOperand_Success;
1200 }
1201 
1202 static bool operandsHasOptionalOp(const OperandVector &Operands,
1203                                   const OptionalOperand &OOp) {
1204   for (unsigned i = 0; i < Operands.size(); i++) {
1205     const AMDGPUOperand &ParsedOp = ((const AMDGPUOperand &)*Operands[i]);
1206     if ((ParsedOp.isImm() && ParsedOp.getImmTy() == OOp.Type) ||
1207         (ParsedOp.isToken() && ParsedOp.getToken() == OOp.Name))
1208       return true;
1209 
1210   }
1211   return false;
1212 }
1213 
1214 AMDGPUAsmParser::OperandMatchResultTy
1215 AMDGPUAsmParser::parseOptionalOps(const ArrayRef<OptionalOperand> &OptionalOps,
1216                                    OperandVector &Operands) {
1217   SMLoc S = Parser.getTok().getLoc();
1218   for (const OptionalOperand &Op : OptionalOps) {
1219     if (operandsHasOptionalOp(Operands, Op))
1220       continue;
1221     AMDGPUAsmParser::OperandMatchResultTy Res;
1222     int64_t Value;
1223     if (Op.IsBit) {
1224       Res = parseNamedBit(Op.Name, Operands, Op.Type);
1225       if (Res == MatchOperand_NoMatch)
1226         continue;
1227       return Res;
1228     }
1229 
1230     Res = parseIntWithPrefix(Op.Name, Value, Op.Default);
1231 
1232     if (Res == MatchOperand_NoMatch)
1233       continue;
1234 
1235     if (Res != MatchOperand_Success)
1236       return Res;
1237 
1238     if (Op.ConvertResult && !Op.ConvertResult(Value)) {
1239       return MatchOperand_ParseFail;
1240     }
1241 
1242     Operands.push_back(AMDGPUOperand::CreateImm(Value, S, Op.Type));
1243     return MatchOperand_Success;
1244   }
1245   return MatchOperand_NoMatch;
1246 }
1247 
1248 //===----------------------------------------------------------------------===//
1249 // ds
1250 //===----------------------------------------------------------------------===//
1251 
1252 static const OptionalOperand DSOptionalOps [] = {
1253   {"offset",  AMDGPUOperand::ImmTyOffset, false, 0, nullptr},
1254   {"gds",     AMDGPUOperand::ImmTyGDS, true, 0, nullptr}
1255 };
1256 
1257 static const OptionalOperand DSOptionalOpsOff01 [] = {
1258   {"offset0", AMDGPUOperand::ImmTyDSOffset0, false, 0, nullptr},
1259   {"offset1", AMDGPUOperand::ImmTyDSOffset1, false, 0, nullptr},
1260   {"gds",     AMDGPUOperand::ImmTyGDS, true, 0, nullptr}
1261 };
1262 
1263 AMDGPUAsmParser::OperandMatchResultTy
1264 AMDGPUAsmParser::parseDSOptionalOps(OperandVector &Operands) {
1265   return parseOptionalOps(DSOptionalOps, Operands);
1266 }
1267 AMDGPUAsmParser::OperandMatchResultTy
1268 AMDGPUAsmParser::parseDSOff01OptionalOps(OperandVector &Operands) {
1269   return parseOptionalOps(DSOptionalOpsOff01, Operands);
1270 }
1271 
1272 AMDGPUAsmParser::OperandMatchResultTy
1273 AMDGPUAsmParser::parseDSOffsetOptional(OperandVector &Operands) {
1274   SMLoc S = Parser.getTok().getLoc();
1275   AMDGPUAsmParser::OperandMatchResultTy Res =
1276     parseIntWithPrefix("offset", Operands, AMDGPUOperand::ImmTyOffset);
1277   if (Res == MatchOperand_NoMatch) {
1278     Operands.push_back(AMDGPUOperand::CreateImm(0, S,
1279                        AMDGPUOperand::ImmTyOffset));
1280     Res = MatchOperand_Success;
1281   }
1282   return Res;
1283 }
1284 
1285 bool AMDGPUOperand::isDSOffset() const {
1286   return isImm() && isUInt<16>(getImm());
1287 }
1288 
1289 bool AMDGPUOperand::isDSOffset01() const {
1290   return isImm() && isUInt<8>(getImm());
1291 }
1292 
1293 void AMDGPUAsmParser::cvtDSOffset01(MCInst &Inst,
1294                                     const OperandVector &Operands) {
1295 
1296   std::map<enum AMDGPUOperand::ImmTy, unsigned> OptionalIdx;
1297 
1298   for (unsigned i = 1, e = Operands.size(); i != e; ++i) {
1299     AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[i]);
1300 
1301     // Add the register arguments
1302     if (Op.isReg()) {
1303       Op.addRegOperands(Inst, 1);
1304       continue;
1305     }
1306 
1307     // Handle optional arguments
1308     OptionalIdx[Op.getImmTy()] = i;
1309   }
1310 
1311   unsigned Offset0Idx = OptionalIdx[AMDGPUOperand::ImmTyDSOffset0];
1312   unsigned Offset1Idx = OptionalIdx[AMDGPUOperand::ImmTyDSOffset1];
1313   unsigned GDSIdx = OptionalIdx[AMDGPUOperand::ImmTyGDS];
1314 
1315   ((AMDGPUOperand &)*Operands[Offset0Idx]).addImmOperands(Inst, 1); // offset0
1316   ((AMDGPUOperand &)*Operands[Offset1Idx]).addImmOperands(Inst, 1); // offset1
1317   ((AMDGPUOperand &)*Operands[GDSIdx]).addImmOperands(Inst, 1); // gds
1318   Inst.addOperand(MCOperand::createReg(AMDGPU::M0)); // m0
1319 }
1320 
1321 void AMDGPUAsmParser::cvtDS(MCInst &Inst, const OperandVector &Operands) {
1322 
1323   std::map<enum AMDGPUOperand::ImmTy, unsigned> OptionalIdx;
1324   bool GDSOnly = false;
1325 
1326   for (unsigned i = 1, e = Operands.size(); i != e; ++i) {
1327     AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[i]);
1328 
1329     // Add the register arguments
1330     if (Op.isReg()) {
1331       Op.addRegOperands(Inst, 1);
1332       continue;
1333     }
1334 
1335     if (Op.isToken() && Op.getToken() == "gds") {
1336       GDSOnly = true;
1337       continue;
1338     }
1339 
1340     // Handle optional arguments
1341     OptionalIdx[Op.getImmTy()] = i;
1342   }
1343 
1344   unsigned OffsetIdx = OptionalIdx[AMDGPUOperand::ImmTyOffset];
1345   ((AMDGPUOperand &)*Operands[OffsetIdx]).addImmOperands(Inst, 1); // offset
1346 
1347   if (!GDSOnly) {
1348     unsigned GDSIdx = OptionalIdx[AMDGPUOperand::ImmTyGDS];
1349     ((AMDGPUOperand &)*Operands[GDSIdx]).addImmOperands(Inst, 1); // gds
1350   }
1351   Inst.addOperand(MCOperand::createReg(AMDGPU::M0)); // m0
1352 }
1353 
1354 
1355 //===----------------------------------------------------------------------===//
1356 // s_waitcnt
1357 //===----------------------------------------------------------------------===//
1358 
1359 bool AMDGPUAsmParser::parseCnt(int64_t &IntVal) {
1360   StringRef CntName = Parser.getTok().getString();
1361   int64_t CntVal;
1362 
1363   Parser.Lex();
1364   if (getLexer().isNot(AsmToken::LParen))
1365     return true;
1366 
1367   Parser.Lex();
1368   if (getLexer().isNot(AsmToken::Integer))
1369     return true;
1370 
1371   if (getParser().parseAbsoluteExpression(CntVal))
1372     return true;
1373 
1374   if (getLexer().isNot(AsmToken::RParen))
1375     return true;
1376 
1377   Parser.Lex();
1378   if (getLexer().is(AsmToken::Amp) || getLexer().is(AsmToken::Comma))
1379     Parser.Lex();
1380 
1381   int CntShift;
1382   int CntMask;
1383 
1384   if (CntName == "vmcnt") {
1385     CntMask = 0xf;
1386     CntShift = 0;
1387   } else if (CntName == "expcnt") {
1388     CntMask = 0x7;
1389     CntShift = 4;
1390   } else if (CntName == "lgkmcnt") {
1391     CntMask = 0x7;
1392     CntShift = 8;
1393   } else {
1394     return true;
1395   }
1396 
1397   IntVal &= ~(CntMask << CntShift);
1398   IntVal |= (CntVal << CntShift);
1399   return false;
1400 }
1401 
1402 AMDGPUAsmParser::OperandMatchResultTy
1403 AMDGPUAsmParser::parseSWaitCntOps(OperandVector &Operands) {
1404   // Disable all counters by default.
1405   // vmcnt   [3:0]
1406   // expcnt  [6:4]
1407   // lgkmcnt [10:8]
1408   int64_t CntVal = 0x77f;
1409   SMLoc S = Parser.getTok().getLoc();
1410 
1411   switch(getLexer().getKind()) {
1412     default: return MatchOperand_ParseFail;
1413     case AsmToken::Integer:
1414       // The operand can be an integer value.
1415       if (getParser().parseAbsoluteExpression(CntVal))
1416         return MatchOperand_ParseFail;
1417       break;
1418 
1419     case AsmToken::Identifier:
1420       do {
1421         if (parseCnt(CntVal))
1422           return MatchOperand_ParseFail;
1423       } while(getLexer().isNot(AsmToken::EndOfStatement));
1424       break;
1425   }
1426   Operands.push_back(AMDGPUOperand::CreateImm(CntVal, S));
1427   return MatchOperand_Success;
1428 }
1429 
1430 bool AMDGPUOperand::isSWaitCnt() const {
1431   return isImm();
1432 }
1433 
1434 //===----------------------------------------------------------------------===//
1435 // sopp branch targets
1436 //===----------------------------------------------------------------------===//
1437 
1438 AMDGPUAsmParser::OperandMatchResultTy
1439 AMDGPUAsmParser::parseSOppBrTarget(OperandVector &Operands) {
1440   SMLoc S = Parser.getTok().getLoc();
1441 
1442   switch (getLexer().getKind()) {
1443     default: return MatchOperand_ParseFail;
1444     case AsmToken::Integer: {
1445       int64_t Imm;
1446       if (getParser().parseAbsoluteExpression(Imm))
1447         return MatchOperand_ParseFail;
1448       Operands.push_back(AMDGPUOperand::CreateImm(Imm, S));
1449       return MatchOperand_Success;
1450     }
1451 
1452     case AsmToken::Identifier:
1453       Operands.push_back(AMDGPUOperand::CreateExpr(
1454           MCSymbolRefExpr::create(getContext().getOrCreateSymbol(
1455                                   Parser.getTok().getString()), getContext()), S));
1456       Parser.Lex();
1457       return MatchOperand_Success;
1458   }
1459 }
1460 
1461 //===----------------------------------------------------------------------===//
1462 // flat
1463 //===----------------------------------------------------------------------===//
1464 
1465 static const OptionalOperand FlatOptionalOps [] = {
1466   {"glc",    AMDGPUOperand::ImmTyGLC, true, 0, nullptr},
1467   {"slc",    AMDGPUOperand::ImmTySLC, true, 0, nullptr},
1468   {"tfe",    AMDGPUOperand::ImmTyTFE, true, 0, nullptr}
1469 };
1470 
1471 static const OptionalOperand FlatAtomicOptionalOps [] = {
1472   {"slc",    AMDGPUOperand::ImmTySLC, true, 0, nullptr},
1473   {"tfe",    AMDGPUOperand::ImmTyTFE, true, 0, nullptr}
1474 };
1475 
1476 AMDGPUAsmParser::OperandMatchResultTy
1477 AMDGPUAsmParser::parseFlatOptionalOps(OperandVector &Operands) {
1478   return parseOptionalOps(FlatOptionalOps, Operands);
1479 }
1480 
1481 AMDGPUAsmParser::OperandMatchResultTy
1482 AMDGPUAsmParser::parseFlatAtomicOptionalOps(OperandVector &Operands) {
1483   return parseOptionalOps(FlatAtomicOptionalOps, Operands);
1484 }
1485 
1486 void AMDGPUAsmParser::cvtFlat(MCInst &Inst,
1487                                const OperandVector &Operands) {
1488   std::map<AMDGPUOperand::ImmTy, unsigned> OptionalIdx;
1489 
1490   for (unsigned i = 1, e = Operands.size(); i != e; ++i) {
1491     AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[i]);
1492 
1493     // Add the register arguments
1494     if (Op.isReg()) {
1495       Op.addRegOperands(Inst, 1);
1496       continue;
1497     }
1498 
1499     // Handle 'glc' token which is sometimes hard-coded into the
1500     // asm string.  There are no MCInst operands for these.
1501     if (Op.isToken())
1502       continue;
1503 
1504     // Handle optional arguments
1505     OptionalIdx[Op.getImmTy()] = i;
1506 
1507   }
1508 
1509   // flat atomic instructions don't have a glc argument.
1510   if (OptionalIdx.count(AMDGPUOperand::ImmTyGLC)) {
1511     unsigned GLCIdx = OptionalIdx[AMDGPUOperand::ImmTyGLC];
1512     ((AMDGPUOperand &)*Operands[GLCIdx]).addImmOperands(Inst, 1);
1513   }
1514 
1515   unsigned SLCIdx = OptionalIdx[AMDGPUOperand::ImmTySLC];
1516   unsigned TFEIdx = OptionalIdx[AMDGPUOperand::ImmTyTFE];
1517 
1518   ((AMDGPUOperand &)*Operands[SLCIdx]).addImmOperands(Inst, 1);
1519   ((AMDGPUOperand &)*Operands[TFEIdx]).addImmOperands(Inst, 1);
1520 }
1521 
1522 //===----------------------------------------------------------------------===//
1523 // mubuf
1524 //===----------------------------------------------------------------------===//
1525 
1526 static const OptionalOperand MubufOptionalOps [] = {
1527   {"offset", AMDGPUOperand::ImmTyOffset, false, 0, nullptr},
1528   {"glc",    AMDGPUOperand::ImmTyGLC, true, 0, nullptr},
1529   {"slc",    AMDGPUOperand::ImmTySLC, true, 0, nullptr},
1530   {"tfe",    AMDGPUOperand::ImmTyTFE, true, 0, nullptr}
1531 };
1532 
1533 AMDGPUAsmParser::OperandMatchResultTy
1534 AMDGPUAsmParser::parseMubufOptionalOps(OperandVector &Operands) {
1535   return parseOptionalOps(MubufOptionalOps, Operands);
1536 }
1537 
1538 AMDGPUAsmParser::OperandMatchResultTy
1539 AMDGPUAsmParser::parseOffset(OperandVector &Operands) {
1540   return parseIntWithPrefix("offset", Operands);
1541 }
1542 
1543 AMDGPUAsmParser::OperandMatchResultTy
1544 AMDGPUAsmParser::parseGLC(OperandVector &Operands) {
1545   return parseNamedBit("glc", Operands);
1546 }
1547 
1548 AMDGPUAsmParser::OperandMatchResultTy
1549 AMDGPUAsmParser::parseSLC(OperandVector &Operands) {
1550   return parseNamedBit("slc", Operands);
1551 }
1552 
1553 AMDGPUAsmParser::OperandMatchResultTy
1554 AMDGPUAsmParser::parseTFE(OperandVector &Operands) {
1555   return parseNamedBit("tfe", Operands);
1556 }
1557 
1558 bool AMDGPUOperand::isMubufOffset() const {
1559   return isImm() && isUInt<12>(getImm());
1560 }
1561 
1562 void AMDGPUAsmParser::cvtMubuf(MCInst &Inst,
1563                                const OperandVector &Operands) {
1564   std::map<enum AMDGPUOperand::ImmTy, unsigned> OptionalIdx;
1565 
1566   for (unsigned i = 1, e = Operands.size(); i != e; ++i) {
1567     AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[i]);
1568 
1569     // Add the register arguments
1570     if (Op.isReg()) {
1571       Op.addRegOperands(Inst, 1);
1572       continue;
1573     }
1574 
1575     // Handle the case where soffset is an immediate
1576     if (Op.isImm() && Op.getImmTy() == AMDGPUOperand::ImmTyNone) {
1577       Op.addImmOperands(Inst, 1);
1578       continue;
1579     }
1580 
1581     // Handle tokens like 'offen' which are sometimes hard-coded into the
1582     // asm string.  There are no MCInst operands for these.
1583     if (Op.isToken()) {
1584       continue;
1585     }
1586     assert(Op.isImm());
1587 
1588     // Handle optional arguments
1589     OptionalIdx[Op.getImmTy()] = i;
1590   }
1591 
1592   assert(OptionalIdx.size() == 4);
1593 
1594   unsigned OffsetIdx = OptionalIdx[AMDGPUOperand::ImmTyOffset];
1595   unsigned GLCIdx = OptionalIdx[AMDGPUOperand::ImmTyGLC];
1596   unsigned SLCIdx = OptionalIdx[AMDGPUOperand::ImmTySLC];
1597   unsigned TFEIdx = OptionalIdx[AMDGPUOperand::ImmTyTFE];
1598 
1599   ((AMDGPUOperand &)*Operands[OffsetIdx]).addImmOperands(Inst, 1);
1600   ((AMDGPUOperand &)*Operands[GLCIdx]).addImmOperands(Inst, 1);
1601   ((AMDGPUOperand &)*Operands[SLCIdx]).addImmOperands(Inst, 1);
1602   ((AMDGPUOperand &)*Operands[TFEIdx]).addImmOperands(Inst, 1);
1603 }
1604 
1605 //===----------------------------------------------------------------------===//
1606 // mimg
1607 //===----------------------------------------------------------------------===//
1608 
1609 AMDGPUAsmParser::OperandMatchResultTy
1610 AMDGPUAsmParser::parseDMask(OperandVector &Operands) {
1611   return parseIntWithPrefix("dmask", Operands);
1612 }
1613 
1614 AMDGPUAsmParser::OperandMatchResultTy
1615 AMDGPUAsmParser::parseUNorm(OperandVector &Operands) {
1616   return parseNamedBit("unorm", Operands);
1617 }
1618 
1619 AMDGPUAsmParser::OperandMatchResultTy
1620 AMDGPUAsmParser::parseR128(OperandVector &Operands) {
1621   return parseNamedBit("r128", Operands);
1622 }
1623 
1624 //===----------------------------------------------------------------------===//
1625 // smrd
1626 //===----------------------------------------------------------------------===//
1627 
1628 bool AMDGPUOperand::isSMRDOffset() const {
1629 
1630   // FIXME: Support 20-bit offsets on VI.  We need to to pass subtarget
1631   // information here.
1632   return isImm() && isUInt<8>(getImm());
1633 }
1634 
1635 bool AMDGPUOperand::isSMRDLiteralOffset() const {
1636   // 32-bit literals are only supported on CI and we only want to use them
1637   // when the offset is > 8-bits.
1638   return isImm() && !isUInt<8>(getImm()) && isUInt<32>(getImm());
1639 }
1640 
1641 //===----------------------------------------------------------------------===//
1642 // vop3
1643 //===----------------------------------------------------------------------===//
1644 
1645 static bool ConvertOmodMul(int64_t &Mul) {
1646   if (Mul != 1 && Mul != 2 && Mul != 4)
1647     return false;
1648 
1649   Mul >>= 1;
1650   return true;
1651 }
1652 
1653 static bool ConvertOmodDiv(int64_t &Div) {
1654   if (Div == 1) {
1655     Div = 0;
1656     return true;
1657   }
1658 
1659   if (Div == 2) {
1660     Div = 3;
1661     return true;
1662   }
1663 
1664   return false;
1665 }
1666 
1667 static const OptionalOperand VOP3OptionalOps [] = {
1668   {"clamp", AMDGPUOperand::ImmTyClamp, true, 0, nullptr},
1669   {"mul",   AMDGPUOperand::ImmTyOMod, false, 1, ConvertOmodMul},
1670   {"div",   AMDGPUOperand::ImmTyOMod, false, 1, ConvertOmodDiv},
1671 };
1672 
1673 static bool isVOP3(OperandVector &Operands) {
1674   if (operandsHaveModifiers(Operands))
1675     return true;
1676 
1677   AMDGPUOperand &DstOp = ((AMDGPUOperand&)*Operands[1]);
1678 
1679   if (DstOp.isReg() && DstOp.isRegClass(AMDGPU::SGPR_64RegClassID))
1680     return true;
1681 
1682   if (Operands.size() >= 5)
1683     return true;
1684 
1685   if (Operands.size() > 3) {
1686     AMDGPUOperand &Src1Op = ((AMDGPUOperand&)*Operands[3]);
1687     if (Src1Op.getReg() && (Src1Op.isRegClass(AMDGPU::SReg_32RegClassID) ||
1688                             Src1Op.isRegClass(AMDGPU::SReg_64RegClassID)))
1689       return true;
1690   }
1691   return false;
1692 }
1693 
1694 AMDGPUAsmParser::OperandMatchResultTy
1695 AMDGPUAsmParser::parseVOP3OptionalOps(OperandVector &Operands) {
1696 
1697   // The value returned by this function may change after parsing
1698   // an operand so store the original value here.
1699   bool HasModifiers = operandsHaveModifiers(Operands);
1700 
1701   bool IsVOP3 = isVOP3(Operands);
1702   if (HasModifiers || IsVOP3 ||
1703       getLexer().isNot(AsmToken::EndOfStatement) ||
1704       getForcedEncodingSize() == 64) {
1705 
1706     AMDGPUAsmParser::OperandMatchResultTy Res =
1707         parseOptionalOps(VOP3OptionalOps, Operands);
1708 
1709     if (!HasModifiers && Res == MatchOperand_Success) {
1710       // We have added a modifier operation, so we need to make sure all
1711       // previous register operands have modifiers
1712       for (unsigned i = 2, e = Operands.size(); i != e; ++i) {
1713         AMDGPUOperand &Op = ((AMDGPUOperand&)*Operands[i]);
1714         if (Op.isReg())
1715           Op.setModifiers(0);
1716       }
1717     }
1718     return Res;
1719   }
1720   return MatchOperand_NoMatch;
1721 }
1722 
1723 void AMDGPUAsmParser::cvtVOP3(MCInst &Inst, const OperandVector &Operands) {
1724 
1725   unsigned i = 1;
1726   const MCInstrDesc &Desc = MII.get(Inst.getOpcode());
1727   if (Desc.getNumDefs() > 0) {
1728     ((AMDGPUOperand &)*Operands[i++]).addRegOperands(Inst, 1);
1729   }
1730 
1731   std::map<enum AMDGPUOperand::ImmTy, unsigned> OptionalIdx;
1732 
1733   if (operandsHaveModifiers(Operands)) {
1734     for (unsigned e = Operands.size(); i != e; ++i) {
1735       AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[i]);
1736 
1737       if (Op.isRegWithInputMods()) {
1738         ((AMDGPUOperand &)*Operands[i]).addRegWithInputModsOperands(Inst, 2);
1739         continue;
1740       }
1741       OptionalIdx[Op.getImmTy()] = i;
1742     }
1743 
1744     unsigned ClampIdx = OptionalIdx[AMDGPUOperand::ImmTyClamp];
1745     unsigned OModIdx = OptionalIdx[AMDGPUOperand::ImmTyOMod];
1746 
1747     ((AMDGPUOperand &)*Operands[ClampIdx]).addImmOperands(Inst, 1);
1748     ((AMDGPUOperand &)*Operands[OModIdx]).addImmOperands(Inst, 1);
1749   } else {
1750     for (unsigned e = Operands.size(); i != e; ++i)
1751       ((AMDGPUOperand &)*Operands[i]).addRegOrImmOperands(Inst, 1);
1752   }
1753 }
1754 
1755 /// Force static initialization.
1756 extern "C" void LLVMInitializeAMDGPUAsmParser() {
1757   RegisterMCAsmParser<AMDGPUAsmParser> A(TheAMDGPUTarget);
1758   RegisterMCAsmParser<AMDGPUAsmParser> B(TheGCNTarget);
1759 }
1760 
1761 #define GET_REGISTER_MATCHER
1762 #define GET_MATCHER_IMPLEMENTATION
1763 #include "AMDGPUGenAsmMatcher.inc"
1764 
1765