1 //===- AsmLexer.cpp - Lexer for Assembly Files ----------------------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This class implements the lexer for assembly files.
10 //
11 //===----------------------------------------------------------------------===//
12 
13 #include "llvm/MC/MCParser/AsmLexer.h"
14 #include "llvm/ADT/APInt.h"
15 #include "llvm/ADT/ArrayRef.h"
16 #include "llvm/ADT/StringExtras.h"
17 #include "llvm/ADT/StringRef.h"
18 #include "llvm/ADT/StringSwitch.h"
19 #include "llvm/MC/MCAsmInfo.h"
20 #include "llvm/MC/MCParser/MCAsmLexer.h"
21 #include "llvm/Support/Compiler.h"
22 #include "llvm/Support/SMLoc.h"
23 #include "llvm/Support/SaveAndRestore.h"
24 #include <cassert>
25 #include <cctype>
26 #include <cstdio>
27 #include <cstring>
28 #include <string>
29 #include <tuple>
30 #include <utility>
31 
32 using namespace llvm;
33 
34 AsmLexer::AsmLexer(const MCAsmInfo &MAI) : MAI(MAI) {
35   AllowAtInIdentifier = !StringRef(MAI.getCommentString()).startswith("@");
36 }
37 
38 AsmLexer::~AsmLexer() = default;
39 
40 void AsmLexer::setBuffer(StringRef Buf, const char *ptr,
41                          bool EndStatementAtEOF) {
42   CurBuf = Buf;
43 
44   if (ptr)
45     CurPtr = ptr;
46   else
47     CurPtr = CurBuf.begin();
48 
49   TokStart = nullptr;
50   this->EndStatementAtEOF = EndStatementAtEOF;
51 }
52 
53 /// ReturnError - Set the error to the specified string at the specified
54 /// location.  This is defined to always return AsmToken::Error.
55 AsmToken AsmLexer::ReturnError(const char *Loc, const std::string &Msg) {
56   SetError(SMLoc::getFromPointer(Loc), Msg);
57 
58   return AsmToken(AsmToken::Error, StringRef(Loc, CurPtr - Loc));
59 }
60 
61 int AsmLexer::getNextChar() {
62   if (CurPtr == CurBuf.end())
63     return EOF;
64   return (unsigned char)*CurPtr++;
65 }
66 
67 /// The leading integral digit sequence and dot should have already been
68 /// consumed, some or all of the fractional digit sequence *can* have been
69 /// consumed.
70 AsmToken AsmLexer::LexFloatLiteral() {
71   // Skip the fractional digit sequence.
72   while (isDigit(*CurPtr))
73     ++CurPtr;
74 
75   if (*CurPtr == '-' || *CurPtr == '+')
76     return ReturnError(CurPtr, "Invalid sign in float literal");
77 
78   // Check for exponent
79   if ((*CurPtr == 'e' || *CurPtr == 'E')) {
80     ++CurPtr;
81 
82     if (*CurPtr == '-' || *CurPtr == '+')
83       ++CurPtr;
84 
85     while (isDigit(*CurPtr))
86       ++CurPtr;
87   }
88 
89   return AsmToken(AsmToken::Real,
90                   StringRef(TokStart, CurPtr - TokStart));
91 }
92 
93 /// LexHexFloatLiteral matches essentially (.[0-9a-fA-F]*)?[pP][+-]?[0-9a-fA-F]+
94 /// while making sure there are enough actual digits around for the constant to
95 /// be valid.
96 ///
97 /// The leading "0x[0-9a-fA-F]*" (i.e. integer part) has already been consumed
98 /// before we get here.
99 AsmToken AsmLexer::LexHexFloatLiteral(bool NoIntDigits) {
100   assert((*CurPtr == 'p' || *CurPtr == 'P' || *CurPtr == '.') &&
101          "unexpected parse state in floating hex");
102   bool NoFracDigits = true;
103 
104   // Skip the fractional part if there is one
105   if (*CurPtr == '.') {
106     ++CurPtr;
107 
108     const char *FracStart = CurPtr;
109     while (isHexDigit(*CurPtr))
110       ++CurPtr;
111 
112     NoFracDigits = CurPtr == FracStart;
113   }
114 
115   if (NoIntDigits && NoFracDigits)
116     return ReturnError(TokStart, "invalid hexadecimal floating-point constant: "
117                                  "expected at least one significand digit");
118 
119   // Make sure we do have some kind of proper exponent part
120   if (*CurPtr != 'p' && *CurPtr != 'P')
121     return ReturnError(TokStart, "invalid hexadecimal floating-point constant: "
122                                  "expected exponent part 'p'");
123   ++CurPtr;
124 
125   if (*CurPtr == '+' || *CurPtr == '-')
126     ++CurPtr;
127 
128   // N.b. exponent digits are *not* hex
129   const char *ExpStart = CurPtr;
130   while (isDigit(*CurPtr))
131     ++CurPtr;
132 
133   if (CurPtr == ExpStart)
134     return ReturnError(TokStart, "invalid hexadecimal floating-point constant: "
135                                  "expected at least one exponent digit");
136 
137   return AsmToken(AsmToken::Real, StringRef(TokStart, CurPtr - TokStart));
138 }
139 
140 /// LexIdentifier: [a-zA-Z_.][a-zA-Z0-9_$.@?]*
141 static bool IsIdentifierChar(char c, bool AllowAt) {
142   return isAlnum(c) || c == '_' || c == '$' || c == '.' ||
143          (c == '@' && AllowAt) || c == '?';
144 }
145 
146 AsmToken AsmLexer::LexIdentifier() {
147   // Check for floating point literals.
148   if (CurPtr[-1] == '.' && isDigit(*CurPtr)) {
149     // Disambiguate a .1243foo identifier from a floating literal.
150     while (isDigit(*CurPtr))
151       ++CurPtr;
152 
153     if (!IsIdentifierChar(*CurPtr, AllowAtInIdentifier) ||
154         *CurPtr == 'e' || *CurPtr == 'E')
155       return LexFloatLiteral();
156   }
157 
158   while (IsIdentifierChar(*CurPtr, AllowAtInIdentifier))
159     ++CurPtr;
160 
161   // Handle . as a special case.
162   if (CurPtr == TokStart+1 && TokStart[0] == '.')
163     return AsmToken(AsmToken::Dot, StringRef(TokStart, 1));
164 
165   return AsmToken(AsmToken::Identifier, StringRef(TokStart, CurPtr - TokStart));
166 }
167 
168 /// LexSlash: Slash: /
169 ///           C-Style Comment: /* ... */
170 AsmToken AsmLexer::LexSlash() {
171   switch (*CurPtr) {
172   case '*':
173     IsAtStartOfStatement = false;
174     break; // C style comment.
175   case '/':
176     ++CurPtr;
177     return LexLineComment();
178   default:
179     IsAtStartOfStatement = false;
180     return AsmToken(AsmToken::Slash, StringRef(TokStart, 1));
181   }
182 
183   // C Style comment.
184   ++CurPtr;  // skip the star.
185   const char *CommentTextStart = CurPtr;
186   while (CurPtr != CurBuf.end()) {
187     switch (*CurPtr++) {
188     case '*':
189       // End of the comment?
190       if (*CurPtr != '/')
191         break;
192       // If we have a CommentConsumer, notify it about the comment.
193       if (CommentConsumer) {
194         CommentConsumer->HandleComment(
195             SMLoc::getFromPointer(CommentTextStart),
196             StringRef(CommentTextStart, CurPtr - 1 - CommentTextStart));
197       }
198       ++CurPtr;   // End the */.
199       return AsmToken(AsmToken::Comment,
200                       StringRef(TokStart, CurPtr - TokStart));
201     }
202   }
203   return ReturnError(TokStart, "unterminated comment");
204 }
205 
206 /// LexLineComment: Comment: #[^\n]*
207 ///                        : //[^\n]*
208 AsmToken AsmLexer::LexLineComment() {
209   // Mark This as an end of statement with a body of the
210   // comment. While it would be nicer to leave this two tokens,
211   // backwards compatability with TargetParsers makes keeping this in this form
212   // better.
213   const char *CommentTextStart = CurPtr;
214   int CurChar = getNextChar();
215   while (CurChar != '\n' && CurChar != '\r' && CurChar != EOF)
216     CurChar = getNextChar();
217   if (CurChar == '\r' && CurPtr != CurBuf.end() && *CurPtr == '\n')
218     ++CurPtr;
219 
220   // If we have a CommentConsumer, notify it about the comment.
221   if (CommentConsumer) {
222     CommentConsumer->HandleComment(
223         SMLoc::getFromPointer(CommentTextStart),
224         StringRef(CommentTextStart, CurPtr - 1 - CommentTextStart));
225   }
226 
227   IsAtStartOfLine = true;
228   // This is a whole line comment. leave newline
229   if (IsAtStartOfStatement)
230     return AsmToken(AsmToken::EndOfStatement,
231                     StringRef(TokStart, CurPtr - TokStart));
232   IsAtStartOfStatement = true;
233 
234   return AsmToken(AsmToken::EndOfStatement,
235                   StringRef(TokStart, CurPtr - 1 - TokStart));
236 }
237 
238 static void SkipIgnoredIntegerSuffix(const char *&CurPtr) {
239   // Skip ULL, UL, U, L and LL suffices.
240   if (CurPtr[0] == 'U')
241     ++CurPtr;
242   if (CurPtr[0] == 'L')
243     ++CurPtr;
244   if (CurPtr[0] == 'L')
245     ++CurPtr;
246 }
247 
248 // Look ahead to search for first non-hex digit, if it's [hH], then we treat the
249 // integer as a hexadecimal, possibly with leading zeroes.
250 static unsigned doHexLookAhead(const char *&CurPtr, unsigned DefaultRadix,
251                                bool LexHex) {
252   const char *FirstNonDec = nullptr;
253   const char *LookAhead = CurPtr;
254   while (true) {
255     if (isDigit(*LookAhead)) {
256       ++LookAhead;
257     } else {
258       if (!FirstNonDec)
259         FirstNonDec = LookAhead;
260 
261       // Keep going if we are looking for a 'h' suffix.
262       if (LexHex && isHexDigit(*LookAhead))
263         ++LookAhead;
264       else
265         break;
266     }
267   }
268   bool isHex = LexHex && (*LookAhead == 'h' || *LookAhead == 'H');
269   CurPtr = isHex || !FirstNonDec ? LookAhead : FirstNonDec;
270   if (isHex)
271     return 16;
272   return DefaultRadix;
273 }
274 
275 static const char *findLastDigit(const char *CurPtr, unsigned DefaultRadix) {
276   while (hexDigitValue(*CurPtr) < DefaultRadix) {
277     ++CurPtr;
278   }
279   return CurPtr;
280 }
281 
282 static AsmToken intToken(StringRef Ref, APInt &Value) {
283   if (Value.isIntN(64))
284     return AsmToken(AsmToken::Integer, Ref, Value);
285   return AsmToken(AsmToken::BigNum, Ref, Value);
286 }
287 
288 static std::string radixName(unsigned Radix) {
289   switch (Radix) {
290   case 2:
291     return "binary";
292   case 8:
293     return "octal";
294   case 10:
295     return "decimal";
296   case 16:
297     return "hexadecimal";
298   default:
299     return "base-" + std::to_string(Radix);
300   }
301 }
302 
303 /// LexDigit: First character is [0-9].
304 ///   Local Label: [0-9][:]
305 ///   Forward/Backward Label: [0-9][fb]
306 ///   Binary integer: 0b[01]+
307 ///   Octal integer: 0[0-7]+
308 ///   Hex integer: 0x[0-9a-fA-F]+ or [0x]?[0-9][0-9a-fA-F]*[hH]
309 ///   Decimal integer: [1-9][0-9]*
310 AsmToken AsmLexer::LexDigit() {
311   // MASM-flavor binary integer: [01]+[yY] (if DefaultRadix < 16, [bByY])
312   // MASM-flavor octal integer: [0-7]+[oOqQ]
313   // MASM-flavor decimal integer: [0-9]+[tT] (if DefaultRadix < 16, [dDtT])
314   // MASM-flavor hexadecimal integer: [0-9][0-9a-fA-F]*[hH]
315   if (LexMasmIntegers && isdigit(CurPtr[-1])) {
316     const char *FirstNonBinary =
317         (CurPtr[-1] != '0' && CurPtr[-1] != '1') ? CurPtr - 1 : nullptr;
318     const char *FirstNonDecimal =
319         (CurPtr[-1] < '0' || CurPtr[-1] > '9') ? CurPtr - 1 : nullptr;
320     const char *OldCurPtr = CurPtr;
321     while (isHexDigit(*CurPtr)) {
322       switch (*CurPtr) {
323       default:
324         if (!FirstNonDecimal) {
325           FirstNonDecimal = CurPtr;
326         }
327         LLVM_FALLTHROUGH;
328       case '9':
329       case '8':
330       case '7':
331       case '6':
332       case '5':
333       case '4':
334       case '3':
335       case '2':
336         if (!FirstNonBinary) {
337           FirstNonBinary = CurPtr;
338         }
339         break;
340       case '1':
341       case '0':
342         break;
343       }
344       ++CurPtr;
345     }
346     if (*CurPtr == '.') {
347       // MASM float literals (other than hex floats) always contain a ".", and
348       // are always written in decimal.
349       ++CurPtr;
350       return LexFloatLiteral();
351     }
352 
353     if (LexMasmHexFloats && (*CurPtr == 'r' || *CurPtr == 'R')) {
354       ++CurPtr;
355       return AsmToken(AsmToken::Real, StringRef(TokStart, CurPtr - TokStart));
356     }
357 
358     unsigned Radix = 0;
359     if (*CurPtr == 'h' || *CurPtr == 'H') {
360       // hexadecimal number
361       ++CurPtr;
362       Radix = 16;
363     } else if (*CurPtr == 't' || *CurPtr == 'T') {
364       // decimal number
365       ++CurPtr;
366       Radix = 10;
367     } else if (*CurPtr == 'o' || *CurPtr == 'O' || *CurPtr == 'q' ||
368                *CurPtr == 'Q') {
369       // octal number
370       ++CurPtr;
371       Radix = 8;
372     } else if (*CurPtr == 'y' || *CurPtr == 'Y') {
373       // binary number
374       ++CurPtr;
375       Radix = 2;
376     } else if (FirstNonDecimal && FirstNonDecimal + 1 == CurPtr &&
377                DefaultRadix < 14 &&
378                (*FirstNonDecimal == 'd' || *FirstNonDecimal == 'D')) {
379       Radix = 10;
380     } else if (FirstNonBinary && FirstNonBinary + 1 == CurPtr &&
381                DefaultRadix < 12 &&
382                (*FirstNonBinary == 'b' || *FirstNonBinary == 'B')) {
383       Radix = 2;
384     }
385 
386     if (Radix) {
387       StringRef Result(TokStart, CurPtr - TokStart);
388       APInt Value(128, 0, true);
389 
390       if (Result.drop_back().getAsInteger(Radix, Value))
391         return ReturnError(TokStart, "invalid " + radixName(Radix) + " number");
392 
393       // MSVC accepts and ignores type suffices on integer literals.
394       SkipIgnoredIntegerSuffix(CurPtr);
395 
396       return intToken(Result, Value);
397     }
398 
399     // default-radix integers, or floating point numbers, fall through
400     CurPtr = OldCurPtr;
401   }
402 
403   // MASM default-radix integers: [0-9a-fA-F]+
404   // (All other integer literals have a radix specifier.)
405   if (LexMasmIntegers && UseMasmDefaultRadix) {
406     CurPtr = findLastDigit(CurPtr, 16);
407     StringRef Result(TokStart, CurPtr - TokStart);
408 
409     APInt Value(128, 0, true);
410     if (Result.getAsInteger(DefaultRadix, Value)) {
411       return ReturnError(TokStart,
412                          "invalid " + radixName(DefaultRadix) + " number");
413     }
414 
415     return intToken(Result, Value);
416   }
417 
418   // Decimal integer: [1-9][0-9]*
419   if (CurPtr[-1] != '0' || CurPtr[0] == '.') {
420     unsigned Radix = doHexLookAhead(CurPtr, 10, LexMasmIntegers);
421     bool isHex = Radix == 16;
422     // Check for floating point literals.
423     if (!isHex && (*CurPtr == '.' || *CurPtr == 'e' || *CurPtr == 'E')) {
424       if (*CurPtr == '.')
425         ++CurPtr;
426       return LexFloatLiteral();
427     }
428 
429     StringRef Result(TokStart, CurPtr - TokStart);
430 
431     APInt Value(128, 0, true);
432     if (Result.getAsInteger(Radix, Value)) {
433       return ReturnError(TokStart, "invalid " + radixName(Radix) + " number");
434     }
435 
436     // The darwin/x86 (and x86-64) assembler accepts and ignores type
437     // suffices on integer literals.
438     SkipIgnoredIntegerSuffix(CurPtr);
439 
440     return intToken(Result, Value);
441   }
442 
443   if (!LexMasmIntegers && ((*CurPtr == 'b') || (*CurPtr == 'B'))) {
444     ++CurPtr;
445     // See if we actually have "0b" as part of something like "jmp 0b\n"
446     if (!isDigit(CurPtr[0])) {
447       --CurPtr;
448       StringRef Result(TokStart, CurPtr - TokStart);
449       return AsmToken(AsmToken::Integer, Result, 0);
450     }
451     const char *NumStart = CurPtr;
452     while (CurPtr[0] == '0' || CurPtr[0] == '1')
453       ++CurPtr;
454 
455     // Requires at least one binary digit.
456     if (CurPtr == NumStart)
457       return ReturnError(TokStart, "invalid binary number");
458 
459     StringRef Result(TokStart, CurPtr - TokStart);
460 
461     APInt Value(128, 0, true);
462     if (Result.substr(2).getAsInteger(2, Value))
463       return ReturnError(TokStart, "invalid binary number");
464 
465     // The darwin/x86 (and x86-64) assembler accepts and ignores ULL and LL
466     // suffixes on integer literals.
467     SkipIgnoredIntegerSuffix(CurPtr);
468 
469     return intToken(Result, Value);
470   }
471 
472   if ((*CurPtr == 'x') || (*CurPtr == 'X')) {
473     ++CurPtr;
474     const char *NumStart = CurPtr;
475     while (isHexDigit(CurPtr[0]))
476       ++CurPtr;
477 
478     // "0x.0p0" is valid, and "0x0p0" (but not "0xp0" for example, which will be
479     // diagnosed by LexHexFloatLiteral).
480     if (CurPtr[0] == '.' || CurPtr[0] == 'p' || CurPtr[0] == 'P')
481       return LexHexFloatLiteral(NumStart == CurPtr);
482 
483     // Otherwise requires at least one hex digit.
484     if (CurPtr == NumStart)
485       return ReturnError(CurPtr-2, "invalid hexadecimal number");
486 
487     APInt Result(128, 0);
488     if (StringRef(TokStart, CurPtr - TokStart).getAsInteger(0, Result))
489       return ReturnError(TokStart, "invalid hexadecimal number");
490 
491     // Consume the optional [hH].
492     if (LexMasmIntegers && (*CurPtr == 'h' || *CurPtr == 'H'))
493       ++CurPtr;
494 
495     // The darwin/x86 (and x86-64) assembler accepts and ignores ULL and LL
496     // suffixes on integer literals.
497     SkipIgnoredIntegerSuffix(CurPtr);
498 
499     return intToken(StringRef(TokStart, CurPtr - TokStart), Result);
500   }
501 
502   // Either octal or hexadecimal.
503   APInt Value(128, 0, true);
504   unsigned Radix = doHexLookAhead(CurPtr, 8, LexMasmIntegers);
505   StringRef Result(TokStart, CurPtr - TokStart);
506   if (Result.getAsInteger(Radix, Value))
507     return ReturnError(TokStart, "invalid " + radixName(Radix) + " number");
508 
509   // Consume the [hH].
510   if (Radix == 16)
511     ++CurPtr;
512 
513   // The darwin/x86 (and x86-64) assembler accepts and ignores ULL and LL
514   // suffixes on integer literals.
515   SkipIgnoredIntegerSuffix(CurPtr);
516 
517   return intToken(Result, Value);
518 }
519 
520 /// LexSingleQuote: Integer: 'b'
521 AsmToken AsmLexer::LexSingleQuote() {
522   int CurChar = getNextChar();
523 
524   if (CurChar == '\\')
525     CurChar = getNextChar();
526 
527   if (CurChar == EOF)
528     return ReturnError(TokStart, "unterminated single quote");
529 
530   CurChar = getNextChar();
531 
532   if (CurChar != '\'')
533     return ReturnError(TokStart, "single quote way too long");
534 
535   // The idea here being that 'c' is basically just an integral
536   // constant.
537   StringRef Res = StringRef(TokStart,CurPtr - TokStart);
538   long long Value;
539 
540   if (Res.startswith("\'\\")) {
541     char theChar = Res[2];
542     switch (theChar) {
543       default: Value = theChar; break;
544       case '\'': Value = '\''; break;
545       case 't': Value = '\t'; break;
546       case 'n': Value = '\n'; break;
547       case 'b': Value = '\b'; break;
548     }
549   } else
550     Value = TokStart[1];
551 
552   return AsmToken(AsmToken::Integer, Res, Value);
553 }
554 
555 /// LexQuote: String: "..."
556 AsmToken AsmLexer::LexQuote() {
557   int CurChar = getNextChar();
558   // TODO: does gas allow multiline string constants?
559   while (CurChar != '"') {
560     if (CurChar == '\\') {
561       // Allow \", etc.
562       CurChar = getNextChar();
563     }
564 
565     if (CurChar == EOF)
566       return ReturnError(TokStart, "unterminated string constant");
567 
568     CurChar = getNextChar();
569   }
570 
571   return AsmToken(AsmToken::String, StringRef(TokStart, CurPtr - TokStart));
572 }
573 
574 StringRef AsmLexer::LexUntilEndOfStatement() {
575   TokStart = CurPtr;
576 
577   while (!isAtStartOfComment(CurPtr) &&     // Start of line comment.
578          !isAtStatementSeparator(CurPtr) && // End of statement marker.
579          *CurPtr != '\n' && *CurPtr != '\r' && CurPtr != CurBuf.end()) {
580     ++CurPtr;
581   }
582   return StringRef(TokStart, CurPtr-TokStart);
583 }
584 
585 StringRef AsmLexer::LexUntilEndOfLine() {
586   TokStart = CurPtr;
587 
588   while (*CurPtr != '\n' && *CurPtr != '\r' && CurPtr != CurBuf.end()) {
589     ++CurPtr;
590   }
591   return StringRef(TokStart, CurPtr-TokStart);
592 }
593 
594 size_t AsmLexer::peekTokens(MutableArrayRef<AsmToken> Buf,
595                             bool ShouldSkipSpace) {
596   SaveAndRestore<const char *> SavedTokenStart(TokStart);
597   SaveAndRestore<const char *> SavedCurPtr(CurPtr);
598   SaveAndRestore<bool> SavedAtStartOfLine(IsAtStartOfLine);
599   SaveAndRestore<bool> SavedAtStartOfStatement(IsAtStartOfStatement);
600   SaveAndRestore<bool> SavedSkipSpace(SkipSpace, ShouldSkipSpace);
601   SaveAndRestore<bool> SavedIsPeeking(IsPeeking, true);
602   std::string SavedErr = getErr();
603   SMLoc SavedErrLoc = getErrLoc();
604 
605   size_t ReadCount;
606   for (ReadCount = 0; ReadCount < Buf.size(); ++ReadCount) {
607     AsmToken Token = LexToken();
608 
609     Buf[ReadCount] = Token;
610 
611     if (Token.is(AsmToken::Eof))
612       break;
613   }
614 
615   SetError(SavedErrLoc, SavedErr);
616   return ReadCount;
617 }
618 
619 bool AsmLexer::isAtStartOfComment(const char *Ptr) {
620   StringRef CommentString = MAI.getCommentString();
621 
622   if (CommentString.size() == 1)
623     return CommentString[0] == Ptr[0];
624 
625   // Allow # preprocessor commments also be counted as comments for "##" cases
626   if (CommentString[1] == '#')
627     return CommentString[0] == Ptr[0];
628 
629   return strncmp(Ptr, CommentString.data(), CommentString.size()) == 0;
630 }
631 
632 bool AsmLexer::isAtStatementSeparator(const char *Ptr) {
633   return strncmp(Ptr, MAI.getSeparatorString(),
634                  strlen(MAI.getSeparatorString())) == 0;
635 }
636 
637 AsmToken AsmLexer::LexToken() {
638   TokStart = CurPtr;
639   // This always consumes at least one character.
640   int CurChar = getNextChar();
641 
642   if (!IsPeeking && CurChar == '#' && IsAtStartOfStatement) {
643     // If this starts with a '#', this may be a cpp
644     // hash directive and otherwise a line comment.
645     AsmToken TokenBuf[2];
646     MutableArrayRef<AsmToken> Buf(TokenBuf, 2);
647     size_t num = peekTokens(Buf, true);
648     // There cannot be a space preceding this
649     if (IsAtStartOfLine && num == 2 && TokenBuf[0].is(AsmToken::Integer) &&
650         TokenBuf[1].is(AsmToken::String)) {
651       CurPtr = TokStart; // reset curPtr;
652       StringRef s = LexUntilEndOfLine();
653       UnLex(TokenBuf[1]);
654       UnLex(TokenBuf[0]);
655       return AsmToken(AsmToken::HashDirective, s);
656     }
657     return LexLineComment();
658   }
659 
660   if (isAtStartOfComment(TokStart))
661     return LexLineComment();
662 
663   if (isAtStatementSeparator(TokStart)) {
664     CurPtr += strlen(MAI.getSeparatorString()) - 1;
665     IsAtStartOfLine = true;
666     IsAtStartOfStatement = true;
667     return AsmToken(AsmToken::EndOfStatement,
668                     StringRef(TokStart, strlen(MAI.getSeparatorString())));
669   }
670 
671   // If we're missing a newline at EOF, make sure we still get an
672   // EndOfStatement token before the Eof token.
673   if (CurChar == EOF && !IsAtStartOfStatement && EndStatementAtEOF) {
674     IsAtStartOfLine = true;
675     IsAtStartOfStatement = true;
676     return AsmToken(AsmToken::EndOfStatement, StringRef(TokStart, 1));
677   }
678   IsAtStartOfLine = false;
679   bool OldIsAtStartOfStatement = IsAtStartOfStatement;
680   IsAtStartOfStatement = false;
681   switch (CurChar) {
682   default:
683     if (MAI.doesAllowSymbolAtNameStart()) {
684       // Handle Microsoft-style identifier: [a-zA-Z_$.@?][a-zA-Z0-9_$.@?]*
685       if (!isDigit(CurChar) &&
686           IsIdentifierChar(CurChar, MAI.doesAllowAtInName()))
687         return LexIdentifier();
688     } else {
689       // Handle identifier: [a-zA-Z_.][a-zA-Z0-9_$.@]*
690       if (isalpha(CurChar) || CurChar == '_' || CurChar == '.')
691         return LexIdentifier();
692     }
693 
694     // Unknown character, emit an error.
695     return ReturnError(TokStart, "invalid character in input");
696   case EOF:
697     if (EndStatementAtEOF) {
698       IsAtStartOfLine = true;
699       IsAtStartOfStatement = true;
700     }
701     return AsmToken(AsmToken::Eof, StringRef(TokStart, 0));
702   case 0:
703   case ' ':
704   case '\t':
705     IsAtStartOfStatement = OldIsAtStartOfStatement;
706     while (*CurPtr == ' ' || *CurPtr == '\t')
707       CurPtr++;
708     if (SkipSpace)
709       return LexToken(); // Ignore whitespace.
710     else
711       return AsmToken(AsmToken::Space, StringRef(TokStart, CurPtr - TokStart));
712   case '\r': {
713     IsAtStartOfLine = true;
714     IsAtStartOfStatement = true;
715     // If this is a CR followed by LF, treat that as one token.
716     if (CurPtr != CurBuf.end() && *CurPtr == '\n')
717       ++CurPtr;
718     return AsmToken(AsmToken::EndOfStatement,
719                     StringRef(TokStart, CurPtr - TokStart));
720   }
721   case '\n':
722     IsAtStartOfLine = true;
723     IsAtStartOfStatement = true;
724     return AsmToken(AsmToken::EndOfStatement, StringRef(TokStart, 1));
725   case ':': return AsmToken(AsmToken::Colon, StringRef(TokStart, 1));
726   case '+': return AsmToken(AsmToken::Plus, StringRef(TokStart, 1));
727   case '~': return AsmToken(AsmToken::Tilde, StringRef(TokStart, 1));
728   case '(': return AsmToken(AsmToken::LParen, StringRef(TokStart, 1));
729   case ')': return AsmToken(AsmToken::RParen, StringRef(TokStart, 1));
730   case '[': return AsmToken(AsmToken::LBrac, StringRef(TokStart, 1));
731   case ']': return AsmToken(AsmToken::RBrac, StringRef(TokStart, 1));
732   case '{': return AsmToken(AsmToken::LCurly, StringRef(TokStart, 1));
733   case '}': return AsmToken(AsmToken::RCurly, StringRef(TokStart, 1));
734   case '*': return AsmToken(AsmToken::Star, StringRef(TokStart, 1));
735   case ',': return AsmToken(AsmToken::Comma, StringRef(TokStart, 1));
736   case '$': return AsmToken(AsmToken::Dollar, StringRef(TokStart, 1));
737   case '@': return AsmToken(AsmToken::At, StringRef(TokStart, 1));
738   case '\\': return AsmToken(AsmToken::BackSlash, StringRef(TokStart, 1));
739   case '=':
740     if (*CurPtr == '=') {
741       ++CurPtr;
742       return AsmToken(AsmToken::EqualEqual, StringRef(TokStart, 2));
743     }
744     return AsmToken(AsmToken::Equal, StringRef(TokStart, 1));
745   case '-':
746     if (*CurPtr == '>') {
747       ++CurPtr;
748       return AsmToken(AsmToken::MinusGreater, StringRef(TokStart, 2));
749     }
750     return AsmToken(AsmToken::Minus, StringRef(TokStart, 1));
751   case '|':
752     if (*CurPtr == '|') {
753       ++CurPtr;
754       return AsmToken(AsmToken::PipePipe, StringRef(TokStart, 2));
755     }
756     return AsmToken(AsmToken::Pipe, StringRef(TokStart, 1));
757   case '^': return AsmToken(AsmToken::Caret, StringRef(TokStart, 1));
758   case '&':
759     if (*CurPtr == '&') {
760       ++CurPtr;
761       return AsmToken(AsmToken::AmpAmp, StringRef(TokStart, 2));
762     }
763     return AsmToken(AsmToken::Amp, StringRef(TokStart, 1));
764   case '!':
765     if (*CurPtr == '=') {
766       ++CurPtr;
767       return AsmToken(AsmToken::ExclaimEqual, StringRef(TokStart, 2));
768     }
769     return AsmToken(AsmToken::Exclaim, StringRef(TokStart, 1));
770   case '%':
771     if (MAI.hasMipsExpressions()) {
772       AsmToken::TokenKind Operator;
773       unsigned OperatorLength;
774 
775       std::tie(Operator, OperatorLength) =
776           StringSwitch<std::pair<AsmToken::TokenKind, unsigned>>(
777               StringRef(CurPtr))
778               .StartsWith("call16", {AsmToken::PercentCall16, 7})
779               .StartsWith("call_hi", {AsmToken::PercentCall_Hi, 8})
780               .StartsWith("call_lo", {AsmToken::PercentCall_Lo, 8})
781               .StartsWith("dtprel_hi", {AsmToken::PercentDtprel_Hi, 10})
782               .StartsWith("dtprel_lo", {AsmToken::PercentDtprel_Lo, 10})
783               .StartsWith("got_disp", {AsmToken::PercentGot_Disp, 9})
784               .StartsWith("got_hi", {AsmToken::PercentGot_Hi, 7})
785               .StartsWith("got_lo", {AsmToken::PercentGot_Lo, 7})
786               .StartsWith("got_ofst", {AsmToken::PercentGot_Ofst, 9})
787               .StartsWith("got_page", {AsmToken::PercentGot_Page, 9})
788               .StartsWith("gottprel", {AsmToken::PercentGottprel, 9})
789               .StartsWith("got", {AsmToken::PercentGot, 4})
790               .StartsWith("gp_rel", {AsmToken::PercentGp_Rel, 7})
791               .StartsWith("higher", {AsmToken::PercentHigher, 7})
792               .StartsWith("highest", {AsmToken::PercentHighest, 8})
793               .StartsWith("hi", {AsmToken::PercentHi, 3})
794               .StartsWith("lo", {AsmToken::PercentLo, 3})
795               .StartsWith("neg", {AsmToken::PercentNeg, 4})
796               .StartsWith("pcrel_hi", {AsmToken::PercentPcrel_Hi, 9})
797               .StartsWith("pcrel_lo", {AsmToken::PercentPcrel_Lo, 9})
798               .StartsWith("tlsgd", {AsmToken::PercentTlsgd, 6})
799               .StartsWith("tlsldm", {AsmToken::PercentTlsldm, 7})
800               .StartsWith("tprel_hi", {AsmToken::PercentTprel_Hi, 9})
801               .StartsWith("tprel_lo", {AsmToken::PercentTprel_Lo, 9})
802               .Default({AsmToken::Percent, 1});
803 
804       if (Operator != AsmToken::Percent) {
805         CurPtr += OperatorLength - 1;
806         return AsmToken(Operator, StringRef(TokStart, OperatorLength));
807       }
808     }
809     return AsmToken(AsmToken::Percent, StringRef(TokStart, 1));
810   case '/':
811     IsAtStartOfStatement = OldIsAtStartOfStatement;
812     return LexSlash();
813   case '#': return AsmToken(AsmToken::Hash, StringRef(TokStart, 1));
814   case '\'': return LexSingleQuote();
815   case '"': return LexQuote();
816   case '0': case '1': case '2': case '3': case '4':
817   case '5': case '6': case '7': case '8': case '9':
818     return LexDigit();
819   case '<':
820     switch (*CurPtr) {
821     case '<':
822       ++CurPtr;
823       return AsmToken(AsmToken::LessLess, StringRef(TokStart, 2));
824     case '=':
825       ++CurPtr;
826       return AsmToken(AsmToken::LessEqual, StringRef(TokStart, 2));
827     case '>':
828       ++CurPtr;
829       return AsmToken(AsmToken::LessGreater, StringRef(TokStart, 2));
830     default:
831       return AsmToken(AsmToken::Less, StringRef(TokStart, 1));
832     }
833   case '>':
834     switch (*CurPtr) {
835     case '>':
836       ++CurPtr;
837       return AsmToken(AsmToken::GreaterGreater, StringRef(TokStart, 2));
838     case '=':
839       ++CurPtr;
840       return AsmToken(AsmToken::GreaterEqual, StringRef(TokStart, 2));
841     default:
842       return AsmToken(AsmToken::Greater, StringRef(TokStart, 1));
843     }
844 
845   // TODO: Quoted identifiers (objc methods etc)
846   // local labels: [0-9][:]
847   // Forward/backward labels: [0-9][fb]
848   // Integers, fp constants, character constants.
849   }
850 }
851