1 //===- AsmLexer.cpp - Lexer for Assembly Files ----------------------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This class implements the lexer for assembly files.
10 //
11 //===----------------------------------------------------------------------===//
12 
13 #include "llvm/MC/MCParser/AsmLexer.h"
14 #include "llvm/ADT/APInt.h"
15 #include "llvm/ADT/ArrayRef.h"
16 #include "llvm/ADT/StringExtras.h"
17 #include "llvm/ADT/StringRef.h"
18 #include "llvm/ADT/StringSwitch.h"
19 #include "llvm/MC/MCAsmInfo.h"
20 #include "llvm/MC/MCParser/MCAsmLexer.h"
21 #include "llvm/Support/Compiler.h"
22 #include "llvm/Support/SMLoc.h"
23 #include "llvm/Support/SaveAndRestore.h"
24 #include <cassert>
25 #include <cctype>
26 #include <cstdio>
27 #include <cstring>
28 #include <string>
29 #include <tuple>
30 #include <utility>
31 
32 using namespace llvm;
33 
34 AsmLexer::AsmLexer(const MCAsmInfo &MAI) : MAI(MAI) {
35   AllowAtInIdentifier = !StringRef(MAI.getCommentString()).startswith("@");
36   LexMotorolaIntegers = MAI.shouldUseMotorolaIntegers();
37 }
38 
39 AsmLexer::~AsmLexer() = default;
40 
41 void AsmLexer::setBuffer(StringRef Buf, const char *ptr,
42                          bool EndStatementAtEOF) {
43   CurBuf = Buf;
44 
45   if (ptr)
46     CurPtr = ptr;
47   else
48     CurPtr = CurBuf.begin();
49 
50   TokStart = nullptr;
51   this->EndStatementAtEOF = EndStatementAtEOF;
52 }
53 
54 /// ReturnError - Set the error to the specified string at the specified
55 /// location.  This is defined to always return AsmToken::Error.
56 AsmToken AsmLexer::ReturnError(const char *Loc, const std::string &Msg) {
57   SetError(SMLoc::getFromPointer(Loc), Msg);
58 
59   return AsmToken(AsmToken::Error, StringRef(Loc, CurPtr - Loc));
60 }
61 
62 int AsmLexer::getNextChar() {
63   if (CurPtr == CurBuf.end())
64     return EOF;
65   return (unsigned char)*CurPtr++;
66 }
67 
68 int AsmLexer::peekNextChar() {
69   if (CurPtr == CurBuf.end())
70     return EOF;
71   return (unsigned char)*CurPtr;
72 }
73 
74 /// The leading integral digit sequence and dot should have already been
75 /// consumed, some or all of the fractional digit sequence *can* have been
76 /// consumed.
77 AsmToken AsmLexer::LexFloatLiteral() {
78   // Skip the fractional digit sequence.
79   while (isDigit(*CurPtr))
80     ++CurPtr;
81 
82   if (*CurPtr == '-' || *CurPtr == '+')
83     return ReturnError(CurPtr, "Invalid sign in float literal");
84 
85   // Check for exponent
86   if ((*CurPtr == 'e' || *CurPtr == 'E')) {
87     ++CurPtr;
88 
89     if (*CurPtr == '-' || *CurPtr == '+')
90       ++CurPtr;
91 
92     while (isDigit(*CurPtr))
93       ++CurPtr;
94   }
95 
96   return AsmToken(AsmToken::Real,
97                   StringRef(TokStart, CurPtr - TokStart));
98 }
99 
100 /// LexHexFloatLiteral matches essentially (.[0-9a-fA-F]*)?[pP][+-]?[0-9a-fA-F]+
101 /// while making sure there are enough actual digits around for the constant to
102 /// be valid.
103 ///
104 /// The leading "0x[0-9a-fA-F]*" (i.e. integer part) has already been consumed
105 /// before we get here.
106 AsmToken AsmLexer::LexHexFloatLiteral(bool NoIntDigits) {
107   assert((*CurPtr == 'p' || *CurPtr == 'P' || *CurPtr == '.') &&
108          "unexpected parse state in floating hex");
109   bool NoFracDigits = true;
110 
111   // Skip the fractional part if there is one
112   if (*CurPtr == '.') {
113     ++CurPtr;
114 
115     const char *FracStart = CurPtr;
116     while (isHexDigit(*CurPtr))
117       ++CurPtr;
118 
119     NoFracDigits = CurPtr == FracStart;
120   }
121 
122   if (NoIntDigits && NoFracDigits)
123     return ReturnError(TokStart, "invalid hexadecimal floating-point constant: "
124                                  "expected at least one significand digit");
125 
126   // Make sure we do have some kind of proper exponent part
127   if (*CurPtr != 'p' && *CurPtr != 'P')
128     return ReturnError(TokStart, "invalid hexadecimal floating-point constant: "
129                                  "expected exponent part 'p'");
130   ++CurPtr;
131 
132   if (*CurPtr == '+' || *CurPtr == '-')
133     ++CurPtr;
134 
135   // N.b. exponent digits are *not* hex
136   const char *ExpStart = CurPtr;
137   while (isDigit(*CurPtr))
138     ++CurPtr;
139 
140   if (CurPtr == ExpStart)
141     return ReturnError(TokStart, "invalid hexadecimal floating-point constant: "
142                                  "expected at least one exponent digit");
143 
144   return AsmToken(AsmToken::Real, StringRef(TokStart, CurPtr - TokStart));
145 }
146 
147 /// LexIdentifier: [a-zA-Z_.][a-zA-Z0-9_$.@#?]*
148 static bool isIdentifierChar(char C, bool AllowAt, bool AllowHash) {
149   return isAlnum(C) || C == '_' || C == '$' || C == '.' || C == '?' ||
150          (AllowAt && C == '@') || (AllowHash && C == '#');
151 }
152 
153 AsmToken AsmLexer::LexIdentifier() {
154   // Check for floating point literals.
155   if (CurPtr[-1] == '.' && isDigit(*CurPtr)) {
156     // Disambiguate a .1243foo identifier from a floating literal.
157     while (isDigit(*CurPtr))
158       ++CurPtr;
159 
160     if (!isIdentifierChar(*CurPtr, AllowAtInIdentifier,
161                           AllowHashInIdentifier) ||
162         *CurPtr == 'e' || *CurPtr == 'E')
163       return LexFloatLiteral();
164   }
165 
166   while (isIdentifierChar(*CurPtr, AllowAtInIdentifier, AllowHashInIdentifier))
167     ++CurPtr;
168 
169   // Handle . as a special case.
170   if (CurPtr == TokStart+1 && TokStart[0] == '.')
171     return AsmToken(AsmToken::Dot, StringRef(TokStart, 1));
172 
173   return AsmToken(AsmToken::Identifier, StringRef(TokStart, CurPtr - TokStart));
174 }
175 
176 /// LexSlash: Slash: /
177 ///           C-Style Comment: /* ... */
178 AsmToken AsmLexer::LexSlash() {
179   switch (*CurPtr) {
180   case '*':
181     IsAtStartOfStatement = false;
182     break; // C style comment.
183   case '/':
184     ++CurPtr;
185     return LexLineComment();
186   default:
187     IsAtStartOfStatement = false;
188     return AsmToken(AsmToken::Slash, StringRef(TokStart, 1));
189   }
190 
191   // C Style comment.
192   ++CurPtr;  // skip the star.
193   const char *CommentTextStart = CurPtr;
194   while (CurPtr != CurBuf.end()) {
195     switch (*CurPtr++) {
196     case '*':
197       // End of the comment?
198       if (*CurPtr != '/')
199         break;
200       // If we have a CommentConsumer, notify it about the comment.
201       if (CommentConsumer) {
202         CommentConsumer->HandleComment(
203             SMLoc::getFromPointer(CommentTextStart),
204             StringRef(CommentTextStart, CurPtr - 1 - CommentTextStart));
205       }
206       ++CurPtr;   // End the */.
207       return AsmToken(AsmToken::Comment,
208                       StringRef(TokStart, CurPtr - TokStart));
209     }
210   }
211   return ReturnError(TokStart, "unterminated comment");
212 }
213 
214 /// LexLineComment: Comment: #[^\n]*
215 ///                        : //[^\n]*
216 AsmToken AsmLexer::LexLineComment() {
217   // Mark This as an end of statement with a body of the
218   // comment. While it would be nicer to leave this two tokens,
219   // backwards compatability with TargetParsers makes keeping this in this form
220   // better.
221   const char *CommentTextStart = CurPtr;
222   int CurChar = getNextChar();
223   while (CurChar != '\n' && CurChar != '\r' && CurChar != EOF)
224     CurChar = getNextChar();
225   if (CurChar == '\r' && CurPtr != CurBuf.end() && *CurPtr == '\n')
226     ++CurPtr;
227 
228   // If we have a CommentConsumer, notify it about the comment.
229   if (CommentConsumer) {
230     CommentConsumer->HandleComment(
231         SMLoc::getFromPointer(CommentTextStart),
232         StringRef(CommentTextStart, CurPtr - 1 - CommentTextStart));
233   }
234 
235   IsAtStartOfLine = true;
236   // This is a whole line comment. leave newline
237   if (IsAtStartOfStatement)
238     return AsmToken(AsmToken::EndOfStatement,
239                     StringRef(TokStart, CurPtr - TokStart));
240   IsAtStartOfStatement = true;
241 
242   return AsmToken(AsmToken::EndOfStatement,
243                   StringRef(TokStart, CurPtr - 1 - TokStart));
244 }
245 
246 static void SkipIgnoredIntegerSuffix(const char *&CurPtr) {
247   // Skip ULL, UL, U, L and LL suffices.
248   if (CurPtr[0] == 'U')
249     ++CurPtr;
250   if (CurPtr[0] == 'L')
251     ++CurPtr;
252   if (CurPtr[0] == 'L')
253     ++CurPtr;
254 }
255 
256 // Look ahead to search for first non-hex digit, if it's [hH], then we treat the
257 // integer as a hexadecimal, possibly with leading zeroes.
258 static unsigned doHexLookAhead(const char *&CurPtr, unsigned DefaultRadix,
259                                bool LexHex) {
260   const char *FirstNonDec = nullptr;
261   const char *LookAhead = CurPtr;
262   while (true) {
263     if (isDigit(*LookAhead)) {
264       ++LookAhead;
265     } else {
266       if (!FirstNonDec)
267         FirstNonDec = LookAhead;
268 
269       // Keep going if we are looking for a 'h' suffix.
270       if (LexHex && isHexDigit(*LookAhead))
271         ++LookAhead;
272       else
273         break;
274     }
275   }
276   bool isHex = LexHex && (*LookAhead == 'h' || *LookAhead == 'H');
277   CurPtr = isHex || !FirstNonDec ? LookAhead : FirstNonDec;
278   if (isHex)
279     return 16;
280   return DefaultRadix;
281 }
282 
283 static const char *findLastDigit(const char *CurPtr, unsigned DefaultRadix) {
284   while (hexDigitValue(*CurPtr) < DefaultRadix) {
285     ++CurPtr;
286   }
287   return CurPtr;
288 }
289 
290 static AsmToken intToken(StringRef Ref, APInt &Value) {
291   if (Value.isIntN(64))
292     return AsmToken(AsmToken::Integer, Ref, Value);
293   return AsmToken(AsmToken::BigNum, Ref, Value);
294 }
295 
296 static std::string radixName(unsigned Radix) {
297   switch (Radix) {
298   case 2:
299     return "binary";
300   case 8:
301     return "octal";
302   case 10:
303     return "decimal";
304   case 16:
305     return "hexadecimal";
306   default:
307     return "base-" + std::to_string(Radix);
308   }
309 }
310 
311 /// LexDigit: First character is [0-9].
312 ///   Local Label: [0-9][:]
313 ///   Forward/Backward Label: [0-9][fb]
314 ///   Binary integer: 0b[01]+
315 ///   Octal integer: 0[0-7]+
316 ///   Hex integer: 0x[0-9a-fA-F]+ or [0x]?[0-9][0-9a-fA-F]*[hH]
317 ///   Decimal integer: [1-9][0-9]*
318 AsmToken AsmLexer::LexDigit() {
319   // MASM-flavor binary integer: [01]+[yY] (if DefaultRadix < 16, [bByY])
320   // MASM-flavor octal integer: [0-7]+[oOqQ]
321   // MASM-flavor decimal integer: [0-9]+[tT] (if DefaultRadix < 16, [dDtT])
322   // MASM-flavor hexadecimal integer: [0-9][0-9a-fA-F]*[hH]
323   if (LexMasmIntegers && isdigit(CurPtr[-1])) {
324     const char *FirstNonBinary =
325         (CurPtr[-1] != '0' && CurPtr[-1] != '1') ? CurPtr - 1 : nullptr;
326     const char *FirstNonDecimal =
327         (CurPtr[-1] < '0' || CurPtr[-1] > '9') ? CurPtr - 1 : nullptr;
328     const char *OldCurPtr = CurPtr;
329     while (isHexDigit(*CurPtr)) {
330       switch (*CurPtr) {
331       default:
332         if (!FirstNonDecimal) {
333           FirstNonDecimal = CurPtr;
334         }
335         LLVM_FALLTHROUGH;
336       case '9':
337       case '8':
338       case '7':
339       case '6':
340       case '5':
341       case '4':
342       case '3':
343       case '2':
344         if (!FirstNonBinary) {
345           FirstNonBinary = CurPtr;
346         }
347         break;
348       case '1':
349       case '0':
350         break;
351       }
352       ++CurPtr;
353     }
354     if (*CurPtr == '.') {
355       // MASM float literals (other than hex floats) always contain a ".", and
356       // are always written in decimal.
357       ++CurPtr;
358       return LexFloatLiteral();
359     }
360 
361     if (LexMasmHexFloats && (*CurPtr == 'r' || *CurPtr == 'R')) {
362       ++CurPtr;
363       return AsmToken(AsmToken::Real, StringRef(TokStart, CurPtr - TokStart));
364     }
365 
366     unsigned Radix = 0;
367     if (*CurPtr == 'h' || *CurPtr == 'H') {
368       // hexadecimal number
369       ++CurPtr;
370       Radix = 16;
371     } else if (*CurPtr == 't' || *CurPtr == 'T') {
372       // decimal number
373       ++CurPtr;
374       Radix = 10;
375     } else if (*CurPtr == 'o' || *CurPtr == 'O' || *CurPtr == 'q' ||
376                *CurPtr == 'Q') {
377       // octal number
378       ++CurPtr;
379       Radix = 8;
380     } else if (*CurPtr == 'y' || *CurPtr == 'Y') {
381       // binary number
382       ++CurPtr;
383       Radix = 2;
384     } else if (FirstNonDecimal && FirstNonDecimal + 1 == CurPtr &&
385                DefaultRadix < 14 &&
386                (*FirstNonDecimal == 'd' || *FirstNonDecimal == 'D')) {
387       Radix = 10;
388     } else if (FirstNonBinary && FirstNonBinary + 1 == CurPtr &&
389                DefaultRadix < 12 &&
390                (*FirstNonBinary == 'b' || *FirstNonBinary == 'B')) {
391       Radix = 2;
392     }
393 
394     if (Radix) {
395       StringRef Result(TokStart, CurPtr - TokStart);
396       APInt Value(128, 0, true);
397 
398       if (Result.drop_back().getAsInteger(Radix, Value))
399         return ReturnError(TokStart, "invalid " + radixName(Radix) + " number");
400 
401       // MSVC accepts and ignores type suffices on integer literals.
402       SkipIgnoredIntegerSuffix(CurPtr);
403 
404       return intToken(Result, Value);
405     }
406 
407     // default-radix integers, or floating point numbers, fall through
408     CurPtr = OldCurPtr;
409   }
410 
411   // MASM default-radix integers: [0-9a-fA-F]+
412   // (All other integer literals have a radix specifier.)
413   if (LexMasmIntegers && UseMasmDefaultRadix) {
414     CurPtr = findLastDigit(CurPtr, 16);
415     StringRef Result(TokStart, CurPtr - TokStart);
416 
417     APInt Value(128, 0, true);
418     if (Result.getAsInteger(DefaultRadix, Value)) {
419       return ReturnError(TokStart,
420                          "invalid " + radixName(DefaultRadix) + " number");
421     }
422 
423     return intToken(Result, Value);
424   }
425 
426   // Motorola hex integers: $[0-9a-fA-F]+
427   if (LexMotorolaIntegers && CurPtr[-1] == '$') {
428     const char *NumStart = CurPtr;
429     while (isHexDigit(CurPtr[0]))
430       ++CurPtr;
431 
432     APInt Result(128, 0);
433     if (StringRef(NumStart, CurPtr - NumStart).getAsInteger(16, Result))
434       return ReturnError(TokStart, "invalid hexadecimal number");
435 
436     return intToken(StringRef(TokStart, CurPtr - TokStart), Result);
437   }
438 
439   // Motorola binary integers: %[01]+
440   if (LexMotorolaIntegers && CurPtr[-1] == '%') {
441     const char *NumStart = CurPtr;
442     while (*CurPtr == '0' || *CurPtr == '1')
443       ++CurPtr;
444 
445     APInt Result(128, 0);
446     if (StringRef(NumStart, CurPtr - NumStart).getAsInteger(2, Result))
447       return ReturnError(TokStart, "invalid binary number");
448 
449     return intToken(StringRef(TokStart, CurPtr - TokStart), Result);
450   }
451 
452   // Decimal integer: [1-9][0-9]*
453   if (CurPtr[-1] != '0' || CurPtr[0] == '.') {
454     unsigned Radix = doHexLookAhead(CurPtr, 10, LexMasmIntegers);
455     bool isHex = Radix == 16;
456     // Check for floating point literals.
457     if (!isHex && (*CurPtr == '.' || *CurPtr == 'e' || *CurPtr == 'E')) {
458       if (*CurPtr == '.')
459         ++CurPtr;
460       return LexFloatLiteral();
461     }
462 
463     StringRef Result(TokStart, CurPtr - TokStart);
464 
465     APInt Value(128, 0, true);
466     if (Result.getAsInteger(Radix, Value)) {
467       return ReturnError(TokStart, "invalid " + radixName(Radix) + " number");
468     }
469 
470     // The darwin/x86 (and x86-64) assembler accepts and ignores type
471     // suffices on integer literals.
472     SkipIgnoredIntegerSuffix(CurPtr);
473 
474     return intToken(Result, Value);
475   }
476 
477   if (!LexMasmIntegers && ((*CurPtr == 'b') || (*CurPtr == 'B'))) {
478     ++CurPtr;
479     // See if we actually have "0b" as part of something like "jmp 0b\n"
480     if (!isDigit(CurPtr[0])) {
481       --CurPtr;
482       StringRef Result(TokStart, CurPtr - TokStart);
483       return AsmToken(AsmToken::Integer, Result, 0);
484     }
485     const char *NumStart = CurPtr;
486     while (CurPtr[0] == '0' || CurPtr[0] == '1')
487       ++CurPtr;
488 
489     // Requires at least one binary digit.
490     if (CurPtr == NumStart)
491       return ReturnError(TokStart, "invalid binary number");
492 
493     StringRef Result(TokStart, CurPtr - TokStart);
494 
495     APInt Value(128, 0, true);
496     if (Result.substr(2).getAsInteger(2, Value))
497       return ReturnError(TokStart, "invalid binary number");
498 
499     // The darwin/x86 (and x86-64) assembler accepts and ignores ULL and LL
500     // suffixes on integer literals.
501     SkipIgnoredIntegerSuffix(CurPtr);
502 
503     return intToken(Result, Value);
504   }
505 
506   if ((*CurPtr == 'x') || (*CurPtr == 'X')) {
507     ++CurPtr;
508     const char *NumStart = CurPtr;
509     while (isHexDigit(CurPtr[0]))
510       ++CurPtr;
511 
512     // "0x.0p0" is valid, and "0x0p0" (but not "0xp0" for example, which will be
513     // diagnosed by LexHexFloatLiteral).
514     if (CurPtr[0] == '.' || CurPtr[0] == 'p' || CurPtr[0] == 'P')
515       return LexHexFloatLiteral(NumStart == CurPtr);
516 
517     // Otherwise requires at least one hex digit.
518     if (CurPtr == NumStart)
519       return ReturnError(CurPtr-2, "invalid hexadecimal number");
520 
521     APInt Result(128, 0);
522     if (StringRef(TokStart, CurPtr - TokStart).getAsInteger(0, Result))
523       return ReturnError(TokStart, "invalid hexadecimal number");
524 
525     // Consume the optional [hH].
526     if (LexMasmIntegers && (*CurPtr == 'h' || *CurPtr == 'H'))
527       ++CurPtr;
528 
529     // The darwin/x86 (and x86-64) assembler accepts and ignores ULL and LL
530     // suffixes on integer literals.
531     SkipIgnoredIntegerSuffix(CurPtr);
532 
533     return intToken(StringRef(TokStart, CurPtr - TokStart), Result);
534   }
535 
536   // Either octal or hexadecimal.
537   APInt Value(128, 0, true);
538   unsigned Radix = doHexLookAhead(CurPtr, 8, LexMasmIntegers);
539   StringRef Result(TokStart, CurPtr - TokStart);
540   if (Result.getAsInteger(Radix, Value))
541     return ReturnError(TokStart, "invalid " + radixName(Radix) + " number");
542 
543   // Consume the [hH].
544   if (Radix == 16)
545     ++CurPtr;
546 
547   // The darwin/x86 (and x86-64) assembler accepts and ignores ULL and LL
548   // suffixes on integer literals.
549   SkipIgnoredIntegerSuffix(CurPtr);
550 
551   return intToken(Result, Value);
552 }
553 
554 /// LexSingleQuote: Integer: 'b'
555 AsmToken AsmLexer::LexSingleQuote() {
556   int CurChar = getNextChar();
557 
558   if (LexMasmStrings) {
559     while (CurChar != EOF) {
560       if (CurChar != '\'') {
561         CurChar = getNextChar();
562       } else if (peekNextChar() == '\'') {
563         // In MASM single-quote strings, doubled single-quotes mean an escaped
564         // single quote, so should be lexed in.
565         getNextChar();
566         CurChar = getNextChar();
567       } else {
568         break;
569       }
570     }
571     if (CurChar == EOF)
572       return ReturnError(TokStart, "unterminated string constant");
573     return AsmToken(AsmToken::String, StringRef(TokStart, CurPtr - TokStart));
574   }
575 
576   if (CurChar == '\\')
577     CurChar = getNextChar();
578 
579   if (CurChar == EOF)
580     return ReturnError(TokStart, "unterminated single quote");
581 
582   CurChar = getNextChar();
583 
584   if (CurChar != '\'')
585     return ReturnError(TokStart, "single quote way too long");
586 
587   // The idea here being that 'c' is basically just an integral
588   // constant.
589   StringRef Res = StringRef(TokStart,CurPtr - TokStart);
590   long long Value;
591 
592   if (Res.startswith("\'\\")) {
593     char theChar = Res[2];
594     switch (theChar) {
595       default: Value = theChar; break;
596       case '\'': Value = '\''; break;
597       case 't': Value = '\t'; break;
598       case 'n': Value = '\n'; break;
599       case 'b': Value = '\b'; break;
600     }
601   } else
602     Value = TokStart[1];
603 
604   return AsmToken(AsmToken::Integer, Res, Value);
605 }
606 
607 /// LexQuote: String: "..."
608 AsmToken AsmLexer::LexQuote() {
609   int CurChar = getNextChar();
610   if (LexMasmStrings) {
611     while (CurChar != EOF) {
612       if (CurChar != '"') {
613         CurChar = getNextChar();
614       } else if (peekNextChar() == '"') {
615         // In MASM double-quoted strings, doubled double-quotes mean an escaped
616         // double quote, so should be lexed in.
617         getNextChar();
618         CurChar = getNextChar();
619       } else {
620         break;
621       }
622     }
623     if (CurChar == EOF)
624       return ReturnError(TokStart, "unterminated string constant");
625     return AsmToken(AsmToken::String, StringRef(TokStart, CurPtr - TokStart));
626   }
627 
628   // TODO: does gas allow multiline string constants?
629   while (CurChar != '"') {
630     if (CurChar == '\\') {
631       // Allow \", etc.
632       CurChar = getNextChar();
633     }
634 
635     if (CurChar == EOF)
636       return ReturnError(TokStart, "unterminated string constant");
637 
638     CurChar = getNextChar();
639   }
640 
641   return AsmToken(AsmToken::String, StringRef(TokStart, CurPtr - TokStart));
642 }
643 
644 StringRef AsmLexer::LexUntilEndOfStatement() {
645   TokStart = CurPtr;
646 
647   while (!isAtStartOfComment(CurPtr) &&     // Start of line comment.
648          !isAtStatementSeparator(CurPtr) && // End of statement marker.
649          *CurPtr != '\n' && *CurPtr != '\r' && CurPtr != CurBuf.end()) {
650     ++CurPtr;
651   }
652   return StringRef(TokStart, CurPtr-TokStart);
653 }
654 
655 StringRef AsmLexer::LexUntilEndOfLine() {
656   TokStart = CurPtr;
657 
658   while (*CurPtr != '\n' && *CurPtr != '\r' && CurPtr != CurBuf.end()) {
659     ++CurPtr;
660   }
661   return StringRef(TokStart, CurPtr-TokStart);
662 }
663 
664 size_t AsmLexer::peekTokens(MutableArrayRef<AsmToken> Buf,
665                             bool ShouldSkipSpace) {
666   SaveAndRestore<const char *> SavedTokenStart(TokStart);
667   SaveAndRestore<const char *> SavedCurPtr(CurPtr);
668   SaveAndRestore<bool> SavedAtStartOfLine(IsAtStartOfLine);
669   SaveAndRestore<bool> SavedAtStartOfStatement(IsAtStartOfStatement);
670   SaveAndRestore<bool> SavedSkipSpace(SkipSpace, ShouldSkipSpace);
671   SaveAndRestore<bool> SavedIsPeeking(IsPeeking, true);
672   std::string SavedErr = getErr();
673   SMLoc SavedErrLoc = getErrLoc();
674 
675   size_t ReadCount;
676   for (ReadCount = 0; ReadCount < Buf.size(); ++ReadCount) {
677     AsmToken Token = LexToken();
678 
679     Buf[ReadCount] = Token;
680 
681     if (Token.is(AsmToken::Eof))
682       break;
683   }
684 
685   SetError(SavedErrLoc, SavedErr);
686   return ReadCount;
687 }
688 
689 bool AsmLexer::isAtStartOfComment(const char *Ptr) {
690   if (MAI.getRestrictCommentStringToStartOfStatement() && !IsAtStartOfStatement)
691     return false;
692 
693   StringRef CommentString = MAI.getCommentString();
694 
695   if (CommentString.size() == 1)
696     return CommentString[0] == Ptr[0];
697 
698   // Allow # preprocessor commments also be counted as comments for "##" cases
699   if (CommentString[1] == '#')
700     return CommentString[0] == Ptr[0];
701 
702   return strncmp(Ptr, CommentString.data(), CommentString.size()) == 0;
703 }
704 
705 bool AsmLexer::isAtStatementSeparator(const char *Ptr) {
706   return strncmp(Ptr, MAI.getSeparatorString(),
707                  strlen(MAI.getSeparatorString())) == 0;
708 }
709 
710 AsmToken AsmLexer::LexToken() {
711   TokStart = CurPtr;
712   // This always consumes at least one character.
713   int CurChar = getNextChar();
714 
715   if (!IsPeeking && CurChar == '#' && IsAtStartOfStatement) {
716     // If this starts with a '#', this may be a cpp
717     // hash directive and otherwise a line comment.
718     AsmToken TokenBuf[2];
719     MutableArrayRef<AsmToken> Buf(TokenBuf, 2);
720     size_t num = peekTokens(Buf, true);
721     // There cannot be a space preceding this
722     if (IsAtStartOfLine && num == 2 && TokenBuf[0].is(AsmToken::Integer) &&
723         TokenBuf[1].is(AsmToken::String)) {
724       CurPtr = TokStart; // reset curPtr;
725       StringRef s = LexUntilEndOfLine();
726       UnLex(TokenBuf[1]);
727       UnLex(TokenBuf[0]);
728       return AsmToken(AsmToken::HashDirective, s);
729     }
730     return LexLineComment();
731   }
732 
733   if (isAtStartOfComment(TokStart))
734     return LexLineComment();
735 
736   if (isAtStatementSeparator(TokStart)) {
737     CurPtr += strlen(MAI.getSeparatorString()) - 1;
738     IsAtStartOfLine = true;
739     IsAtStartOfStatement = true;
740     return AsmToken(AsmToken::EndOfStatement,
741                     StringRef(TokStart, strlen(MAI.getSeparatorString())));
742   }
743 
744   // If we're missing a newline at EOF, make sure we still get an
745   // EndOfStatement token before the Eof token.
746   if (CurChar == EOF && !IsAtStartOfStatement && EndStatementAtEOF) {
747     IsAtStartOfLine = true;
748     IsAtStartOfStatement = true;
749     return AsmToken(AsmToken::EndOfStatement, StringRef(TokStart, 0));
750   }
751   IsAtStartOfLine = false;
752   bool OldIsAtStartOfStatement = IsAtStartOfStatement;
753   IsAtStartOfStatement = false;
754   switch (CurChar) {
755   default:
756     if (MAI.doesAllowSymbolAtNameStart()) {
757       // Handle Microsoft-style identifier: [a-zA-Z_$.@?][a-zA-Z0-9_$.@#?]*
758       if (!isDigit(CurChar) &&
759           isIdentifierChar(CurChar, MAI.doesAllowAtInName(),
760                            AllowHashInIdentifier))
761         return LexIdentifier();
762     } else {
763       // Handle identifier: [a-zA-Z_.][a-zA-Z0-9_$.@]*
764       if (isalpha(CurChar) || CurChar == '_' || CurChar == '.')
765         return LexIdentifier();
766     }
767 
768     // Unknown character, emit an error.
769     return ReturnError(TokStart, "invalid character in input");
770   case EOF:
771     if (EndStatementAtEOF) {
772       IsAtStartOfLine = true;
773       IsAtStartOfStatement = true;
774     }
775     return AsmToken(AsmToken::Eof, StringRef(TokStart, 0));
776   case 0:
777   case ' ':
778   case '\t':
779     IsAtStartOfStatement = OldIsAtStartOfStatement;
780     while (*CurPtr == ' ' || *CurPtr == '\t')
781       CurPtr++;
782     if (SkipSpace)
783       return LexToken(); // Ignore whitespace.
784     else
785       return AsmToken(AsmToken::Space, StringRef(TokStart, CurPtr - TokStart));
786   case '\r': {
787     IsAtStartOfLine = true;
788     IsAtStartOfStatement = true;
789     // If this is a CR followed by LF, treat that as one token.
790     if (CurPtr != CurBuf.end() && *CurPtr == '\n')
791       ++CurPtr;
792     return AsmToken(AsmToken::EndOfStatement,
793                     StringRef(TokStart, CurPtr - TokStart));
794   }
795   case '\n':
796     IsAtStartOfLine = true;
797     IsAtStartOfStatement = true;
798     return AsmToken(AsmToken::EndOfStatement, StringRef(TokStart, 1));
799   case ':': return AsmToken(AsmToken::Colon, StringRef(TokStart, 1));
800   case '+': return AsmToken(AsmToken::Plus, StringRef(TokStart, 1));
801   case '~': return AsmToken(AsmToken::Tilde, StringRef(TokStart, 1));
802   case '(': return AsmToken(AsmToken::LParen, StringRef(TokStart, 1));
803   case ')': return AsmToken(AsmToken::RParen, StringRef(TokStart, 1));
804   case '[': return AsmToken(AsmToken::LBrac, StringRef(TokStart, 1));
805   case ']': return AsmToken(AsmToken::RBrac, StringRef(TokStart, 1));
806   case '{': return AsmToken(AsmToken::LCurly, StringRef(TokStart, 1));
807   case '}': return AsmToken(AsmToken::RCurly, StringRef(TokStart, 1));
808   case '*': return AsmToken(AsmToken::Star, StringRef(TokStart, 1));
809   case ',': return AsmToken(AsmToken::Comma, StringRef(TokStart, 1));
810   case '$':
811     if (LexMotorolaIntegers && isHexDigit(*CurPtr)) {
812       return LexDigit();
813     }
814 
815     return AsmToken(AsmToken::Dollar, StringRef(TokStart, 1));
816   case '@': return AsmToken(AsmToken::At, StringRef(TokStart, 1));
817   case '\\': return AsmToken(AsmToken::BackSlash, StringRef(TokStart, 1));
818   case '=':
819     if (*CurPtr == '=') {
820       ++CurPtr;
821       return AsmToken(AsmToken::EqualEqual, StringRef(TokStart, 2));
822     }
823     return AsmToken(AsmToken::Equal, StringRef(TokStart, 1));
824   case '-':
825     if (*CurPtr == '>') {
826       ++CurPtr;
827       return AsmToken(AsmToken::MinusGreater, StringRef(TokStart, 2));
828     }
829     return AsmToken(AsmToken::Minus, StringRef(TokStart, 1));
830   case '|':
831     if (*CurPtr == '|') {
832       ++CurPtr;
833       return AsmToken(AsmToken::PipePipe, StringRef(TokStart, 2));
834     }
835     return AsmToken(AsmToken::Pipe, StringRef(TokStart, 1));
836   case '^': return AsmToken(AsmToken::Caret, StringRef(TokStart, 1));
837   case '&':
838     if (*CurPtr == '&') {
839       ++CurPtr;
840       return AsmToken(AsmToken::AmpAmp, StringRef(TokStart, 2));
841     }
842     return AsmToken(AsmToken::Amp, StringRef(TokStart, 1));
843   case '!':
844     if (*CurPtr == '=') {
845       ++CurPtr;
846       return AsmToken(AsmToken::ExclaimEqual, StringRef(TokStart, 2));
847     }
848     return AsmToken(AsmToken::Exclaim, StringRef(TokStart, 1));
849   case '%':
850     if (LexMotorolaIntegers && (*CurPtr == '0' || *CurPtr == '1')) {
851       return LexDigit();
852     }
853 
854     if (MAI.hasMipsExpressions()) {
855       AsmToken::TokenKind Operator;
856       unsigned OperatorLength;
857 
858       std::tie(Operator, OperatorLength) =
859           StringSwitch<std::pair<AsmToken::TokenKind, unsigned>>(
860               StringRef(CurPtr))
861               .StartsWith("call16", {AsmToken::PercentCall16, 7})
862               .StartsWith("call_hi", {AsmToken::PercentCall_Hi, 8})
863               .StartsWith("call_lo", {AsmToken::PercentCall_Lo, 8})
864               .StartsWith("dtprel_hi", {AsmToken::PercentDtprel_Hi, 10})
865               .StartsWith("dtprel_lo", {AsmToken::PercentDtprel_Lo, 10})
866               .StartsWith("got_disp", {AsmToken::PercentGot_Disp, 9})
867               .StartsWith("got_hi", {AsmToken::PercentGot_Hi, 7})
868               .StartsWith("got_lo", {AsmToken::PercentGot_Lo, 7})
869               .StartsWith("got_ofst", {AsmToken::PercentGot_Ofst, 9})
870               .StartsWith("got_page", {AsmToken::PercentGot_Page, 9})
871               .StartsWith("gottprel", {AsmToken::PercentGottprel, 9})
872               .StartsWith("got", {AsmToken::PercentGot, 4})
873               .StartsWith("gp_rel", {AsmToken::PercentGp_Rel, 7})
874               .StartsWith("higher", {AsmToken::PercentHigher, 7})
875               .StartsWith("highest", {AsmToken::PercentHighest, 8})
876               .StartsWith("hi", {AsmToken::PercentHi, 3})
877               .StartsWith("lo", {AsmToken::PercentLo, 3})
878               .StartsWith("neg", {AsmToken::PercentNeg, 4})
879               .StartsWith("pcrel_hi", {AsmToken::PercentPcrel_Hi, 9})
880               .StartsWith("pcrel_lo", {AsmToken::PercentPcrel_Lo, 9})
881               .StartsWith("tlsgd", {AsmToken::PercentTlsgd, 6})
882               .StartsWith("tlsldm", {AsmToken::PercentTlsldm, 7})
883               .StartsWith("tprel_hi", {AsmToken::PercentTprel_Hi, 9})
884               .StartsWith("tprel_lo", {AsmToken::PercentTprel_Lo, 9})
885               .Default({AsmToken::Percent, 1});
886 
887       if (Operator != AsmToken::Percent) {
888         CurPtr += OperatorLength - 1;
889         return AsmToken(Operator, StringRef(TokStart, OperatorLength));
890       }
891     }
892     return AsmToken(AsmToken::Percent, StringRef(TokStart, 1));
893   case '/':
894     IsAtStartOfStatement = OldIsAtStartOfStatement;
895     return LexSlash();
896   case '#': return AsmToken(AsmToken::Hash, StringRef(TokStart, 1));
897   case '\'': return LexSingleQuote();
898   case '"': return LexQuote();
899   case '0': case '1': case '2': case '3': case '4':
900   case '5': case '6': case '7': case '8': case '9':
901     return LexDigit();
902   case '<':
903     switch (*CurPtr) {
904     case '<':
905       ++CurPtr;
906       return AsmToken(AsmToken::LessLess, StringRef(TokStart, 2));
907     case '=':
908       ++CurPtr;
909       return AsmToken(AsmToken::LessEqual, StringRef(TokStart, 2));
910     case '>':
911       ++CurPtr;
912       return AsmToken(AsmToken::LessGreater, StringRef(TokStart, 2));
913     default:
914       return AsmToken(AsmToken::Less, StringRef(TokStart, 1));
915     }
916   case '>':
917     switch (*CurPtr) {
918     case '>':
919       ++CurPtr;
920       return AsmToken(AsmToken::GreaterGreater, StringRef(TokStart, 2));
921     case '=':
922       ++CurPtr;
923       return AsmToken(AsmToken::GreaterEqual, StringRef(TokStart, 2));
924     default:
925       return AsmToken(AsmToken::Greater, StringRef(TokStart, 1));
926     }
927 
928   // TODO: Quoted identifiers (objc methods etc)
929   // local labels: [0-9][:]
930   // Forward/backward labels: [0-9][fb]
931   // Integers, fp constants, character constants.
932   }
933 }
934