1 //===- AsmLexer.cpp - Lexer for Assembly Files ----------------------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This class implements the lexer for assembly files.
10 //
11 //===----------------------------------------------------------------------===//
12 
13 #include "llvm/MC/MCParser/AsmLexer.h"
14 #include "llvm/ADT/APInt.h"
15 #include "llvm/ADT/ArrayRef.h"
16 #include "llvm/ADT/StringExtras.h"
17 #include "llvm/ADT/StringRef.h"
18 #include "llvm/ADT/StringSwitch.h"
19 #include "llvm/MC/MCAsmInfo.h"
20 #include "llvm/MC/MCParser/MCAsmLexer.h"
21 #include "llvm/Support/Compiler.h"
22 #include "llvm/Support/SMLoc.h"
23 #include "llvm/Support/SaveAndRestore.h"
24 #include <cassert>
25 #include <cctype>
26 #include <cstdio>
27 #include <cstring>
28 #include <string>
29 #include <tuple>
30 #include <utility>
31 
32 using namespace llvm;
33 
34 AsmLexer::AsmLexer(const MCAsmInfo &MAI) : MAI(MAI) {
35   AllowAtInIdentifier = !StringRef(MAI.getCommentString()).startswith("@");
36   LexMotorolaIntegers = MAI.shouldUseMotorolaIntegers();
37 }
38 
39 AsmLexer::~AsmLexer() = default;
40 
41 void AsmLexer::setBuffer(StringRef Buf, const char *ptr,
42                          bool EndStatementAtEOF) {
43   CurBuf = Buf;
44 
45   if (ptr)
46     CurPtr = ptr;
47   else
48     CurPtr = CurBuf.begin();
49 
50   TokStart = nullptr;
51   this->EndStatementAtEOF = EndStatementAtEOF;
52 }
53 
54 /// ReturnError - Set the error to the specified string at the specified
55 /// location.  This is defined to always return AsmToken::Error.
56 AsmToken AsmLexer::ReturnError(const char *Loc, const std::string &Msg) {
57   SetError(SMLoc::getFromPointer(Loc), Msg);
58 
59   return AsmToken(AsmToken::Error, StringRef(Loc, CurPtr - Loc));
60 }
61 
62 int AsmLexer::getNextChar() {
63   if (CurPtr == CurBuf.end())
64     return EOF;
65   return (unsigned char)*CurPtr++;
66 }
67 
68 int AsmLexer::peekNextChar() {
69   if (CurPtr == CurBuf.end())
70     return EOF;
71   return (unsigned char)*CurPtr;
72 }
73 
74 /// The leading integral digit sequence and dot should have already been
75 /// consumed, some or all of the fractional digit sequence *can* have been
76 /// consumed.
77 AsmToken AsmLexer::LexFloatLiteral() {
78   // Skip the fractional digit sequence.
79   while (isDigit(*CurPtr))
80     ++CurPtr;
81 
82   if (*CurPtr == '-' || *CurPtr == '+')
83     return ReturnError(CurPtr, "Invalid sign in float literal");
84 
85   // Check for exponent
86   if ((*CurPtr == 'e' || *CurPtr == 'E')) {
87     ++CurPtr;
88 
89     if (*CurPtr == '-' || *CurPtr == '+')
90       ++CurPtr;
91 
92     while (isDigit(*CurPtr))
93       ++CurPtr;
94   }
95 
96   return AsmToken(AsmToken::Real,
97                   StringRef(TokStart, CurPtr - TokStart));
98 }
99 
100 /// LexHexFloatLiteral matches essentially (.[0-9a-fA-F]*)?[pP][+-]?[0-9a-fA-F]+
101 /// while making sure there are enough actual digits around for the constant to
102 /// be valid.
103 ///
104 /// The leading "0x[0-9a-fA-F]*" (i.e. integer part) has already been consumed
105 /// before we get here.
106 AsmToken AsmLexer::LexHexFloatLiteral(bool NoIntDigits) {
107   assert((*CurPtr == 'p' || *CurPtr == 'P' || *CurPtr == '.') &&
108          "unexpected parse state in floating hex");
109   bool NoFracDigits = true;
110 
111   // Skip the fractional part if there is one
112   if (*CurPtr == '.') {
113     ++CurPtr;
114 
115     const char *FracStart = CurPtr;
116     while (isHexDigit(*CurPtr))
117       ++CurPtr;
118 
119     NoFracDigits = CurPtr == FracStart;
120   }
121 
122   if (NoIntDigits && NoFracDigits)
123     return ReturnError(TokStart, "invalid hexadecimal floating-point constant: "
124                                  "expected at least one significand digit");
125 
126   // Make sure we do have some kind of proper exponent part
127   if (*CurPtr != 'p' && *CurPtr != 'P')
128     return ReturnError(TokStart, "invalid hexadecimal floating-point constant: "
129                                  "expected exponent part 'p'");
130   ++CurPtr;
131 
132   if (*CurPtr == '+' || *CurPtr == '-')
133     ++CurPtr;
134 
135   // N.b. exponent digits are *not* hex
136   const char *ExpStart = CurPtr;
137   while (isDigit(*CurPtr))
138     ++CurPtr;
139 
140   if (CurPtr == ExpStart)
141     return ReturnError(TokStart, "invalid hexadecimal floating-point constant: "
142                                  "expected at least one exponent digit");
143 
144   return AsmToken(AsmToken::Real, StringRef(TokStart, CurPtr - TokStart));
145 }
146 
147 /// LexIdentifier: [a-zA-Z_.][a-zA-Z0-9_$.@#?]*
148 static bool isIdentifierChar(char C, bool AllowAt, bool AllowHash) {
149   return isAlnum(C) || C == '_' || C == '$' || C == '.' || C == '?' ||
150          (AllowAt && C == '@') || (AllowHash && C == '#');
151 }
152 
153 AsmToken AsmLexer::LexIdentifier() {
154   // Check for floating point literals.
155   if (CurPtr[-1] == '.' && isDigit(*CurPtr)) {
156     // Disambiguate a .1243foo identifier from a floating literal.
157     while (isDigit(*CurPtr))
158       ++CurPtr;
159 
160     if (!isIdentifierChar(*CurPtr, AllowAtInIdentifier,
161                           AllowHashInIdentifier) ||
162         *CurPtr == 'e' || *CurPtr == 'E')
163       return LexFloatLiteral();
164   }
165 
166   while (isIdentifierChar(*CurPtr, AllowAtInIdentifier, AllowHashInIdentifier))
167     ++CurPtr;
168 
169   // Handle . as a special case.
170   if (CurPtr == TokStart+1 && TokStart[0] == '.')
171     return AsmToken(AsmToken::Dot, StringRef(TokStart, 1));
172 
173   return AsmToken(AsmToken::Identifier, StringRef(TokStart, CurPtr - TokStart));
174 }
175 
176 /// LexSlash: Slash: /
177 ///           C-Style Comment: /* ... */
178 AsmToken AsmLexer::LexSlash() {
179   switch (*CurPtr) {
180   case '*':
181     IsAtStartOfStatement = false;
182     break; // C style comment.
183   case '/':
184     ++CurPtr;
185     return LexLineComment();
186   default:
187     IsAtStartOfStatement = false;
188     return AsmToken(AsmToken::Slash, StringRef(TokStart, 1));
189   }
190 
191   // C Style comment.
192   ++CurPtr;  // skip the star.
193   const char *CommentTextStart = CurPtr;
194   while (CurPtr != CurBuf.end()) {
195     switch (*CurPtr++) {
196     case '*':
197       // End of the comment?
198       if (*CurPtr != '/')
199         break;
200       // If we have a CommentConsumer, notify it about the comment.
201       if (CommentConsumer) {
202         CommentConsumer->HandleComment(
203             SMLoc::getFromPointer(CommentTextStart),
204             StringRef(CommentTextStart, CurPtr - 1 - CommentTextStart));
205       }
206       ++CurPtr;   // End the */.
207       return AsmToken(AsmToken::Comment,
208                       StringRef(TokStart, CurPtr - TokStart));
209     }
210   }
211   return ReturnError(TokStart, "unterminated comment");
212 }
213 
214 /// LexLineComment: Comment: #[^\n]*
215 ///                        : //[^\n]*
216 AsmToken AsmLexer::LexLineComment() {
217   // Mark This as an end of statement with a body of the
218   // comment. While it would be nicer to leave this two tokens,
219   // backwards compatability with TargetParsers makes keeping this in this form
220   // better.
221   const char *CommentTextStart = CurPtr;
222   int CurChar = getNextChar();
223   while (CurChar != '\n' && CurChar != '\r' && CurChar != EOF)
224     CurChar = getNextChar();
225   if (CurChar == '\r' && CurPtr != CurBuf.end() && *CurPtr == '\n')
226     ++CurPtr;
227 
228   // If we have a CommentConsumer, notify it about the comment.
229   if (CommentConsumer) {
230     CommentConsumer->HandleComment(
231         SMLoc::getFromPointer(CommentTextStart),
232         StringRef(CommentTextStart, CurPtr - 1 - CommentTextStart));
233   }
234 
235   IsAtStartOfLine = true;
236   // This is a whole line comment. leave newline
237   if (IsAtStartOfStatement)
238     return AsmToken(AsmToken::EndOfStatement,
239                     StringRef(TokStart, CurPtr - TokStart));
240   IsAtStartOfStatement = true;
241 
242   return AsmToken(AsmToken::EndOfStatement,
243                   StringRef(TokStart, CurPtr - 1 - TokStart));
244 }
245 
246 static void SkipIgnoredIntegerSuffix(const char *&CurPtr) {
247   // Skip ULL, UL, U, L and LL suffices.
248   if (CurPtr[0] == 'U')
249     ++CurPtr;
250   if (CurPtr[0] == 'L')
251     ++CurPtr;
252   if (CurPtr[0] == 'L')
253     ++CurPtr;
254 }
255 
256 // Look ahead to search for first non-hex digit, if it's [hH], then we treat the
257 // integer as a hexadecimal, possibly with leading zeroes.
258 static unsigned doHexLookAhead(const char *&CurPtr, unsigned DefaultRadix,
259                                bool LexHex) {
260   const char *FirstNonDec = nullptr;
261   const char *LookAhead = CurPtr;
262   while (true) {
263     if (isDigit(*LookAhead)) {
264       ++LookAhead;
265     } else {
266       if (!FirstNonDec)
267         FirstNonDec = LookAhead;
268 
269       // Keep going if we are looking for a 'h' suffix.
270       if (LexHex && isHexDigit(*LookAhead))
271         ++LookAhead;
272       else
273         break;
274     }
275   }
276   bool isHex = LexHex && (*LookAhead == 'h' || *LookAhead == 'H');
277   CurPtr = isHex || !FirstNonDec ? LookAhead : FirstNonDec;
278   if (isHex)
279     return 16;
280   return DefaultRadix;
281 }
282 
283 static const char *findLastDigit(const char *CurPtr, unsigned DefaultRadix) {
284   while (hexDigitValue(*CurPtr) < DefaultRadix) {
285     ++CurPtr;
286   }
287   return CurPtr;
288 }
289 
290 static AsmToken intToken(StringRef Ref, APInt &Value) {
291   if (Value.isIntN(64))
292     return AsmToken(AsmToken::Integer, Ref, Value);
293   return AsmToken(AsmToken::BigNum, Ref, Value);
294 }
295 
296 static std::string radixName(unsigned Radix) {
297   switch (Radix) {
298   case 2:
299     return "binary";
300   case 8:
301     return "octal";
302   case 10:
303     return "decimal";
304   case 16:
305     return "hexadecimal";
306   default:
307     return "base-" + std::to_string(Radix);
308   }
309 }
310 
311 /// LexDigit: First character is [0-9].
312 ///   Local Label: [0-9][:]
313 ///   Forward/Backward Label: [0-9][fb]
314 ///   Binary integer: 0b[01]+
315 ///   Octal integer: 0[0-7]+
316 ///   Hex integer: 0x[0-9a-fA-F]+ or [0x]?[0-9][0-9a-fA-F]*[hH]
317 ///   Decimal integer: [1-9][0-9]*
318 AsmToken AsmLexer::LexDigit() {
319   // MASM-flavor binary integer: [01]+[yY] (if DefaultRadix < 16, [bByY])
320   // MASM-flavor octal integer: [0-7]+[oOqQ]
321   // MASM-flavor decimal integer: [0-9]+[tT] (if DefaultRadix < 16, [dDtT])
322   // MASM-flavor hexadecimal integer: [0-9][0-9a-fA-F]*[hH]
323   if (LexMasmIntegers && isdigit(CurPtr[-1])) {
324     const char *FirstNonBinary =
325         (CurPtr[-1] != '0' && CurPtr[-1] != '1') ? CurPtr - 1 : nullptr;
326     const char *FirstNonDecimal =
327         (CurPtr[-1] < '0' || CurPtr[-1] > '9') ? CurPtr - 1 : nullptr;
328     const char *OldCurPtr = CurPtr;
329     while (isHexDigit(*CurPtr)) {
330       switch (*CurPtr) {
331       default:
332         if (!FirstNonDecimal) {
333           FirstNonDecimal = CurPtr;
334         }
335         LLVM_FALLTHROUGH;
336       case '9':
337       case '8':
338       case '7':
339       case '6':
340       case '5':
341       case '4':
342       case '3':
343       case '2':
344         if (!FirstNonBinary) {
345           FirstNonBinary = CurPtr;
346         }
347         break;
348       case '1':
349       case '0':
350         break;
351       }
352       ++CurPtr;
353     }
354     if (*CurPtr == '.') {
355       // MASM float literals (other than hex floats) always contain a ".", and
356       // are always written in decimal.
357       ++CurPtr;
358       return LexFloatLiteral();
359     }
360 
361     if (LexMasmHexFloats && (*CurPtr == 'r' || *CurPtr == 'R')) {
362       ++CurPtr;
363       return AsmToken(AsmToken::Real, StringRef(TokStart, CurPtr - TokStart));
364     }
365 
366     unsigned Radix = 0;
367     if (*CurPtr == 'h' || *CurPtr == 'H') {
368       // hexadecimal number
369       ++CurPtr;
370       Radix = 16;
371     } else if (*CurPtr == 't' || *CurPtr == 'T') {
372       // decimal number
373       ++CurPtr;
374       Radix = 10;
375     } else if (*CurPtr == 'o' || *CurPtr == 'O' || *CurPtr == 'q' ||
376                *CurPtr == 'Q') {
377       // octal number
378       ++CurPtr;
379       Radix = 8;
380     } else if (*CurPtr == 'y' || *CurPtr == 'Y') {
381       // binary number
382       ++CurPtr;
383       Radix = 2;
384     } else if (FirstNonDecimal && FirstNonDecimal + 1 == CurPtr &&
385                DefaultRadix < 14 &&
386                (*FirstNonDecimal == 'd' || *FirstNonDecimal == 'D')) {
387       Radix = 10;
388     } else if (FirstNonBinary && FirstNonBinary + 1 == CurPtr &&
389                DefaultRadix < 12 &&
390                (*FirstNonBinary == 'b' || *FirstNonBinary == 'B')) {
391       Radix = 2;
392     }
393 
394     if (Radix) {
395       StringRef Result(TokStart, CurPtr - TokStart);
396       APInt Value(128, 0, true);
397 
398       if (Result.drop_back().getAsInteger(Radix, Value))
399         return ReturnError(TokStart, "invalid " + radixName(Radix) + " number");
400 
401       // MSVC accepts and ignores type suffices on integer literals.
402       SkipIgnoredIntegerSuffix(CurPtr);
403 
404       return intToken(Result, Value);
405     }
406 
407     // default-radix integers, or floating point numbers, fall through
408     CurPtr = OldCurPtr;
409   }
410 
411   // MASM default-radix integers: [0-9a-fA-F]+
412   // (All other integer literals have a radix specifier.)
413   if (LexMasmIntegers && UseMasmDefaultRadix) {
414     CurPtr = findLastDigit(CurPtr, 16);
415     StringRef Result(TokStart, CurPtr - TokStart);
416 
417     APInt Value(128, 0, true);
418     if (Result.getAsInteger(DefaultRadix, Value)) {
419       return ReturnError(TokStart,
420                          "invalid " + radixName(DefaultRadix) + " number");
421     }
422 
423     return intToken(Result, Value);
424   }
425 
426   // Motorola hex integers: $[0-9a-fA-F]+
427   if (LexMotorolaIntegers && CurPtr[-1] == '$') {
428     const char *NumStart = CurPtr;
429     while (isHexDigit(CurPtr[0]))
430       ++CurPtr;
431 
432     APInt Result(128, 0);
433     if (StringRef(NumStart, CurPtr - NumStart).getAsInteger(16, Result))
434       return ReturnError(TokStart, "invalid hexadecimal number");
435 
436     return intToken(StringRef(TokStart, CurPtr - TokStart), Result);
437   }
438 
439   // Motorola binary integers: %[01]+
440   if (LexMotorolaIntegers && CurPtr[-1] == '%') {
441     const char *NumStart = CurPtr;
442     while (*CurPtr == '0' || *CurPtr == '1')
443       ++CurPtr;
444 
445     APInt Result(128, 0);
446     if (StringRef(NumStart, CurPtr - NumStart).getAsInteger(2, Result))
447       return ReturnError(TokStart, "invalid binary number");
448 
449     return intToken(StringRef(TokStart, CurPtr - TokStart), Result);
450   }
451 
452   // Decimal integer: [1-9][0-9]*
453   if (CurPtr[-1] != '0' || CurPtr[0] == '.') {
454     unsigned Radix = doHexLookAhead(CurPtr, 10, LexMasmIntegers);
455     bool isHex = Radix == 16;
456     // Check for floating point literals.
457     if (!isHex && (*CurPtr == '.' || *CurPtr == 'e' || *CurPtr == 'E')) {
458       if (*CurPtr == '.')
459         ++CurPtr;
460       return LexFloatLiteral();
461     }
462 
463     StringRef Result(TokStart, CurPtr - TokStart);
464 
465     APInt Value(128, 0, true);
466     if (Result.getAsInteger(Radix, Value)) {
467       return ReturnError(TokStart, "invalid " + radixName(Radix) + " number");
468     }
469 
470     // The darwin/x86 (and x86-64) assembler accepts and ignores type
471     // suffices on integer literals.
472     SkipIgnoredIntegerSuffix(CurPtr);
473 
474     return intToken(Result, Value);
475   }
476 
477   if (!LexMasmIntegers && ((*CurPtr == 'b') || (*CurPtr == 'B'))) {
478     ++CurPtr;
479     // See if we actually have "0b" as part of something like "jmp 0b\n"
480     if (!isDigit(CurPtr[0])) {
481       --CurPtr;
482       StringRef Result(TokStart, CurPtr - TokStart);
483       return AsmToken(AsmToken::Integer, Result, 0);
484     }
485     const char *NumStart = CurPtr;
486     while (CurPtr[0] == '0' || CurPtr[0] == '1')
487       ++CurPtr;
488 
489     // Requires at least one binary digit.
490     if (CurPtr == NumStart)
491       return ReturnError(TokStart, "invalid binary number");
492 
493     StringRef Result(TokStart, CurPtr - TokStart);
494 
495     APInt Value(128, 0, true);
496     if (Result.substr(2).getAsInteger(2, Value))
497       return ReturnError(TokStart, "invalid binary number");
498 
499     // The darwin/x86 (and x86-64) assembler accepts and ignores ULL and LL
500     // suffixes on integer literals.
501     SkipIgnoredIntegerSuffix(CurPtr);
502 
503     return intToken(Result, Value);
504   }
505 
506   if ((*CurPtr == 'x') || (*CurPtr == 'X')) {
507     ++CurPtr;
508     const char *NumStart = CurPtr;
509     while (isHexDigit(CurPtr[0]))
510       ++CurPtr;
511 
512     // "0x.0p0" is valid, and "0x0p0" (but not "0xp0" for example, which will be
513     // diagnosed by LexHexFloatLiteral).
514     if (CurPtr[0] == '.' || CurPtr[0] == 'p' || CurPtr[0] == 'P')
515       return LexHexFloatLiteral(NumStart == CurPtr);
516 
517     // Otherwise requires at least one hex digit.
518     if (CurPtr == NumStart)
519       return ReturnError(CurPtr-2, "invalid hexadecimal number");
520 
521     APInt Result(128, 0);
522     if (StringRef(TokStart, CurPtr - TokStart).getAsInteger(0, Result))
523       return ReturnError(TokStart, "invalid hexadecimal number");
524 
525     // Consume the optional [hH].
526     if (LexMasmIntegers && (*CurPtr == 'h' || *CurPtr == 'H'))
527       ++CurPtr;
528 
529     // The darwin/x86 (and x86-64) assembler accepts and ignores ULL and LL
530     // suffixes on integer literals.
531     SkipIgnoredIntegerSuffix(CurPtr);
532 
533     return intToken(StringRef(TokStart, CurPtr - TokStart), Result);
534   }
535 
536   // Either octal or hexadecimal.
537   APInt Value(128, 0, true);
538   unsigned Radix = doHexLookAhead(CurPtr, 8, LexMasmIntegers);
539   StringRef Result(TokStart, CurPtr - TokStart);
540   if (Result.getAsInteger(Radix, Value))
541     return ReturnError(TokStart, "invalid " + radixName(Radix) + " number");
542 
543   // Consume the [hH].
544   if (Radix == 16)
545     ++CurPtr;
546 
547   // The darwin/x86 (and x86-64) assembler accepts and ignores ULL and LL
548   // suffixes on integer literals.
549   SkipIgnoredIntegerSuffix(CurPtr);
550 
551   return intToken(Result, Value);
552 }
553 
554 /// LexSingleQuote: Integer: 'b'
555 AsmToken AsmLexer::LexSingleQuote() {
556   int CurChar = getNextChar();
557 
558   if (LexMasmStrings) {
559     while (CurChar != EOF) {
560       if (CurChar != '\'') {
561         CurChar = getNextChar();
562       } else if (peekNextChar() == '\'') {
563         // In MASM single-quote strings, doubled single-quotes mean an escaped
564         // single quote, so should be lexed in.
565         getNextChar();
566         CurChar = getNextChar();
567       } else {
568         break;
569       }
570     }
571     if (CurChar == EOF)
572       return ReturnError(TokStart, "unterminated string constant");
573     return AsmToken(AsmToken::String, StringRef(TokStart, CurPtr - TokStart));
574   }
575 
576   if (CurChar == '\\')
577     CurChar = getNextChar();
578 
579   if (CurChar == EOF)
580     return ReturnError(TokStart, "unterminated single quote");
581 
582   CurChar = getNextChar();
583 
584   if (CurChar != '\'')
585     return ReturnError(TokStart, "single quote way too long");
586 
587   // The idea here being that 'c' is basically just an integral
588   // constant.
589   StringRef Res = StringRef(TokStart,CurPtr - TokStart);
590   long long Value;
591 
592   if (Res.startswith("\'\\")) {
593     char theChar = Res[2];
594     switch (theChar) {
595       default: Value = theChar; break;
596       case '\'': Value = '\''; break;
597       case 't': Value = '\t'; break;
598       case 'n': Value = '\n'; break;
599       case 'b': Value = '\b'; break;
600       case 'f': Value = '\f'; break;
601       case 'r': Value = '\r'; break;
602     }
603   } else
604     Value = TokStart[1];
605 
606   return AsmToken(AsmToken::Integer, Res, Value);
607 }
608 
609 /// LexQuote: String: "..."
610 AsmToken AsmLexer::LexQuote() {
611   int CurChar = getNextChar();
612   if (LexMasmStrings) {
613     while (CurChar != EOF) {
614       if (CurChar != '"') {
615         CurChar = getNextChar();
616       } else if (peekNextChar() == '"') {
617         // In MASM double-quoted strings, doubled double-quotes mean an escaped
618         // double quote, so should be lexed in.
619         getNextChar();
620         CurChar = getNextChar();
621       } else {
622         break;
623       }
624     }
625     if (CurChar == EOF)
626       return ReturnError(TokStart, "unterminated string constant");
627     return AsmToken(AsmToken::String, StringRef(TokStart, CurPtr - TokStart));
628   }
629 
630   // TODO: does gas allow multiline string constants?
631   while (CurChar != '"') {
632     if (CurChar == '\\') {
633       // Allow \", etc.
634       CurChar = getNextChar();
635     }
636 
637     if (CurChar == EOF)
638       return ReturnError(TokStart, "unterminated string constant");
639 
640     CurChar = getNextChar();
641   }
642 
643   return AsmToken(AsmToken::String, StringRef(TokStart, CurPtr - TokStart));
644 }
645 
646 StringRef AsmLexer::LexUntilEndOfStatement() {
647   TokStart = CurPtr;
648 
649   while (!isAtStartOfComment(CurPtr) &&     // Start of line comment.
650          !isAtStatementSeparator(CurPtr) && // End of statement marker.
651          *CurPtr != '\n' && *CurPtr != '\r' && CurPtr != CurBuf.end()) {
652     ++CurPtr;
653   }
654   return StringRef(TokStart, CurPtr-TokStart);
655 }
656 
657 StringRef AsmLexer::LexUntilEndOfLine() {
658   TokStart = CurPtr;
659 
660   while (*CurPtr != '\n' && *CurPtr != '\r' && CurPtr != CurBuf.end()) {
661     ++CurPtr;
662   }
663   return StringRef(TokStart, CurPtr-TokStart);
664 }
665 
666 size_t AsmLexer::peekTokens(MutableArrayRef<AsmToken> Buf,
667                             bool ShouldSkipSpace) {
668   SaveAndRestore<const char *> SavedTokenStart(TokStart);
669   SaveAndRestore<const char *> SavedCurPtr(CurPtr);
670   SaveAndRestore<bool> SavedAtStartOfLine(IsAtStartOfLine);
671   SaveAndRestore<bool> SavedAtStartOfStatement(IsAtStartOfStatement);
672   SaveAndRestore<bool> SavedSkipSpace(SkipSpace, ShouldSkipSpace);
673   SaveAndRestore<bool> SavedIsPeeking(IsPeeking, true);
674   std::string SavedErr = getErr();
675   SMLoc SavedErrLoc = getErrLoc();
676 
677   size_t ReadCount;
678   for (ReadCount = 0; ReadCount < Buf.size(); ++ReadCount) {
679     AsmToken Token = LexToken();
680 
681     Buf[ReadCount] = Token;
682 
683     if (Token.is(AsmToken::Eof))
684       break;
685   }
686 
687   SetError(SavedErrLoc, SavedErr);
688   return ReadCount;
689 }
690 
691 bool AsmLexer::isAtStartOfComment(const char *Ptr) {
692   if (MAI.getRestrictCommentStringToStartOfStatement() && !IsAtStartOfStatement)
693     return false;
694 
695   StringRef CommentString = MAI.getCommentString();
696 
697   if (CommentString.size() == 1)
698     return CommentString[0] == Ptr[0];
699 
700   // Allow # preprocessor commments also be counted as comments for "##" cases
701   if (CommentString[1] == '#')
702     return CommentString[0] == Ptr[0];
703 
704   return strncmp(Ptr, CommentString.data(), CommentString.size()) == 0;
705 }
706 
707 bool AsmLexer::isAtStatementSeparator(const char *Ptr) {
708   return strncmp(Ptr, MAI.getSeparatorString(),
709                  strlen(MAI.getSeparatorString())) == 0;
710 }
711 
712 AsmToken AsmLexer::LexToken() {
713   TokStart = CurPtr;
714   // This always consumes at least one character.
715   int CurChar = getNextChar();
716 
717   if (!IsPeeking && CurChar == '#' && IsAtStartOfStatement) {
718     // If this starts with a '#', this may be a cpp
719     // hash directive and otherwise a line comment.
720     AsmToken TokenBuf[2];
721     MutableArrayRef<AsmToken> Buf(TokenBuf, 2);
722     size_t num = peekTokens(Buf, true);
723     // There cannot be a space preceding this
724     if (IsAtStartOfLine && num == 2 && TokenBuf[0].is(AsmToken::Integer) &&
725         TokenBuf[1].is(AsmToken::String)) {
726       CurPtr = TokStart; // reset curPtr;
727       StringRef s = LexUntilEndOfLine();
728       UnLex(TokenBuf[1]);
729       UnLex(TokenBuf[0]);
730       return AsmToken(AsmToken::HashDirective, s);
731     }
732     return LexLineComment();
733   }
734 
735   if (isAtStartOfComment(TokStart))
736     return LexLineComment();
737 
738   if (isAtStatementSeparator(TokStart)) {
739     CurPtr += strlen(MAI.getSeparatorString()) - 1;
740     IsAtStartOfLine = true;
741     IsAtStartOfStatement = true;
742     return AsmToken(AsmToken::EndOfStatement,
743                     StringRef(TokStart, strlen(MAI.getSeparatorString())));
744   }
745 
746   // If we're missing a newline at EOF, make sure we still get an
747   // EndOfStatement token before the Eof token.
748   if (CurChar == EOF && !IsAtStartOfStatement && EndStatementAtEOF) {
749     IsAtStartOfLine = true;
750     IsAtStartOfStatement = true;
751     return AsmToken(AsmToken::EndOfStatement, StringRef(TokStart, 0));
752   }
753   IsAtStartOfLine = false;
754   bool OldIsAtStartOfStatement = IsAtStartOfStatement;
755   IsAtStartOfStatement = false;
756   switch (CurChar) {
757   default:
758     if (MAI.doesAllowSymbolAtNameStart()) {
759       // Handle Microsoft-style identifier: [a-zA-Z_$.@?][a-zA-Z0-9_$.@#?]*
760       if (!isDigit(CurChar) &&
761           isIdentifierChar(CurChar, MAI.doesAllowAtInName(),
762                            AllowHashInIdentifier))
763         return LexIdentifier();
764     } else {
765       // Handle identifier: [a-zA-Z_.][a-zA-Z0-9_$.@]*
766       if (isalpha(CurChar) || CurChar == '_' || CurChar == '.')
767         return LexIdentifier();
768     }
769 
770     // Unknown character, emit an error.
771     return ReturnError(TokStart, "invalid character in input");
772   case EOF:
773     if (EndStatementAtEOF) {
774       IsAtStartOfLine = true;
775       IsAtStartOfStatement = true;
776     }
777     return AsmToken(AsmToken::Eof, StringRef(TokStart, 0));
778   case 0:
779   case ' ':
780   case '\t':
781     IsAtStartOfStatement = OldIsAtStartOfStatement;
782     while (*CurPtr == ' ' || *CurPtr == '\t')
783       CurPtr++;
784     if (SkipSpace)
785       return LexToken(); // Ignore whitespace.
786     else
787       return AsmToken(AsmToken::Space, StringRef(TokStart, CurPtr - TokStart));
788   case '\r': {
789     IsAtStartOfLine = true;
790     IsAtStartOfStatement = true;
791     // If this is a CR followed by LF, treat that as one token.
792     if (CurPtr != CurBuf.end() && *CurPtr == '\n')
793       ++CurPtr;
794     return AsmToken(AsmToken::EndOfStatement,
795                     StringRef(TokStart, CurPtr - TokStart));
796   }
797   case '\n':
798     IsAtStartOfLine = true;
799     IsAtStartOfStatement = true;
800     return AsmToken(AsmToken::EndOfStatement, StringRef(TokStart, 1));
801   case ':': return AsmToken(AsmToken::Colon, StringRef(TokStart, 1));
802   case '+': return AsmToken(AsmToken::Plus, StringRef(TokStart, 1));
803   case '~': return AsmToken(AsmToken::Tilde, StringRef(TokStart, 1));
804   case '(': return AsmToken(AsmToken::LParen, StringRef(TokStart, 1));
805   case ')': return AsmToken(AsmToken::RParen, StringRef(TokStart, 1));
806   case '[': return AsmToken(AsmToken::LBrac, StringRef(TokStart, 1));
807   case ']': return AsmToken(AsmToken::RBrac, StringRef(TokStart, 1));
808   case '{': return AsmToken(AsmToken::LCurly, StringRef(TokStart, 1));
809   case '}': return AsmToken(AsmToken::RCurly, StringRef(TokStart, 1));
810   case '*': return AsmToken(AsmToken::Star, StringRef(TokStart, 1));
811   case ',': return AsmToken(AsmToken::Comma, StringRef(TokStart, 1));
812   case '$':
813     if (LexMotorolaIntegers && isHexDigit(*CurPtr)) {
814       return LexDigit();
815     }
816 
817     return AsmToken(AsmToken::Dollar, StringRef(TokStart, 1));
818   case '@': return AsmToken(AsmToken::At, StringRef(TokStart, 1));
819   case '\\': return AsmToken(AsmToken::BackSlash, StringRef(TokStart, 1));
820   case '=':
821     if (*CurPtr == '=') {
822       ++CurPtr;
823       return AsmToken(AsmToken::EqualEqual, StringRef(TokStart, 2));
824     }
825     return AsmToken(AsmToken::Equal, StringRef(TokStart, 1));
826   case '-':
827     if (*CurPtr == '>') {
828       ++CurPtr;
829       return AsmToken(AsmToken::MinusGreater, StringRef(TokStart, 2));
830     }
831     return AsmToken(AsmToken::Minus, StringRef(TokStart, 1));
832   case '|':
833     if (*CurPtr == '|') {
834       ++CurPtr;
835       return AsmToken(AsmToken::PipePipe, StringRef(TokStart, 2));
836     }
837     return AsmToken(AsmToken::Pipe, StringRef(TokStart, 1));
838   case '^': return AsmToken(AsmToken::Caret, StringRef(TokStart, 1));
839   case '&':
840     if (*CurPtr == '&') {
841       ++CurPtr;
842       return AsmToken(AsmToken::AmpAmp, StringRef(TokStart, 2));
843     }
844     return AsmToken(AsmToken::Amp, StringRef(TokStart, 1));
845   case '!':
846     if (*CurPtr == '=') {
847       ++CurPtr;
848       return AsmToken(AsmToken::ExclaimEqual, StringRef(TokStart, 2));
849     }
850     return AsmToken(AsmToken::Exclaim, StringRef(TokStart, 1));
851   case '%':
852     if (LexMotorolaIntegers && (*CurPtr == '0' || *CurPtr == '1')) {
853       return LexDigit();
854     }
855 
856     if (MAI.hasMipsExpressions()) {
857       AsmToken::TokenKind Operator;
858       unsigned OperatorLength;
859 
860       std::tie(Operator, OperatorLength) =
861           StringSwitch<std::pair<AsmToken::TokenKind, unsigned>>(
862               StringRef(CurPtr))
863               .StartsWith("call16", {AsmToken::PercentCall16, 7})
864               .StartsWith("call_hi", {AsmToken::PercentCall_Hi, 8})
865               .StartsWith("call_lo", {AsmToken::PercentCall_Lo, 8})
866               .StartsWith("dtprel_hi", {AsmToken::PercentDtprel_Hi, 10})
867               .StartsWith("dtprel_lo", {AsmToken::PercentDtprel_Lo, 10})
868               .StartsWith("got_disp", {AsmToken::PercentGot_Disp, 9})
869               .StartsWith("got_hi", {AsmToken::PercentGot_Hi, 7})
870               .StartsWith("got_lo", {AsmToken::PercentGot_Lo, 7})
871               .StartsWith("got_ofst", {AsmToken::PercentGot_Ofst, 9})
872               .StartsWith("got_page", {AsmToken::PercentGot_Page, 9})
873               .StartsWith("gottprel", {AsmToken::PercentGottprel, 9})
874               .StartsWith("got", {AsmToken::PercentGot, 4})
875               .StartsWith("gp_rel", {AsmToken::PercentGp_Rel, 7})
876               .StartsWith("higher", {AsmToken::PercentHigher, 7})
877               .StartsWith("highest", {AsmToken::PercentHighest, 8})
878               .StartsWith("hi", {AsmToken::PercentHi, 3})
879               .StartsWith("lo", {AsmToken::PercentLo, 3})
880               .StartsWith("neg", {AsmToken::PercentNeg, 4})
881               .StartsWith("pcrel_hi", {AsmToken::PercentPcrel_Hi, 9})
882               .StartsWith("pcrel_lo", {AsmToken::PercentPcrel_Lo, 9})
883               .StartsWith("tlsgd", {AsmToken::PercentTlsgd, 6})
884               .StartsWith("tlsldm", {AsmToken::PercentTlsldm, 7})
885               .StartsWith("tprel_hi", {AsmToken::PercentTprel_Hi, 9})
886               .StartsWith("tprel_lo", {AsmToken::PercentTprel_Lo, 9})
887               .Default({AsmToken::Percent, 1});
888 
889       if (Operator != AsmToken::Percent) {
890         CurPtr += OperatorLength - 1;
891         return AsmToken(Operator, StringRef(TokStart, OperatorLength));
892       }
893     }
894     return AsmToken(AsmToken::Percent, StringRef(TokStart, 1));
895   case '/':
896     IsAtStartOfStatement = OldIsAtStartOfStatement;
897     return LexSlash();
898   case '#': return AsmToken(AsmToken::Hash, StringRef(TokStart, 1));
899   case '\'': return LexSingleQuote();
900   case '"': return LexQuote();
901   case '0': case '1': case '2': case '3': case '4':
902   case '5': case '6': case '7': case '8': case '9':
903     return LexDigit();
904   case '<':
905     switch (*CurPtr) {
906     case '<':
907       ++CurPtr;
908       return AsmToken(AsmToken::LessLess, StringRef(TokStart, 2));
909     case '=':
910       ++CurPtr;
911       return AsmToken(AsmToken::LessEqual, StringRef(TokStart, 2));
912     case '>':
913       ++CurPtr;
914       return AsmToken(AsmToken::LessGreater, StringRef(TokStart, 2));
915     default:
916       return AsmToken(AsmToken::Less, StringRef(TokStart, 1));
917     }
918   case '>':
919     switch (*CurPtr) {
920     case '>':
921       ++CurPtr;
922       return AsmToken(AsmToken::GreaterGreater, StringRef(TokStart, 2));
923     case '=':
924       ++CurPtr;
925       return AsmToken(AsmToken::GreaterEqual, StringRef(TokStart, 2));
926     default:
927       return AsmToken(AsmToken::Greater, StringRef(TokStart, 1));
928     }
929 
930   // TODO: Quoted identifiers (objc methods etc)
931   // local labels: [0-9][:]
932   // Forward/backward labels: [0-9][fb]
933   // Integers, fp constants, character constants.
934   }
935 }
936