1 //===- Lexer.h - MLIR PDLL Frontend Lexer -----------------------*- C++ -*-===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 
9 #ifndef LIB_TOOLS_PDLL_PARSER_LEXER_H_
10 #define LIB_TOOLS_PDLL_PARSER_LEXER_H_
11 
12 #include "mlir/Support/LLVM.h"
13 #include "llvm/ADT/StringRef.h"
14 #include "llvm/Support/SMLoc.h"
15 
16 namespace llvm {
17 class SourceMgr;
18 } // namespace llvm
19 
20 namespace mlir {
21 struct LogicalResult;
22 
23 namespace pdll {
24 class CodeCompleteContext;
25 
26 namespace ast {
27 class DiagnosticEngine;
28 } // namespace ast
29 
30 //===----------------------------------------------------------------------===//
31 // Token
32 //===----------------------------------------------------------------------===//
33 
34 class Token {
35 public:
36   enum Kind {
37     /// Markers.
38     eof,
39     error,
40     /// Token signifying a code completion location.
41     code_complete,
42     /// Token signifying a code completion location within a string.
43     code_complete_string,
44 
45     /// Keywords.
46     KW_BEGIN,
47     /// Dependent keywords, i.e. those that are treated as keywords depending on
48     /// the current parser context.
49     KW_DEPENDENT_BEGIN,
50     kw_attr,
51     kw_op,
52     kw_type,
53     KW_DEPENDENT_END,
54 
55     /// General keywords.
56     kw_Attr,
57     kw_erase,
58     kw_let,
59     kw_Constraint,
60     kw_Op,
61     kw_OpName,
62     kw_Pattern,
63     kw_replace,
64     kw_return,
65     kw_rewrite,
66     kw_Rewrite,
67     kw_Type,
68     kw_TypeRange,
69     kw_Value,
70     kw_ValueRange,
71     kw_with,
72     KW_END,
73 
74     /// Punctuation.
75     arrow,
76     colon,
77     comma,
78     dot,
79     equal,
80     equal_arrow,
81     semicolon,
82     /// Paired punctuation.
83     less,
84     greater,
85     l_brace,
86     r_brace,
87     l_paren,
88     r_paren,
89     l_square,
90     r_square,
91     underscore,
92 
93     /// Tokens.
94     directive,
95     identifier,
96     integer,
97     string_block,
98     string
99   };
100   Token(Kind kind, StringRef spelling) : kind(kind), spelling(spelling) {}
101 
102   /// Given a token containing a string literal, return its value, including
103   /// removing the quote characters and unescaping the contents of the string.
104   std::string getStringValue() const;
105 
106   /// Returns true if the current token is a string literal.
107   bool isString() const { return isAny(Token::string, Token::string_block); }
108 
109   /// Returns true if the current token is a keyword.
110   bool isKeyword() const {
111     return kind > Token::KW_BEGIN && kind < Token::KW_END;
112   }
113 
114   /// Returns true if the current token is a keyword in a dependent context, and
115   /// in any other situation (e.g. variable names) may be treated as an
116   /// identifier.
117   bool isDependentKeyword() const {
118     return kind > Token::KW_DEPENDENT_BEGIN && kind < Token::KW_DEPENDENT_END;
119   }
120 
121   /// Return the bytes that make up this token.
122   StringRef getSpelling() const { return spelling; }
123 
124   /// Return the kind of this token.
125   Kind getKind() const { return kind; }
126 
127   /// Return true if this token is one of the specified kinds.
128   bool isAny(Kind k1, Kind k2) const { return is(k1) || is(k2); }
129   template <typename... T>
130   bool isAny(Kind k1, Kind k2, Kind k3, T... others) const {
131     return is(k1) || isAny(k2, k3, others...);
132   }
133 
134   /// Return if the token does not have the given kind.
135   bool isNot(Kind k) const { return k != kind; }
136   template <typename... T> bool isNot(Kind k1, Kind k2, T... others) const {
137     return !isAny(k1, k2, others...);
138   }
139 
140   /// Return if the token has the given kind.
141   bool is(Kind k) const { return kind == k; }
142 
143   /// Return a location for the start of this token.
144   SMLoc getStartLoc() const {
145     return SMLoc::getFromPointer(spelling.data());
146   }
147   /// Return a location at the end of this token.
148   SMLoc getEndLoc() const {
149     return SMLoc::getFromPointer(spelling.data() + spelling.size());
150   }
151   /// Return a location for the range of this token.
152   SMRange getLoc() const {
153     return SMRange(getStartLoc(), getEndLoc());
154   }
155 
156 private:
157   /// Discriminator that indicates the kind of token this is.
158   Kind kind;
159 
160   /// A reference to the entire token contents; this is always a pointer into
161   /// a memory buffer owned by the source manager.
162   StringRef spelling;
163 };
164 
165 //===----------------------------------------------------------------------===//
166 // Lexer
167 //===----------------------------------------------------------------------===//
168 
169 class Lexer {
170 public:
171   Lexer(llvm::SourceMgr &mgr, ast::DiagnosticEngine &diagEngine,
172         CodeCompleteContext *codeCompleteContext);
173   ~Lexer();
174 
175   /// Return a reference to the source manager used by the lexer.
176   llvm::SourceMgr &getSourceMgr() { return srcMgr; }
177 
178   /// Return a reference to the diagnostic engine used by the lexer.
179   ast::DiagnosticEngine &getDiagEngine() { return diagEngine; }
180 
181   /// Push an include of the given file. This will cause the lexer to start
182   /// processing the provided file. Returns failure if the file could not be
183   /// opened, success otherwise.
184   LogicalResult pushInclude(StringRef filename, SMRange includeLoc);
185 
186   /// Lex the next token and return it.
187   Token lexToken();
188 
189   /// Change the position of the lexer cursor. The next token we lex will start
190   /// at the designated point in the input.
191   void resetPointer(const char *newPointer) { curPtr = newPointer; }
192 
193   /// Emit an error to the lexer with the given location and message.
194   Token emitError(SMRange loc, const Twine &msg);
195   Token emitError(const char *loc, const Twine &msg);
196   Token emitErrorAndNote(SMRange loc, const Twine &msg,
197                          SMRange noteLoc, const Twine &note);
198 
199 private:
200   Token formToken(Token::Kind kind, const char *tokStart) {
201     return Token(kind, StringRef(tokStart, curPtr - tokStart));
202   }
203 
204   /// Return the next character in the stream.
205   int getNextChar();
206 
207   /// Lex methods.
208   void lexComment();
209   Token lexDirective(const char *tokStart);
210   Token lexIdentifier(const char *tokStart);
211   Token lexNumber(const char *tokStart);
212   Token lexString(const char *tokStart, bool isStringBlock);
213 
214   llvm::SourceMgr &srcMgr;
215   int curBufferID;
216   StringRef curBuffer;
217   const char *curPtr;
218 
219   /// The engine used to emit diagnostics during lexing/parsing.
220   ast::DiagnosticEngine &diagEngine;
221 
222   /// A flag indicating if we added a default diagnostic handler to the provided
223   /// diagEngine.
224   bool addedHandlerToDiagEngine;
225 
226   /// The optional code completion point within the input file.
227   const char *codeCompletionLocation;
228 };
229 } // namespace pdll
230 } // namespace mlir
231 
232 #endif // LIB_TOOLS_PDLL_PARSER_LEXER_H_
233