1 //===--- Lex.cpp - extract token stream from source code ---------*- C++-*-===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 
9 #include "clang-pseudo/Token.h"
10 #include "clang/Basic/SourceLocation.h"
11 #include "clang/Basic/TokenKinds.h"
12 #include "clang/Lex/Lexer.h"
13 #include "clang/Lex/LiteralSupport.h"
14 
15 namespace clang {
16 namespace pseudo {
17 
18 TokenStream lex(const std::string &Code, const clang::LangOptions &LangOpts) {
19   clang::SourceLocation Start;
20   // Tokenize using clang's lexer in raw mode.
21   // std::string guarantees null-termination, which the lexer needs.
22   clang::Lexer Lexer(Start, LangOpts, Code.data(), Code.data(),
23                      Code.data() + Code.size());
24   Lexer.SetCommentRetentionState(true);
25 
26   TokenStream Result;
27   clang::Token CT;
28   unsigned LastOffset = 0;
29   unsigned Line = 0;
30   unsigned Indent = 0;
31   for (Lexer.LexFromRawLexer(CT); CT.getKind() != clang::tok::eof;
32        Lexer.LexFromRawLexer(CT)) {
33     unsigned Offset =
34         CT.getLocation().getRawEncoding() - Start.getRawEncoding();
35 
36     Token Tok;
37     Tok.Data = &Code[Offset];
38     Tok.Length = CT.getLength();
39     Tok.Kind = CT.getKind();
40 
41     // Update current line number and indentation from raw source code.
42     unsigned NewLineStart = 0;
43     for (unsigned i = LastOffset; i < Offset; ++i) {
44       if (Code[i] == '\n') {
45         NewLineStart = i + 1;
46         ++Line;
47       }
48     }
49     if (NewLineStart || !LastOffset) {
50       Indent = 0;
51       for (char c : StringRef(Code).slice(NewLineStart, Offset)) {
52         if (c == ' ')
53           ++Indent;
54         else if (c == '\t')
55           Indent += 8;
56         else
57           break;
58       }
59     }
60     Tok.Indent = Indent;
61     Tok.Line = Line;
62 
63     if (CT.isAtStartOfLine())
64       Tok.setFlag(LexFlags::StartsPPLine);
65     if (CT.needsCleaning() || CT.hasUCN())
66       Tok.setFlag(LexFlags::NeedsCleaning);
67 
68     Result.push(Tok);
69     LastOffset = Offset;
70   }
71   Result.finalize();
72   return Result;
73 }
74 
75 TokenStream cook(const TokenStream &Code, const LangOptions &LangOpts) {
76   auto CleanedStorage = std::make_shared<llvm::BumpPtrAllocator>();
77   clang::IdentifierTable Identifiers(LangOpts);
78   TokenStream Result(CleanedStorage);
79 
80   for (auto Tok : Code.tokens()) {
81     if (Tok.flag(LexFlags::NeedsCleaning)) {
82       // Remove escaped newlines and trigraphs.
83       llvm::SmallString<64> CleanBuffer;
84       const char *Pos = Tok.text().begin();
85       while (Pos < Tok.text().end()) {
86         unsigned CharSize = 0;
87         CleanBuffer.push_back(
88             clang::Lexer::getCharAndSizeNoWarn(Pos, CharSize, LangOpts));
89         assert(CharSize != 0 && "no progress!");
90         Pos += CharSize;
91       }
92       // Remove universal character names (UCN).
93       llvm::SmallString<64> UCNBuffer;
94       clang::expandUCNs(UCNBuffer, CleanBuffer);
95 
96       llvm::StringRef Text = llvm::StringRef(UCNBuffer).copy(*CleanedStorage);
97       Tok.Data = Text.data();
98       Tok.Length = Text.size();
99       Tok.Flags &= ~static_cast<decltype(Tok.Flags)>(LexFlags::NeedsCleaning);
100     }
101 
102     if (Tok.Kind == tok::raw_identifier) {
103       // Cook raw_identifiers into identifier, keyword, etc.
104       Tok.Kind = Identifiers.get(Tok.text()).getTokenID();
105     } else if (Tok.Kind == tok::greatergreater) {
106       // Split the greatergreater token.
107       // FIXME: split lessless token to support Cuda triple angle brackets <<<.
108       assert(Tok.text() == ">>");
109       Tok.Kind = tok::greater;
110       Tok.Length = 1;
111       Result.push(Tok);
112       // Line is wrong if the first greater is followed by an escaped newline!
113       Tok.Data = Tok.text().data() + 1;
114     }
115 
116     Result.push(std::move(Tok));
117   }
118 
119   Result.finalize();
120   return Result;
121 }
122 
123 } // namespace pseudo
124 } // namespace clang
125