1 //===--- PPCaching.cpp - Handle caching lexed tokens ----------------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This file implements pieces of the Preprocessor interface that manage the
10 // caching of lexed tokens.
11 //
12 //===----------------------------------------------------------------------===//
13 
14 #include "clang/Lex/Preprocessor.h"
15 using namespace clang;
16 
17 // EnableBacktrackAtThisPos - From the point that this method is called, and
18 // until CommitBacktrackedTokens() or Backtrack() is called, the Preprocessor
19 // keeps track of the lexed tokens so that a subsequent Backtrack() call will
20 // make the Preprocessor re-lex the same tokens.
21 //
22 // Nested backtracks are allowed, meaning that EnableBacktrackAtThisPos can
23 // be called multiple times and CommitBacktrackedTokens/Backtrack calls will
24 // be combined with the EnableBacktrackAtThisPos calls in reverse order.
25 void Preprocessor::EnableBacktrackAtThisPos() {
26   BacktrackPositions.push_back(CachedLexPos);
27   EnterCachingLexMode();
28 }
29 
30 // Disable the last EnableBacktrackAtThisPos call.
31 void Preprocessor::CommitBacktrackedTokens() {
32   assert(!BacktrackPositions.empty()
33          && "EnableBacktrackAtThisPos was not called!");
34   BacktrackPositions.pop_back();
35 }
36 
37 Preprocessor::CachedTokensRange Preprocessor::LastCachedTokenRange() {
38   assert(isBacktrackEnabled());
39   auto PrevCachedLexPos = BacktrackPositions.back();
40   return CachedTokensRange{PrevCachedLexPos, CachedLexPos};
41 }
42 
43 void Preprocessor::EraseCachedTokens(CachedTokensRange TokenRange) {
44   assert(TokenRange.Begin <= TokenRange.End);
45   if (CachedLexPos == TokenRange.Begin && TokenRange.Begin != TokenRange.End) {
46     // We have backtracked to the start of the token range as we want to consume
47     // them again. Erase the tokens only after consuming then.
48     assert(!CachedTokenRangeToErase);
49     CachedTokenRangeToErase = TokenRange;
50     return;
51   }
52   // The cached tokens were committed, so they should be erased now.
53   assert(TokenRange.End == CachedLexPos);
54   CachedTokens.erase(CachedTokens.begin() + TokenRange.Begin,
55                      CachedTokens.begin() + TokenRange.End);
56   CachedLexPos = TokenRange.Begin;
57   ExitCachingLexMode();
58 }
59 
60 // Make Preprocessor re-lex the tokens that were lexed since
61 // EnableBacktrackAtThisPos() was previously called.
62 void Preprocessor::Backtrack() {
63   assert(!BacktrackPositions.empty()
64          && "EnableBacktrackAtThisPos was not called!");
65   CachedLexPos = BacktrackPositions.back();
66   BacktrackPositions.pop_back();
67   recomputeCurLexerKind();
68 }
69 
70 void Preprocessor::CachingLex(Token &Result) {
71   if (!InCachingLexMode())
72     return;
73 
74   if (CachedLexPos < CachedTokens.size()) {
75     Result = CachedTokens[CachedLexPos++];
76     // Erase the some of the cached tokens after they are consumed when
77     // asked to do so.
78     if (CachedTokenRangeToErase &&
79         CachedTokenRangeToErase->End == CachedLexPos) {
80       EraseCachedTokens(*CachedTokenRangeToErase);
81       CachedTokenRangeToErase = None;
82     }
83     return;
84   }
85 
86   ExitCachingLexMode();
87   Lex(Result);
88 
89   if (isBacktrackEnabled()) {
90     // Cache the lexed token.
91     EnterCachingLexMode();
92     CachedTokens.push_back(Result);
93     ++CachedLexPos;
94     return;
95   }
96 
97   if (CachedLexPos < CachedTokens.size()) {
98     EnterCachingLexMode();
99   } else {
100     // All cached tokens were consumed.
101     CachedTokens.clear();
102     CachedLexPos = 0;
103   }
104 }
105 
106 void Preprocessor::EnterCachingLexMode() {
107   if (InCachingLexMode()) {
108     assert(CurLexerKind == CLK_CachingLexer && "Unexpected lexer kind");
109     return;
110   }
111 
112   PushIncludeMacroStack();
113   CurLexerKind = CLK_CachingLexer;
114 }
115 
116 
117 const Token &Preprocessor::PeekAhead(unsigned N) {
118   assert(CachedLexPos + N > CachedTokens.size() && "Confused caching.");
119   ExitCachingLexMode();
120   for (size_t C = CachedLexPos + N - CachedTokens.size(); C > 0; --C) {
121     CachedTokens.push_back(Token());
122     Lex(CachedTokens.back());
123   }
124   EnterCachingLexMode();
125   return CachedTokens.back();
126 }
127 
128 void Preprocessor::AnnotatePreviousCachedTokens(const Token &Tok) {
129   assert(Tok.isAnnotation() && "Expected annotation token");
130   assert(CachedLexPos != 0 && "Expected to have some cached tokens");
131   assert(CachedTokens[CachedLexPos-1].getLastLoc() == Tok.getAnnotationEndLoc()
132          && "The annotation should be until the most recent cached token");
133 
134   // Start from the end of the cached tokens list and look for the token
135   // that is the beginning of the annotation token.
136   for (CachedTokensTy::size_type i = CachedLexPos; i != 0; --i) {
137     CachedTokensTy::iterator AnnotBegin = CachedTokens.begin() + i-1;
138     if (AnnotBegin->getLocation() == Tok.getLocation()) {
139       assert((BacktrackPositions.empty() || BacktrackPositions.back() <= i) &&
140              "The backtrack pos points inside the annotated tokens!");
141       // Replace the cached tokens with the single annotation token.
142       if (i < CachedLexPos)
143         CachedTokens.erase(AnnotBegin + 1, CachedTokens.begin() + CachedLexPos);
144       *AnnotBegin = Tok;
145       CachedLexPos = i;
146       return;
147     }
148   }
149 }
150 
151 bool Preprocessor::IsPreviousCachedToken(const Token &Tok) const {
152   // There's currently no cached token...
153   if (!CachedLexPos)
154     return false;
155 
156   const Token LastCachedTok = CachedTokens[CachedLexPos - 1];
157   if (LastCachedTok.getKind() != Tok.getKind())
158     return false;
159 
160   int RelOffset = 0;
161   if ((!getSourceManager().isInSameSLocAddrSpace(
162           Tok.getLocation(), getLastCachedTokenLocation(), &RelOffset)) ||
163       RelOffset)
164     return false;
165 
166   return true;
167 }
168 
169 void Preprocessor::ReplacePreviousCachedToken(ArrayRef<Token> NewToks) {
170   assert(CachedLexPos != 0 && "Expected to have some cached tokens");
171   CachedTokens.insert(CachedTokens.begin() + CachedLexPos - 1, NewToks.begin(),
172                       NewToks.end());
173   CachedTokens.erase(CachedTokens.begin() + CachedLexPos - 1 + NewToks.size());
174   CachedLexPos += NewToks.size() - 1;
175 }
176