1 //===--- Token.cpp - Tokens and token streams in the pseudoparser ---------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 
9 #include "clang-pseudo/Token.h"
10 #include "llvm/ADT/StringExtras.h"
11 #include "llvm/Support/Format.h"
12 #include "llvm/Support/FormatVariadic.h"
13 
14 namespace clang {
15 namespace pseudo {
16 
17 llvm::raw_ostream &operator<<(llvm::raw_ostream &OS, const Token &T) {
18   OS << llvm::formatv("{0} {1}:{2} ", clang::tok::getTokenName(T.Kind), T.Line,
19                       T.Indent);
20   OS << '"';
21   llvm::printEscapedString(T.text(), OS);
22   OS << '"';
23   if (T.Flags)
24     OS << llvm::format(" flags=%x", T.Flags);
25   return OS;
26 }
27 
28 llvm::raw_ostream &operator<<(llvm::raw_ostream &OS, const TokenStream &TS) {
29   OS << "Index               Kind    Line  Text\n";
30   for (const auto &T : TS.tokens()) {
31     OS << llvm::format("%5d:  %16s %4d:%-2d  ", TS.index(T),
32                        clang::tok::getTokenName(T.Kind), T.Line, T.Indent);
33     OS << '"';
34     llvm::printEscapedString(T.text(), OS);
35     OS << '"';
36     if (T.Flags)
37       OS << llvm::format("  flags=%x", T.Flags);
38     OS << '\n';
39   }
40   return OS;
41 }
42 
43 llvm::raw_ostream &operator<<(llvm::raw_ostream &OS, const Token::Range &R) {
44   OS << llvm::formatv("[{0},{1})", R.Begin, R.End);
45   return OS;
46 }
47 
48 TokenStream::TokenStream(std::shared_ptr<void> Payload)
49     : Payload(std::move(Payload)) {
50   Storage.emplace_back();
51   Storage.back().Kind = clang::tok::eof;
52 }
53 
54 void TokenStream::finalize() {
55   assert(!isFinalized());
56   unsigned LastLine = Storage.back().Line;
57   Storage.emplace_back();
58   Storage.back().Kind = tok::eof;
59   Storage.back().Line = LastLine + 1;
60 
61   Tokens = Storage;
62   Tokens = Tokens.drop_front().drop_back();
63 }
64 
65 bool TokenStream::isFinalized() const {
66   assert(!Storage.empty() && Storage.front().Kind == tok::eof);
67   if (Storage.size() == 1)
68     return false;
69   return Storage.back().Kind == tok::eof;
70 }
71 
72 void TokenStream::print(llvm::raw_ostream &OS) const {
73   bool FirstToken = true;
74   unsigned LastLine = -1;
75   StringRef LastText;
76   for (const auto &T : tokens()) {
77     StringRef Text = T.text();
78     if (FirstToken) {
79       FirstToken = false;
80     } else if (T.Line == LastLine) {
81       if (LastText.data() + LastText.size() != Text.data())
82         OS << ' ';
83     } else {
84       OS << '\n';
85       OS.indent(T.Indent);
86     }
87     OS << Text;
88     LastLine = T.Line;
89     LastText = Text;
90   }
91   if (!FirstToken)
92     OS << '\n';
93 }
94 
95 TokenStream stripComments(const TokenStream &Input) {
96   TokenStream Out;
97   for (const Token &T : Input.tokens()) {
98     if (T.Kind == tok::comment)
99       continue;
100     Out.push(T);
101   }
102   Out.finalize();
103   return Out;
104 }
105 
106 } // namespace pseudo
107 } // namespace clang
108