1 //===-- lib/Parser/token-sequence.h -----------------------------*- C++ -*-===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 
9 #ifndef FORTRAN_PARSER_TOKEN_SEQUENCE_H_
10 #define FORTRAN_PARSER_TOKEN_SEQUENCE_H_
11 
12 // A buffer class capable of holding a contiguous sequence of characters
13 // and a partitioning thereof into preprocessing tokens, along with their
14 // associated provenances.
15 
16 #include "flang/Parser/char-block.h"
17 #include "flang/Parser/provenance.h"
18 #include <cstddef>
19 #include <cstring>
20 #include <string>
21 #include <utility>
22 #include <vector>
23 
24 namespace llvm {
25 class raw_ostream;
26 }
27 
28 namespace Fortran::parser {
29 
30 // Buffers a contiguous sequence of characters that has been partitioned into
31 // a sequence of preprocessing tokens with provenances.
32 class TokenSequence {
33 public:
34   TokenSequence() {}
35   TokenSequence(const TokenSequence &that) { Put(that); }
36   TokenSequence(
37       const TokenSequence &that, std::size_t at, std::size_t count = 1) {
38     Put(that, at, count);
39   }
40   TokenSequence(TokenSequence &&that)
41       : start_{std::move(that.start_)}, nextStart_{that.nextStart_},
42         char_{std::move(that.char_)}, provenances_{
43                                           std::move(that.provenances_)} {}
44   TokenSequence(const std::string &s, Provenance p) { Put(s, p); }
45 
46   TokenSequence &operator=(const TokenSequence &that) {
47     clear();
48     Put(that);
49     return *this;
50   }
51   TokenSequence &operator=(TokenSequence &&that);
52   bool empty() const { return start_.empty(); }
53   void clear();
54   void pop_back();
55   void shrink_to_fit();
56   void swap(TokenSequence &);
57 
58   std::size_t SizeInTokens() const { return start_.size(); }
59   std::size_t SizeInChars() const { return char_.size(); }
60 
61   CharBlock ToCharBlock() const { return {&char_[0], char_.size()}; }
62   std::string ToString() const { return ToCharBlock().ToString(); }
63 
64   CharBlock TokenAt(std::size_t token) const {
65     return {&char_[start_.at(token)], TokenBytes(token)};
66   }
67   char CharAt(std::size_t j) const { return char_.at(j); }
68   CharBlock CurrentOpenToken() const {
69     return {&char_[nextStart_], char_.size() - nextStart_};
70   }
71 
72   std::size_t SkipBlanks(std::size_t) const;
73 
74   // True if anything remains in the sequence at & after the given offset
75   // except blanks and line-ending C++ and Fortran free-form comments.
76   bool IsAnythingLeft(std::size_t) const;
77 
78   void PutNextTokenChar(char ch, Provenance provenance) {
79     char_.emplace_back(ch);
80     provenances_.Put({provenance, 1});
81   }
82 
83   void CloseToken() {
84     start_.emplace_back(nextStart_);
85     nextStart_ = char_.size();
86   }
87 
88   void ReopenLastToken() {
89     nextStart_ = start_.back();
90     start_.pop_back();
91   }
92 
93   void RemoveLastToken();
94 
95   void Put(const TokenSequence &);
96   void Put(const TokenSequence &, ProvenanceRange);
97   void Put(const TokenSequence &, std::size_t at, std::size_t tokens = 1);
98   void Put(const char *, std::size_t, Provenance);
99   void Put(const CharBlock &, Provenance);
100   void Put(const std::string &, Provenance);
101   void Put(llvm::raw_string_ostream &, Provenance);
102 
103   Provenance GetTokenProvenance(
104       std::size_t token, std::size_t offset = 0) const;
105   ProvenanceRange GetTokenProvenanceRange(
106       std::size_t token, std::size_t offset = 0) const;
107   ProvenanceRange GetIntervalProvenanceRange(
108       std::size_t token, std::size_t tokens = 1) const;
109   ProvenanceRange GetProvenanceRange() const;
110 
111   char *GetMutableCharData() { return &char_[0]; }
112   TokenSequence &ToLowerCase();
113   bool HasBlanks(std::size_t firstChar = 0) const;
114   bool HasRedundantBlanks(std::size_t firstChar = 0) const;
115   TokenSequence &RemoveBlanks(std::size_t firstChar = 0);
116   TokenSequence &RemoveRedundantBlanks(std::size_t firstChar = 0);
117   TokenSequence &ClipComment(bool skipFirst = false);
118   void Emit(CookedSource &) const;
119   void Dump(llvm::raw_ostream &) const;
120 
121 private:
122   std::size_t TokenBytes(std::size_t token) const {
123     return (token + 1 >= start_.size() ? char_.size() : start_[token + 1]) -
124         start_[token];
125   }
126 
127   std::vector<std::size_t> start_;
128   std::size_t nextStart_{0};
129   std::vector<char> char_;
130   OffsetToProvenanceMappings provenances_;
131 };
132 } // namespace Fortran::parser
133 #endif // FORTRAN_PARSER_TOKEN_SEQUENCE_H_
134