1 //===-- lib/Parser/token-sequence.h -----------------------------*- C++ -*-===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 
9 #ifndef FORTRAN_PARSER_TOKEN_SEQUENCE_H_
10 #define FORTRAN_PARSER_TOKEN_SEQUENCE_H_
11 
12 // A buffer class capable of holding a contiguous sequence of characters
13 // and a partitioning thereof into preprocessing tokens, along with their
14 // associated provenances.
15 
16 #include "flang/Parser/char-block.h"
17 #include "flang/Parser/provenance.h"
18 #include <cstddef>
19 #include <cstring>
20 #include <string>
21 #include <utility>
22 #include <vector>
23 
24 namespace llvm {
25 class raw_ostream;
26 }
27 
28 namespace Fortran::parser {
29 
30 // Buffers a contiguous sequence of characters that has been partitioned into
31 // a sequence of preprocessing tokens with provenances.
32 class TokenSequence {
33 public:
34   TokenSequence() {}
35   TokenSequence(const TokenSequence &that) { Put(that); }
36   TokenSequence(
37       const TokenSequence &that, std::size_t at, std::size_t count = 1) {
38     Put(that, at, count);
39   }
40   TokenSequence(TokenSequence &&that)
41       : start_{std::move(that.start_)}, nextStart_{that.nextStart_},
42         char_{std::move(that.char_)}, provenances_{
43                                           std::move(that.provenances_)} {}
44   TokenSequence(const std::string &s, Provenance p) { Put(s, p); }
45 
46   TokenSequence &operator=(const TokenSequence &that) {
47     clear();
48     Put(that);
49     return *this;
50   }
51   TokenSequence &operator=(TokenSequence &&that);
52   bool empty() const { return start_.empty(); }
53   void clear();
54   void pop_back();
55   void shrink_to_fit();
56   void swap(TokenSequence &);
57 
58   std::size_t SizeInTokens() const { return start_.size(); }
59   std::size_t SizeInChars() const { return char_.size(); }
60 
61   CharBlock ToCharBlock() const { return {&char_[0], char_.size()}; }
62   std::string ToString() const { return ToCharBlock().ToString(); }
63 
64   CharBlock TokenAt(std::size_t token) const {
65     return {&char_[start_.at(token)], TokenBytes(token)};
66   }
67   char CharAt(std::size_t j) const { return char_.at(j); }
68   CharBlock CurrentOpenToken() const {
69     return {&char_[nextStart_], char_.size() - nextStart_};
70   }
71 
72   std::size_t SkipBlanks(std::size_t) const;
73 
74   void PutNextTokenChar(char ch, Provenance provenance) {
75     char_.emplace_back(ch);
76     provenances_.Put({provenance, 1});
77   }
78 
79   void CloseToken() {
80     start_.emplace_back(nextStart_);
81     nextStart_ = char_.size();
82   }
83 
84   void ReopenLastToken() {
85     nextStart_ = start_.back();
86     start_.pop_back();
87   }
88 
89   void RemoveLastToken();
90 
91   void Put(const TokenSequence &);
92   void Put(const TokenSequence &, ProvenanceRange);
93   void Put(const TokenSequence &, std::size_t at, std::size_t tokens = 1);
94   void Put(const char *, std::size_t, Provenance);
95   void Put(const CharBlock &, Provenance);
96   void Put(const std::string &, Provenance);
97   void Put(llvm::raw_string_ostream &, Provenance);
98 
99   Provenance GetTokenProvenance(
100       std::size_t token, std::size_t offset = 0) const;
101   ProvenanceRange GetTokenProvenanceRange(
102       std::size_t token, std::size_t offset = 0) const;
103   ProvenanceRange GetIntervalProvenanceRange(
104       std::size_t token, std::size_t tokens = 1) const;
105   ProvenanceRange GetProvenanceRange() const;
106 
107   char *GetMutableCharData() { return &char_[0]; }
108   TokenSequence &ToLowerCase();
109   bool HasBlanks(std::size_t firstChar = 0) const;
110   bool HasRedundantBlanks(std::size_t firstChar = 0) const;
111   TokenSequence &RemoveBlanks(std::size_t firstChar = 0);
112   TokenSequence &RemoveRedundantBlanks(std::size_t firstChar = 0);
113   TokenSequence &ClipComment(bool skipFirst = false);
114   void Emit(CookedSource &) const;
115   void Dump(llvm::raw_ostream &) const;
116 
117 private:
118   std::size_t TokenBytes(std::size_t token) const {
119     return (token + 1 >= start_.size() ? char_.size() : start_[token + 1]) -
120         start_[token];
121   }
122 
123   std::vector<std::size_t> start_;
124   std::size_t nextStart_{0};
125   std::vector<char> char_;
126   OffsetToProvenanceMappings provenances_;
127 };
128 } // namespace Fortran::parser
129 #endif // FORTRAN_PARSER_TOKEN_SEQUENCE_H_
130