1 //===- unittest/Format/TokenAnnotatorTest.cpp - Formatting unit tests -----===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 
9 #include "clang/Format/Format.h"
10 
11 #include "FormatTestUtils.h"
12 #include "TestLexer.h"
13 #include "gtest/gtest.h"
14 
15 namespace clang {
16 namespace format {
17 
18 // Not really the equality, but everything we need.
19 static bool operator==(const FormatToken &LHS,
20                        const FormatToken &RHS) noexcept {
21   return LHS.Tok.getKind() == RHS.Tok.getKind() &&
22          LHS.getType() == RHS.getType();
23 }
24 
25 namespace {
26 
27 class TokenAnnotatorTest : public ::testing::Test {
28 protected:
29   TokenList annotate(llvm::StringRef Code,
30                      const FormatStyle &Style = getLLVMStyle()) {
31     return TestLexer(Allocator, Buffers, Style).annotate(Code);
32   }
33   llvm::SpecificBumpPtrAllocator<FormatToken> Allocator;
34   std::vector<std::unique_ptr<llvm::MemoryBuffer>> Buffers;
35 };
36 
37 #define EXPECT_TOKEN_KIND(FormatTok, Kind)                                     \
38   EXPECT_EQ((FormatTok)->Tok.getKind(), Kind) << *(FormatTok)
39 #define EXPECT_TOKEN_TYPE(FormatTok, Type)                                     \
40   EXPECT_EQ((FormatTok)->getType(), Type) << *(FormatTok)
41 #define EXPECT_TOKEN(FormatTok, Kind, Type)                                    \
42   do {                                                                         \
43     EXPECT_TOKEN_KIND(FormatTok, Kind);                                        \
44     EXPECT_TOKEN_TYPE(FormatTok, Type);                                        \
45   } while (false)
46 
47 TEST_F(TokenAnnotatorTest, UnderstandsUsesOfStarAndAmp) {
48   auto Tokens = annotate("auto x = [](const decltype(x) &ptr) {};");
49   EXPECT_EQ(Tokens.size(), 18u) << Tokens;
50   EXPECT_TOKEN(Tokens[7], tok::kw_decltype, TT_Unknown);
51   EXPECT_TOKEN(Tokens[8], tok::l_paren, TT_TypeDeclarationParen);
52   EXPECT_TOKEN(Tokens[9], tok::identifier, TT_Unknown);
53   EXPECT_TOKEN(Tokens[10], tok::r_paren, TT_TypeDeclarationParen);
54   EXPECT_TOKEN(Tokens[11], tok::amp, TT_PointerOrReference);
55 
56   Tokens = annotate("auto x = [](const decltype(x) *ptr) {};");
57   EXPECT_EQ(Tokens.size(), 18u) << Tokens;
58   EXPECT_TOKEN(Tokens[10], tok::r_paren, TT_TypeDeclarationParen);
59   EXPECT_TOKEN(Tokens[11], tok::star, TT_PointerOrReference);
60 
61   Tokens = annotate("#define lambda [](const decltype(x) &ptr) {}");
62   EXPECT_EQ(Tokens.size(), 17u) << Tokens;
63   EXPECT_TOKEN(Tokens[7], tok::kw_decltype, TT_Unknown);
64   EXPECT_TOKEN(Tokens[8], tok::l_paren, TT_TypeDeclarationParen);
65   EXPECT_TOKEN(Tokens[9], tok::identifier, TT_Unknown);
66   EXPECT_TOKEN(Tokens[10], tok::r_paren, TT_TypeDeclarationParen);
67   EXPECT_TOKEN(Tokens[11], tok::amp, TT_PointerOrReference);
68 
69   Tokens = annotate("#define lambda [](const decltype(x) *ptr) {}");
70   EXPECT_EQ(Tokens.size(), 17u) << Tokens;
71   EXPECT_TOKEN(Tokens[10], tok::r_paren, TT_TypeDeclarationParen);
72   EXPECT_TOKEN(Tokens[11], tok::star, TT_PointerOrReference);
73 
74   Tokens = annotate("void f() {\n"
75                     "  while (p < a && *p == 'a')\n"
76                     "    p++;\n"
77                     "}");
78   EXPECT_EQ(Tokens.size(), 21u) << Tokens;
79   EXPECT_TOKEN(Tokens[10], tok::ampamp, TT_BinaryOperator);
80   EXPECT_TOKEN(Tokens[11], tok::star, TT_UnaryOperator);
81 }
82 
83 TEST_F(TokenAnnotatorTest, UnderstandsClasses) {
84   auto Tokens = annotate("class C {};");
85   EXPECT_EQ(Tokens.size(), 6u) << Tokens;
86   EXPECT_TOKEN(Tokens[2], tok::l_brace, TT_ClassLBrace);
87 
88   Tokens = annotate("const class C {} c;");
89   EXPECT_EQ(Tokens.size(), 8u) << Tokens;
90   EXPECT_TOKEN(Tokens[3], tok::l_brace, TT_ClassLBrace);
91 
92   Tokens = annotate("const class {} c;");
93   EXPECT_EQ(Tokens.size(), 7u) << Tokens;
94   EXPECT_TOKEN(Tokens[2], tok::l_brace, TT_ClassLBrace);
95 }
96 
97 TEST_F(TokenAnnotatorTest, UnderstandsStructs) {
98   auto Tokens = annotate("struct S {};");
99   EXPECT_EQ(Tokens.size(), 6u) << Tokens;
100   EXPECT_TOKEN(Tokens[2], tok::l_brace, TT_StructLBrace);
101 
102   Tokens = annotate("template <typename T> struct S<const T[N]> {};");
103   EXPECT_EQ(Tokens.size(), 18u) << Tokens;
104   EXPECT_TOKEN(Tokens[7], tok::less, TT_TemplateOpener);
105   EXPECT_TOKEN(Tokens[10], tok::l_square, TT_ArraySubscriptLSquare);
106   EXPECT_TOKEN(Tokens[13], tok::greater, TT_TemplateCloser);
107   EXPECT_TOKEN(Tokens[14], tok::l_brace, TT_StructLBrace);
108 
109   Tokens = annotate("template <typename T> struct S<T const[N]> {};");
110   EXPECT_EQ(Tokens.size(), 18u) << Tokens;
111   EXPECT_TOKEN(Tokens[7], tok::less, TT_TemplateOpener);
112   EXPECT_TOKEN(Tokens[10], tok::l_square, TT_ArraySubscriptLSquare);
113   EXPECT_TOKEN(Tokens[13], tok::greater, TT_TemplateCloser);
114   EXPECT_TOKEN(Tokens[14], tok::l_brace, TT_StructLBrace);
115 }
116 
117 TEST_F(TokenAnnotatorTest, UnderstandsUnions) {
118   auto Tokens = annotate("union U {};");
119   EXPECT_EQ(Tokens.size(), 6u) << Tokens;
120   EXPECT_TOKEN(Tokens[2], tok::l_brace, TT_UnionLBrace);
121 
122   Tokens = annotate("union U { void f() { return; } };");
123   EXPECT_EQ(Tokens.size(), 14u) << Tokens;
124   EXPECT_TOKEN(Tokens[2], tok::l_brace, TT_UnionLBrace);
125   EXPECT_TOKEN(Tokens[7], tok::l_brace, TT_FunctionLBrace);
126 }
127 
128 TEST_F(TokenAnnotatorTest, UnderstandsEnums) {
129   auto Tokens = annotate("enum E {};");
130   EXPECT_EQ(Tokens.size(), 6u) << Tokens;
131   EXPECT_TOKEN(Tokens[2], tok::l_brace, TT_EnumLBrace);
132 }
133 
134 TEST_F(TokenAnnotatorTest, UnderstandsDefaultedAndDeletedFunctions) {
135   auto Tokens = annotate("auto operator<=>(const T &) const & = default;");
136   EXPECT_EQ(Tokens.size(), 14u) << Tokens;
137   EXPECT_TOKEN(Tokens[9], tok::amp, TT_PointerOrReference);
138 
139   Tokens = annotate("template <typename T> void F(T) && = delete;");
140   EXPECT_EQ(Tokens.size(), 15u) << Tokens;
141   EXPECT_TOKEN(Tokens[10], tok::ampamp, TT_PointerOrReference);
142 }
143 
144 TEST_F(TokenAnnotatorTest, UnderstandsVariables) {
145   auto Tokens =
146       annotate("inline bool var = is_integral_v<int> && is_signed_v<int>;");
147   EXPECT_EQ(Tokens.size(), 15u) << Tokens;
148   EXPECT_TOKEN(Tokens[8], tok::ampamp, TT_BinaryOperator);
149 }
150 
151 TEST_F(TokenAnnotatorTest, UnderstandsVariableTemplates) {
152   auto Tokens =
153       annotate("template <typename T> "
154                "inline bool var = is_integral_v<int> && is_signed_v<int>;");
155   EXPECT_EQ(Tokens.size(), 20u) << Tokens;
156   EXPECT_TOKEN(Tokens[13], tok::ampamp, TT_BinaryOperator);
157 }
158 
159 TEST_F(TokenAnnotatorTest, UnderstandsLBracesInMacroDefinition) {
160   auto Tokens = annotate("#define BEGIN NS {");
161   EXPECT_EQ(Tokens.size(), 6u) << Tokens;
162   EXPECT_TOKEN(Tokens[4], tok::l_brace, TT_Unknown);
163 }
164 
165 TEST_F(TokenAnnotatorTest, UnderstandsDelete) {
166   auto Tokens = annotate("delete (void *)p;");
167   EXPECT_EQ(Tokens.size(), 8u) << Tokens;
168   EXPECT_TOKEN(Tokens[4], tok::r_paren, TT_CastRParen);
169 
170   Tokens = annotate("delete[] (void *)p;");
171   EXPECT_EQ(Tokens.size(), 10u) << Tokens;
172   EXPECT_TOKEN(Tokens[6], tok::r_paren, TT_CastRParen);
173 
174   Tokens = annotate("delete[] /*comment*/ (void *)p;");
175   EXPECT_EQ(Tokens.size(), 11u) << Tokens;
176   EXPECT_TOKEN(Tokens[7], tok::r_paren, TT_CastRParen);
177 
178   Tokens = annotate("delete[/*comment*/] (void *)p;");
179   EXPECT_EQ(Tokens.size(), 11u) << Tokens;
180   EXPECT_TOKEN(Tokens[7], tok::r_paren, TT_CastRParen);
181 
182   Tokens = annotate("delete/*comment*/[] (void *)p;");
183   EXPECT_EQ(Tokens.size(), 11u) << Tokens;
184   EXPECT_TOKEN(Tokens[7], tok::r_paren, TT_CastRParen);
185 }
186 
187 TEST_F(TokenAnnotatorTest, UnderstandsFunctionRefQualifiers) {
188   auto Tokens = annotate("void f() &;");
189   EXPECT_EQ(Tokens.size(), 7u) << Tokens;
190   EXPECT_TOKEN(Tokens[4], tok::amp, TT_PointerOrReference);
191 
192   Tokens = annotate("void operator=(T) &&;");
193   EXPECT_EQ(Tokens.size(), 9u) << Tokens;
194   EXPECT_TOKEN(Tokens[6], tok::ampamp, TT_PointerOrReference);
195 
196   Tokens = annotate("template <typename T> void f() &;");
197   EXPECT_EQ(Tokens.size(), 12u) << Tokens;
198   EXPECT_TOKEN(Tokens[9], tok::amp, TT_PointerOrReference);
199 
200   Tokens = annotate("template <typename T> void operator=(T) &;");
201   EXPECT_EQ(Tokens.size(), 14u) << Tokens;
202   EXPECT_TOKEN(Tokens[11], tok::amp, TT_PointerOrReference);
203 }
204 
205 TEST_F(TokenAnnotatorTest, UnderstandsRequiresClausesAndConcepts) {
206   auto Tokens = annotate("template <typename T>\n"
207                          "concept C = (Foo && Bar) && (Bar && Baz);");
208 
209   ASSERT_EQ(Tokens.size(), 21u) << Tokens;
210   EXPECT_TOKEN(Tokens[10], tok::ampamp, TT_BinaryOperator);
211   EXPECT_TOKEN(Tokens[13], tok::ampamp, TT_BinaryOperator);
212   EXPECT_TOKEN(Tokens[16], tok::ampamp, TT_BinaryOperator);
213 
214   Tokens = annotate("template <typename T>\n"
215                     "concept C = requires(T t) {\n"
216                     "  { t.foo() };\n"
217                     "} && Bar<T> && Baz<T>;");
218   ASSERT_EQ(Tokens.size(), 35u) << Tokens;
219   EXPECT_TOKEN(Tokens[8], tok::kw_requires, TT_RequiresExpression);
220   EXPECT_TOKEN(Tokens[9], tok::l_paren, TT_RequiresExpressionLParen);
221   EXPECT_TOKEN(Tokens[13], tok::l_brace, TT_RequiresExpressionLBrace);
222   EXPECT_TOKEN(Tokens[23], tok::ampamp, TT_BinaryOperator);
223   EXPECT_TOKEN(Tokens[28], tok::ampamp, TT_BinaryOperator);
224 
225   Tokens = annotate("template<typename T>\n"
226                     "requires C1<T> && (C21<T> || C22<T> && C2e<T>) && C3<T>\n"
227                     "struct Foo;");
228   ASSERT_EQ(Tokens.size(), 36u) << Tokens;
229   EXPECT_TOKEN(Tokens[5], tok::kw_requires, TT_RequiresClause);
230   EXPECT_TOKEN(Tokens[6], tok::identifier, TT_Unknown);
231   EXPECT_EQ(Tokens[6]->FakeLParens.size(), 1u);
232   EXPECT_TOKEN(Tokens[10], tok::ampamp, TT_BinaryOperator);
233   EXPECT_TOKEN(Tokens[16], tok::pipepipe, TT_BinaryOperator);
234   EXPECT_TOKEN(Tokens[21], tok::ampamp, TT_BinaryOperator);
235   EXPECT_TOKEN(Tokens[27], tok::ampamp, TT_BinaryOperator);
236   EXPECT_TOKEN(Tokens[31], tok::greater, TT_TemplateCloser);
237   EXPECT_EQ(Tokens[31]->FakeRParens, 1u);
238   EXPECT_TRUE(Tokens[31]->ClosesRequiresClause);
239 
240   Tokens =
241       annotate("template<typename T>\n"
242                "requires (C1<T> && (C21<T> || C22<T> && C2e<T>) && C3<T>)\n"
243                "struct Foo;");
244   ASSERT_EQ(Tokens.size(), 38u) << Tokens;
245   EXPECT_TOKEN(Tokens[5], tok::kw_requires, TT_RequiresClause);
246   EXPECT_TOKEN(Tokens[7], tok::identifier, TT_Unknown);
247   EXPECT_EQ(Tokens[7]->FakeLParens.size(), 1u);
248   EXPECT_TOKEN(Tokens[11], tok::ampamp, TT_BinaryOperator);
249   EXPECT_TOKEN(Tokens[17], tok::pipepipe, TT_BinaryOperator);
250   EXPECT_TOKEN(Tokens[22], tok::ampamp, TT_BinaryOperator);
251   EXPECT_TOKEN(Tokens[28], tok::ampamp, TT_BinaryOperator);
252   EXPECT_TOKEN(Tokens[32], tok::greater, TT_TemplateCloser);
253   EXPECT_EQ(Tokens[32]->FakeRParens, 1u);
254   EXPECT_TOKEN(Tokens[33], tok::r_paren, TT_Unknown);
255   EXPECT_TRUE(Tokens[33]->ClosesRequiresClause);
256 
257   Tokens = annotate("template <typename T>\n"
258                     "void foo(T) noexcept requires Bar<T>;");
259   ASSERT_EQ(Tokens.size(), 18u) << Tokens;
260   EXPECT_TOKEN(Tokens[11], tok::kw_requires, TT_RequiresClause);
261 
262   Tokens = annotate("template <typename T>\n"
263                     "struct S {\n"
264                     "  void foo() const requires Bar<T>;\n"
265                     "  void bar() const & requires Baz<T>;\n"
266                     "  void bar() && requires Baz2<T>;\n"
267                     "  void baz() const & noexcept requires Baz<T>;\n"
268                     "  void baz() && noexcept requires Baz2<T>;\n"
269                     "};\n"
270                     "\n"
271                     "void S::bar() const & requires Baz<T> { }");
272   ASSERT_EQ(Tokens.size(), 85u) << Tokens;
273   EXPECT_TOKEN(Tokens[13], tok::kw_requires, TT_RequiresClause);
274   EXPECT_TOKEN(Tokens[25], tok::kw_requires, TT_RequiresClause);
275   EXPECT_TOKEN(Tokens[36], tok::kw_requires, TT_RequiresClause);
276   EXPECT_TOKEN(Tokens[49], tok::kw_requires, TT_RequiresClause);
277   EXPECT_TOKEN(Tokens[61], tok::kw_requires, TT_RequiresClause);
278   EXPECT_TOKEN(Tokens[77], tok::kw_requires, TT_RequiresClause);
279 
280   Tokens = annotate("void Class::member() && requires(Constant) {}");
281   ASSERT_EQ(Tokens.size(), 14u) << Tokens;
282   EXPECT_TOKEN(Tokens[7], tok::kw_requires, TT_RequiresClause);
283 
284   Tokens = annotate("void Class::member() && requires(Constant<T>) {}");
285   ASSERT_EQ(Tokens.size(), 17u) << Tokens;
286   EXPECT_TOKEN(Tokens[7], tok::kw_requires, TT_RequiresClause);
287 
288   Tokens =
289       annotate("void Class::member() && requires(Namespace::Constant<T>) {}");
290   ASSERT_EQ(Tokens.size(), 19u) << Tokens;
291   EXPECT_TOKEN(Tokens[7], tok::kw_requires, TT_RequiresClause);
292 
293   Tokens = annotate("void Class::member() && requires(typename "
294                     "Namespace::Outer<T>::Inner::Constant) {}");
295   ASSERT_EQ(Tokens.size(), 24u) << Tokens;
296   EXPECT_TOKEN(Tokens[7], tok::kw_requires, TT_RequiresClause);
297 
298   Tokens = annotate("struct [[nodiscard]] zero_t {\n"
299                     "  template<class T>\n"
300                     "    requires requires { number_zero_v<T>; }\n"
301                     "  [[nodiscard]] constexpr operator T() const { "
302                     "return number_zero_v<T>; }\n"
303                     "};");
304   ASSERT_EQ(Tokens.size(), 44u);
305   EXPECT_TOKEN(Tokens[13], tok::kw_requires, TT_RequiresClause);
306   EXPECT_TOKEN(Tokens[14], tok::kw_requires, TT_RequiresExpression);
307   EXPECT_TOKEN(Tokens[15], tok::l_brace, TT_RequiresExpressionLBrace);
308   EXPECT_TOKEN(Tokens[21], tok::r_brace, TT_Unknown);
309   EXPECT_EQ(Tokens[21]->MatchingParen, Tokens[15]);
310   EXPECT_TRUE(Tokens[21]->ClosesRequiresClause);
311 }
312 
313 TEST_F(TokenAnnotatorTest, UnderstandsRequiresExpressions) {
314   auto Tokens = annotate("bool b = requires(int i) { i + 5; };");
315   ASSERT_EQ(Tokens.size(), 16u) << Tokens;
316   EXPECT_TOKEN(Tokens[3], tok::kw_requires, TT_RequiresExpression);
317   EXPECT_TOKEN(Tokens[4], tok::l_paren, TT_RequiresExpressionLParen);
318   EXPECT_TOKEN(Tokens[8], tok::l_brace, TT_RequiresExpressionLBrace);
319 
320   Tokens = annotate("if (requires(int i) { i + 5; }) return;");
321   ASSERT_EQ(Tokens.size(), 17u) << Tokens;
322   EXPECT_TOKEN(Tokens[2], tok::kw_requires, TT_RequiresExpression);
323   EXPECT_TOKEN(Tokens[3], tok::l_paren, TT_RequiresExpressionLParen);
324   EXPECT_TOKEN(Tokens[7], tok::l_brace, TT_RequiresExpressionLBrace);
325 
326   Tokens = annotate("if (func() && requires(int i) { i + 5; }) return;");
327   ASSERT_EQ(Tokens.size(), 21u) << Tokens;
328   EXPECT_TOKEN(Tokens[6], tok::kw_requires, TT_RequiresExpression);
329   EXPECT_TOKEN(Tokens[7], tok::l_paren, TT_RequiresExpressionLParen);
330   EXPECT_TOKEN(Tokens[11], tok::l_brace, TT_RequiresExpressionLBrace);
331 
332   Tokens = annotate("foo(requires(const T t) {});");
333   ASSERT_EQ(Tokens.size(), 13u) << Tokens;
334   EXPECT_TOKEN(Tokens[2], tok::kw_requires, TT_RequiresExpression);
335   EXPECT_TOKEN(Tokens[3], tok::l_paren, TT_RequiresExpressionLParen);
336   EXPECT_TOKEN(Tokens[8], tok::l_brace, TT_RequiresExpressionLBrace);
337 
338   Tokens = annotate("foo(requires(const int t) {});");
339   ASSERT_EQ(Tokens.size(), 13u) << Tokens;
340   EXPECT_TOKEN(Tokens[2], tok::kw_requires, TT_RequiresExpression);
341   EXPECT_TOKEN(Tokens[3], tok::l_paren, TT_RequiresExpressionLParen);
342   EXPECT_TOKEN(Tokens[8], tok::l_brace, TT_RequiresExpressionLBrace);
343 
344   Tokens = annotate("foo(requires(const T t) {});");
345   ASSERT_EQ(Tokens.size(), 13u) << Tokens;
346   EXPECT_TOKEN(Tokens[2], tok::kw_requires, TT_RequiresExpression);
347   EXPECT_TOKEN(Tokens[3], tok::l_paren, TT_RequiresExpressionLParen);
348   EXPECT_TOKEN(Tokens[8], tok::l_brace, TT_RequiresExpressionLBrace);
349 
350   Tokens = annotate("foo(requires(int const* volatile t) {});");
351   ASSERT_EQ(Tokens.size(), 15u) << Tokens;
352   EXPECT_TOKEN(Tokens[2], tok::kw_requires, TT_RequiresExpression);
353   EXPECT_TOKEN(Tokens[3], tok::l_paren, TT_RequiresExpressionLParen);
354   EXPECT_TOKEN(Tokens[10], tok::l_brace, TT_RequiresExpressionLBrace);
355 
356   Tokens = annotate("foo(requires(T const* volatile t) {});");
357   ASSERT_EQ(Tokens.size(), 15u) << Tokens;
358   EXPECT_TOKEN(Tokens[2], tok::kw_requires, TT_RequiresExpression);
359   EXPECT_TOKEN(Tokens[3], tok::l_paren, TT_RequiresExpressionLParen);
360   EXPECT_TOKEN(Tokens[10], tok::l_brace, TT_RequiresExpressionLBrace);
361 
362   Tokens =
363       annotate("foo(requires(const typename Outer<T>::Inner * const t) {});");
364   ASSERT_EQ(Tokens.size(), 21u) << Tokens;
365   EXPECT_TOKEN(Tokens[2], tok::kw_requires, TT_RequiresExpression);
366   EXPECT_TOKEN(Tokens[3], tok::l_paren, TT_RequiresExpressionLParen);
367   EXPECT_TOKEN(Tokens[16], tok::l_brace, TT_RequiresExpressionLBrace);
368 
369   Tokens = annotate("template <typename T>\n"
370                     "concept C = requires(T T) {\n"
371                     "  requires Bar<T> && Foo<T>;\n"
372                     "};");
373   ASSERT_EQ(Tokens.size(), 28u) << Tokens;
374   EXPECT_TOKEN(Tokens[8], tok::kw_requires, TT_RequiresExpression);
375   EXPECT_TOKEN(Tokens[9], tok::l_paren, TT_RequiresExpressionLParen);
376   EXPECT_TOKEN(Tokens[13], tok::l_brace, TT_RequiresExpressionLBrace);
377   EXPECT_TOKEN(Tokens[14], tok::kw_requires,
378                TT_RequiresClauseInARequiresExpression);
379 
380   Tokens = annotate("template <typename T>\n"
381                     "concept C = requires(T T) {\n"
382                     "  { t.func() } -> std::same_as<int>;"
383                     "  requires Bar<T> && Foo<T>;\n"
384                     "};");
385   ASSERT_EQ(Tokens.size(), 43u) << Tokens;
386   EXPECT_TOKEN(Tokens[8], tok::kw_requires, TT_RequiresExpression);
387   EXPECT_TOKEN(Tokens[9], tok::l_paren, TT_RequiresExpressionLParen);
388   EXPECT_TOKEN(Tokens[13], tok::l_brace, TT_RequiresExpressionLBrace);
389   EXPECT_TOKEN(Tokens[29], tok::kw_requires,
390                TT_RequiresClauseInARequiresExpression);
391 
392   // Invalid Code, but we don't want to crash. See http://llvm.org/PR54350.
393   Tokens = annotate("bool r10 = requires (struct new_struct { int x; } s) { "
394                     "requires true; };");
395   ASSERT_EQ(Tokens.size(), 21u) << Tokens;
396   EXPECT_TOKEN(Tokens[3], tok::kw_requires, TT_RequiresExpression);
397   EXPECT_TOKEN(Tokens[4], tok::l_paren, TT_RequiresExpressionLParen);
398   EXPECT_TOKEN(Tokens[14], tok::l_brace, TT_RequiresExpressionLBrace);
399 }
400 
401 TEST_F(TokenAnnotatorTest, RequiresDoesNotChangeParsingOfTheRest) {
402   auto NumberOfAdditionalRequiresClauseTokens = 5u;
403   auto NumberOfTokensBeforeRequires = 5u;
404 
405   auto BaseTokens = annotate("template<typename T>\n"
406                              "T Pi = 3.14;");
407   auto ConstrainedTokens = annotate("template<typename T>\n"
408                                     "  requires Foo<T>\n"
409                                     "T Pi = 3.14;");
410 
411   auto NumberOfBaseTokens = 11u;
412 
413   ASSERT_EQ(BaseTokens.size(), NumberOfBaseTokens) << BaseTokens;
414   ASSERT_EQ(ConstrainedTokens.size(),
415             NumberOfBaseTokens + NumberOfAdditionalRequiresClauseTokens)
416       << ConstrainedTokens;
417 
418   for (auto I = 0u; I < NumberOfBaseTokens; ++I)
419     if (I < NumberOfTokensBeforeRequires)
420       EXPECT_EQ(*BaseTokens[I], *ConstrainedTokens[I]) << I;
421     else
422       EXPECT_EQ(*BaseTokens[I],
423                 *ConstrainedTokens[I + NumberOfAdditionalRequiresClauseTokens])
424           << I;
425 
426   BaseTokens = annotate("template<typename T>\n"
427                         "struct Bar;");
428   ConstrainedTokens = annotate("template<typename T>\n"
429                                "  requires Foo<T>\n"
430                                "struct Bar;");
431   NumberOfBaseTokens = 9u;
432 
433   ASSERT_EQ(BaseTokens.size(), NumberOfBaseTokens) << BaseTokens;
434   ASSERT_EQ(ConstrainedTokens.size(),
435             NumberOfBaseTokens + NumberOfAdditionalRequiresClauseTokens)
436       << ConstrainedTokens;
437 
438   for (auto I = 0u; I < NumberOfBaseTokens; ++I)
439     if (I < NumberOfTokensBeforeRequires)
440       EXPECT_EQ(*BaseTokens[I], *ConstrainedTokens[I]) << I;
441     else
442       EXPECT_EQ(*BaseTokens[I],
443                 *ConstrainedTokens[I + NumberOfAdditionalRequiresClauseTokens])
444           << I;
445 
446   BaseTokens = annotate("template<typename T>\n"
447                         "struct Bar {"
448                         "  T foo();\n"
449                         "  T bar();\n"
450                         "};");
451   ConstrainedTokens = annotate("template<typename T>\n"
452                                "  requires Foo<T>\n"
453                                "struct Bar {"
454                                "  T foo();\n"
455                                "  T bar();\n"
456                                "};");
457   NumberOfBaseTokens = 21u;
458 
459   ASSERT_EQ(BaseTokens.size(), NumberOfBaseTokens) << BaseTokens;
460   ASSERT_EQ(ConstrainedTokens.size(),
461             NumberOfBaseTokens + NumberOfAdditionalRequiresClauseTokens)
462       << ConstrainedTokens;
463 
464   for (auto I = 0u; I < NumberOfBaseTokens; ++I)
465     if (I < NumberOfTokensBeforeRequires)
466       EXPECT_EQ(*BaseTokens[I], *ConstrainedTokens[I]) << I;
467     else
468       EXPECT_EQ(*BaseTokens[I],
469                 *ConstrainedTokens[I + NumberOfAdditionalRequiresClauseTokens])
470           << I;
471 
472   BaseTokens = annotate("template<typename T>\n"
473                         "Bar(T) -> Bar<T>;");
474   ConstrainedTokens = annotate("template<typename T>\n"
475                                "  requires Foo<T>\n"
476                                "Bar(T) -> Bar<T>;");
477   NumberOfBaseTokens = 16u;
478 
479   ASSERT_EQ(BaseTokens.size(), NumberOfBaseTokens) << BaseTokens;
480   ASSERT_EQ(ConstrainedTokens.size(),
481             NumberOfBaseTokens + NumberOfAdditionalRequiresClauseTokens)
482       << ConstrainedTokens;
483 
484   for (auto I = 0u; I < NumberOfBaseTokens; ++I)
485     if (I < NumberOfTokensBeforeRequires)
486       EXPECT_EQ(*BaseTokens[I], *ConstrainedTokens[I]) << I;
487     else
488       EXPECT_EQ(*BaseTokens[I],
489                 *ConstrainedTokens[I + NumberOfAdditionalRequiresClauseTokens])
490           << I;
491 
492   BaseTokens = annotate("template<typename T>\n"
493                         "T foo();");
494   ConstrainedTokens = annotate("template<typename T>\n"
495                                "  requires Foo<T>\n"
496                                "T foo();");
497   NumberOfBaseTokens = 11u;
498 
499   ASSERT_EQ(BaseTokens.size(), NumberOfBaseTokens) << BaseTokens;
500   ASSERT_EQ(ConstrainedTokens.size(),
501             NumberOfBaseTokens + NumberOfAdditionalRequiresClauseTokens)
502       << ConstrainedTokens;
503 
504   for (auto I = 0u; I < NumberOfBaseTokens; ++I)
505     if (I < NumberOfTokensBeforeRequires)
506       EXPECT_EQ(*BaseTokens[I], *ConstrainedTokens[I]) << I;
507     else
508       EXPECT_EQ(*BaseTokens[I],
509                 *ConstrainedTokens[I + NumberOfAdditionalRequiresClauseTokens])
510           << I;
511 
512   BaseTokens = annotate("template<typename T>\n"
513                         "T foo() {\n"
514                         "  auto bar = baz();\n"
515                         "  return bar + T{};\n"
516                         "}");
517   ConstrainedTokens = annotate("template<typename T>\n"
518                                "  requires Foo<T>\n"
519                                "T foo() {\n"
520                                "  auto bar = baz();\n"
521                                "  return bar + T{};\n"
522                                "}");
523   NumberOfBaseTokens = 26u;
524 
525   ASSERT_EQ(BaseTokens.size(), NumberOfBaseTokens) << BaseTokens;
526   ASSERT_EQ(ConstrainedTokens.size(),
527             NumberOfBaseTokens + NumberOfAdditionalRequiresClauseTokens)
528       << ConstrainedTokens;
529 
530   for (auto I = 0u; I < NumberOfBaseTokens; ++I)
531     if (I < NumberOfTokensBeforeRequires)
532       EXPECT_EQ(*BaseTokens[I], *ConstrainedTokens[I]) << I;
533     else
534       EXPECT_EQ(*BaseTokens[I],
535                 *ConstrainedTokens[I + NumberOfAdditionalRequiresClauseTokens])
536           << I;
537 
538   BaseTokens = annotate("template<typename T>\n"
539                         "T foo();");
540   ConstrainedTokens = annotate("template<typename T>\n"
541                                "T foo() requires Foo<T>;");
542   NumberOfBaseTokens = 11u;
543   NumberOfTokensBeforeRequires = 9u;
544 
545   ASSERT_EQ(BaseTokens.size(), NumberOfBaseTokens) << BaseTokens;
546   ASSERT_EQ(ConstrainedTokens.size(),
547             NumberOfBaseTokens + NumberOfAdditionalRequiresClauseTokens)
548       << ConstrainedTokens;
549 
550   for (auto I = 0u; I < NumberOfBaseTokens; ++I)
551     if (I < NumberOfTokensBeforeRequires)
552       EXPECT_EQ(*BaseTokens[I], *ConstrainedTokens[I]) << I;
553     else
554       EXPECT_EQ(*BaseTokens[I],
555                 *ConstrainedTokens[I + NumberOfAdditionalRequiresClauseTokens])
556           << I;
557 
558   BaseTokens = annotate("template<typename T>\n"
559                         "T foo() {\n"
560                         "  auto bar = baz();\n"
561                         "  return bar + T{};\n"
562                         "}");
563   ConstrainedTokens = annotate("template<typename T>\n"
564                                "T foo() requires Foo<T> {\n"
565                                "  auto bar = baz();\n"
566                                "  return bar + T{};\n"
567                                "}");
568   NumberOfBaseTokens = 26u;
569 
570   ASSERT_EQ(BaseTokens.size(), NumberOfBaseTokens) << BaseTokens;
571   ASSERT_EQ(ConstrainedTokens.size(),
572             NumberOfBaseTokens + NumberOfAdditionalRequiresClauseTokens)
573       << ConstrainedTokens;
574 
575   for (auto I = 0u; I < NumberOfBaseTokens; ++I)
576     if (I < NumberOfTokensBeforeRequires)
577       EXPECT_EQ(*BaseTokens[I], *ConstrainedTokens[I]) << I;
578     else
579       EXPECT_EQ(*BaseTokens[I],
580                 *ConstrainedTokens[I + NumberOfAdditionalRequiresClauseTokens])
581           << I;
582 
583   BaseTokens = annotate("template<typename T>\n"
584                         "Bar(T) -> Bar<typename T::I>;");
585   ConstrainedTokens = annotate("template<typename T>\n"
586                                "  requires requires(T &&t) {\n"
587                                "             typename T::I;\n"
588                                "           }\n"
589                                "Bar(T) -> Bar<typename T::I>;");
590   NumberOfBaseTokens = 19u;
591   NumberOfAdditionalRequiresClauseTokens = 14u;
592   NumberOfTokensBeforeRequires = 5u;
593 
594   ASSERT_EQ(BaseTokens.size(), NumberOfBaseTokens) << BaseTokens;
595   ASSERT_EQ(ConstrainedTokens.size(),
596             NumberOfBaseTokens + NumberOfAdditionalRequiresClauseTokens)
597       << ConstrainedTokens;
598 
599   for (auto I = 0u; I < NumberOfBaseTokens; ++I)
600     if (I < NumberOfTokensBeforeRequires)
601       EXPECT_EQ(*BaseTokens[I], *ConstrainedTokens[I]) << I;
602     else
603       EXPECT_EQ(*BaseTokens[I],
604                 *ConstrainedTokens[I + NumberOfAdditionalRequiresClauseTokens])
605           << I;
606 
607   BaseTokens = annotate("struct [[nodiscard]] zero_t {\n"
608                         "  template<class T>\n"
609                         "  [[nodiscard]] constexpr operator T() const { "
610                         "return number_zero_v<T>; }\n"
611                         "};");
612 
613   ConstrainedTokens = annotate("struct [[nodiscard]] zero_t {\n"
614                                "  template<class T>\n"
615                                "    requires requires { number_zero_v<T>; }\n"
616                                "  [[nodiscard]] constexpr operator T() const { "
617                                "return number_zero_v<T>; }\n"
618                                "};");
619   NumberOfBaseTokens = 35u;
620   NumberOfAdditionalRequiresClauseTokens = 9u;
621   NumberOfTokensBeforeRequires = 13u;
622 
623   ASSERT_EQ(BaseTokens.size(), NumberOfBaseTokens) << BaseTokens;
624   ASSERT_EQ(ConstrainedTokens.size(),
625             NumberOfBaseTokens + NumberOfAdditionalRequiresClauseTokens)
626       << ConstrainedTokens;
627 
628   for (auto I = 0u; I < NumberOfBaseTokens; ++I)
629     if (I < NumberOfTokensBeforeRequires)
630       EXPECT_EQ(*BaseTokens[I], *ConstrainedTokens[I]) << I;
631     else
632       EXPECT_EQ(*BaseTokens[I],
633                 *ConstrainedTokens[I + NumberOfAdditionalRequiresClauseTokens])
634           << I;
635 }
636 
637 TEST_F(TokenAnnotatorTest, UnderstandsAsm) {
638   auto Tokens = annotate("__asm{\n"
639                          "a:\n"
640                          "};");
641   ASSERT_EQ(Tokens.size(), 7u) << Tokens;
642   EXPECT_TOKEN(Tokens[0], tok::kw_asm, TT_Unknown);
643   EXPECT_TOKEN(Tokens[1], tok::l_brace, TT_InlineASMBrace);
644   EXPECT_TOKEN(Tokens[4], tok::r_brace, TT_InlineASMBrace);
645 }
646 
647 TEST_F(TokenAnnotatorTest, UnderstandsObjCBlock) {
648   auto Tokens = annotate("int (^)() = ^ ()\n"
649                          "  external_source_symbol() { //\n"
650                          "  return 1;\n"
651                          "};");
652   ASSERT_EQ(Tokens.size(), 21u) << Tokens;
653   EXPECT_TOKEN(Tokens[1], tok::l_paren, TT_ObjCBlockLParen);
654   EXPECT_TOKEN(Tokens[13], tok::l_brace, TT_ObjCBlockLBrace);
655 
656   Tokens = annotate("int *p = ^int*(){ //\n"
657                     "  return nullptr;\n"
658                     "}();");
659   ASSERT_EQ(Tokens.size(), 19u) << Tokens;
660   EXPECT_TOKEN(Tokens[9], tok::l_brace, TT_ObjCBlockLBrace);
661 }
662 
663 } // namespace
664 } // namespace format
665 } // namespace clang
666