1 //===- unittests/AST/CommentLexer.cpp ------ Comment lexer tests ----------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 
9 #include "clang/AST/CommentLexer.h"
10 #include "clang/AST/CommentCommandTraits.h"
11 #include "clang/Basic/CommentOptions.h"
12 #include "clang/Basic/Diagnostic.h"
13 #include "clang/Basic/DiagnosticOptions.h"
14 #include "clang/Basic/FileManager.h"
15 #include "clang/Basic/SourceManager.h"
16 #include "llvm/ADT/STLExtras.h"
17 #include "gtest/gtest.h"
18 #include <vector>
19 
20 using namespace llvm;
21 using namespace clang;
22 
23 namespace clang {
24 namespace comments {
25 
26 namespace {
27 class CommentLexerTest : public ::testing::Test {
28 protected:
29   CommentLexerTest()
30     : FileMgr(FileMgrOpts),
31       DiagID(new DiagnosticIDs()),
32       Diags(DiagID, new DiagnosticOptions, new IgnoringDiagConsumer()),
33       SourceMgr(Diags, FileMgr),
34       Traits(Allocator, CommentOptions()) {
35   }
36 
37   FileSystemOptions FileMgrOpts;
38   FileManager FileMgr;
39   IntrusiveRefCntPtr<DiagnosticIDs> DiagID;
40   DiagnosticsEngine Diags;
41   SourceManager SourceMgr;
42   llvm::BumpPtrAllocator Allocator;
43   CommandTraits Traits;
44 
45   void lexString(const char *Source, std::vector<Token> &Toks);
46 
47   StringRef getCommandName(const Token &Tok) {
48     return Traits.getCommandInfo(Tok.getCommandID())->Name;
49   }
50 
51   StringRef getVerbatimBlockName(const Token &Tok) {
52     return Traits.getCommandInfo(Tok.getVerbatimBlockID())->Name;
53   }
54 
55   StringRef getVerbatimLineName(const Token &Tok) {
56     return Traits.getCommandInfo(Tok.getVerbatimLineID())->Name;
57   }
58 };
59 
60 void CommentLexerTest::lexString(const char *Source,
61                                  std::vector<Token> &Toks) {
62   std::unique_ptr<MemoryBuffer> Buf = MemoryBuffer::getMemBuffer(Source);
63   FileID File = SourceMgr.createFileID(std::move(Buf));
64   SourceLocation Begin = SourceMgr.getLocForStartOfFile(File);
65 
66   Lexer L(Allocator, Diags, Traits, Begin, Source, Source + strlen(Source));
67 
68   while (1) {
69     Token Tok;
70     L.lex(Tok);
71     if (Tok.is(tok::eof))
72       break;
73     Toks.push_back(Tok);
74   }
75 }
76 
77 } // unnamed namespace
78 
79 // Empty source range should be handled.
80 TEST_F(CommentLexerTest, Basic1) {
81   const char *Source = "";
82   std::vector<Token> Toks;
83 
84   lexString(Source, Toks);
85 
86   ASSERT_EQ(0U, Toks.size());
87 }
88 
89 // Empty comments should be handled.
90 TEST_F(CommentLexerTest, Basic2) {
91   const char *Sources[] = {
92     "//", "///", "//!", "///<", "//!<"
93   };
94   for (size_t i = 0, e = array_lengthof(Sources); i != e; i++) {
95     std::vector<Token> Toks;
96 
97     lexString(Sources[i], Toks);
98 
99     ASSERT_EQ(1U, Toks.size());
100 
101     ASSERT_EQ(tok::newline, Toks[0].getKind());
102   }
103 }
104 
105 // Empty comments should be handled.
106 TEST_F(CommentLexerTest, Basic3) {
107   const char *Sources[] = {
108     "/**/", "/***/", "/*!*/", "/**<*/", "/*!<*/"
109   };
110   for (size_t i = 0, e = array_lengthof(Sources); i != e; i++) {
111     std::vector<Token> Toks;
112 
113     lexString(Sources[i], Toks);
114 
115     ASSERT_EQ(2U, Toks.size());
116 
117     ASSERT_EQ(tok::newline, Toks[0].getKind());
118     ASSERT_EQ(tok::newline, Toks[1].getKind());
119   }
120 }
121 
122 // Single comment with plain text.
123 TEST_F(CommentLexerTest, Basic4) {
124   const char *Sources[] = {
125     "// Meow",   "/// Meow",    "//! Meow",
126     "// Meow\n", "// Meow\r\n", "//! Meow\r",
127   };
128 
129   for (size_t i = 0, e = array_lengthof(Sources); i != e; i++) {
130     std::vector<Token> Toks;
131 
132     lexString(Sources[i], Toks);
133 
134     ASSERT_EQ(2U, Toks.size());
135 
136     ASSERT_EQ(tok::text,          Toks[0].getKind());
137     ASSERT_EQ(StringRef(" Meow"), Toks[0].getText());
138 
139     ASSERT_EQ(tok::newline,       Toks[1].getKind());
140   }
141 }
142 
143 // Single comment with plain text.
144 TEST_F(CommentLexerTest, Basic5) {
145   const char *Sources[] = {
146     "/* Meow*/", "/** Meow*/",  "/*! Meow*/"
147   };
148 
149   for (size_t i = 0, e = array_lengthof(Sources); i != e; i++) {
150     std::vector<Token> Toks;
151 
152     lexString(Sources[i], Toks);
153 
154     ASSERT_EQ(3U, Toks.size());
155 
156     ASSERT_EQ(tok::text,          Toks[0].getKind());
157     ASSERT_EQ(StringRef(" Meow"), Toks[0].getText());
158 
159     ASSERT_EQ(tok::newline,       Toks[1].getKind());
160     ASSERT_EQ(tok::newline,       Toks[2].getKind());
161   }
162 }
163 
164 // Test newline escaping.
165 TEST_F(CommentLexerTest, Basic6) {
166   const char *Sources[] = {
167     "// Aaa\\\n"   " Bbb\\ \n"   " Ccc?" "?/\n",
168     "// Aaa\\\r\n" " Bbb\\ \r\n" " Ccc?" "?/\r\n",
169     "// Aaa\\\r"   " Bbb\\ \r"   " Ccc?" "?/\r"
170   };
171 
172   for (size_t i = 0, e = array_lengthof(Sources); i != e; i++) {
173     std::vector<Token> Toks;
174 
175     lexString(Sources[i], Toks);
176 
177     ASSERT_EQ(10U, Toks.size());
178 
179     ASSERT_EQ(tok::text,         Toks[0].getKind());
180     ASSERT_EQ(StringRef(" Aaa"), Toks[0].getText());
181     ASSERT_EQ(tok::text,         Toks[1].getKind());
182     ASSERT_EQ(StringRef("\\"),   Toks[1].getText());
183     ASSERT_EQ(tok::newline,      Toks[2].getKind());
184 
185     ASSERT_EQ(tok::text,         Toks[3].getKind());
186     ASSERT_EQ(StringRef(" Bbb"), Toks[3].getText());
187     ASSERT_EQ(tok::text,         Toks[4].getKind());
188     ASSERT_EQ(StringRef("\\"),   Toks[4].getText());
189     ASSERT_EQ(tok::text,         Toks[5].getKind());
190     ASSERT_EQ(StringRef(" "),    Toks[5].getText());
191     ASSERT_EQ(tok::newline,      Toks[6].getKind());
192 
193     ASSERT_EQ(tok::text,         Toks[7].getKind());
194     ASSERT_EQ(StringRef(" Ccc?" "?/"), Toks[7].getText());
195     ASSERT_EQ(tok::newline,      Toks[8].getKind());
196 
197     ASSERT_EQ(tok::newline,      Toks[9].getKind());
198   }
199 }
200 
201 // Check that we skip C-style aligned stars correctly.
202 TEST_F(CommentLexerTest, Basic7) {
203   const char *Source =
204     "/* Aaa\n"
205     " * Bbb\r\n"
206     "\t* Ccc\n"
207     "  ! Ddd\n"
208     "  * Eee\n"
209     "  ** Fff\n"
210     " */";
211   std::vector<Token> Toks;
212 
213   lexString(Source, Toks);
214 
215   ASSERT_EQ(15U, Toks.size());
216 
217   ASSERT_EQ(tok::text,         Toks[0].getKind());
218   ASSERT_EQ(StringRef(" Aaa"), Toks[0].getText());
219   ASSERT_EQ(tok::newline,      Toks[1].getKind());
220 
221   ASSERT_EQ(tok::text,         Toks[2].getKind());
222   ASSERT_EQ(StringRef(" Bbb"), Toks[2].getText());
223   ASSERT_EQ(tok::newline,      Toks[3].getKind());
224 
225   ASSERT_EQ(tok::text,         Toks[4].getKind());
226   ASSERT_EQ(StringRef(" Ccc"), Toks[4].getText());
227   ASSERT_EQ(tok::newline,      Toks[5].getKind());
228 
229   ASSERT_EQ(tok::text,            Toks[6].getKind());
230   ASSERT_EQ(StringRef("  ! Ddd"), Toks[6].getText());
231   ASSERT_EQ(tok::newline,         Toks[7].getKind());
232 
233   ASSERT_EQ(tok::text,         Toks[8].getKind());
234   ASSERT_EQ(StringRef(" Eee"), Toks[8].getText());
235   ASSERT_EQ(tok::newline,      Toks[9].getKind());
236 
237   ASSERT_EQ(tok::text,          Toks[10].getKind());
238   ASSERT_EQ(StringRef("* Fff"), Toks[10].getText());
239   ASSERT_EQ(tok::newline,       Toks[11].getKind());
240 
241   ASSERT_EQ(tok::text,         Toks[12].getKind());
242   ASSERT_EQ(StringRef(" "),    Toks[12].getText());
243 
244   ASSERT_EQ(tok::newline,      Toks[13].getKind());
245   ASSERT_EQ(tok::newline,      Toks[14].getKind());
246 }
247 
248 // A command marker followed by comment end.
249 TEST_F(CommentLexerTest, DoxygenCommand1) {
250   const char *Sources[] = { "//@", "///@", "//!@" };
251   for (size_t i = 0, e = array_lengthof(Sources); i != e; i++) {
252     std::vector<Token> Toks;
253 
254     lexString(Sources[i], Toks);
255 
256     ASSERT_EQ(2U, Toks.size());
257 
258     ASSERT_EQ(tok::text,          Toks[0].getKind());
259     ASSERT_EQ(StringRef("@"),     Toks[0].getText());
260 
261     ASSERT_EQ(tok::newline,       Toks[1].getKind());
262   }
263 }
264 
265 // A command marker followed by comment end.
266 TEST_F(CommentLexerTest, DoxygenCommand2) {
267   const char *Sources[] = { "/*@*/", "/**@*/", "/*!@*/"};
268   for (size_t i = 0, e = array_lengthof(Sources); i != e; i++) {
269     std::vector<Token> Toks;
270 
271     lexString(Sources[i], Toks);
272 
273     ASSERT_EQ(3U, Toks.size());
274 
275     ASSERT_EQ(tok::text,          Toks[0].getKind());
276     ASSERT_EQ(StringRef("@"),     Toks[0].getText());
277 
278     ASSERT_EQ(tok::newline,       Toks[1].getKind());
279     ASSERT_EQ(tok::newline,       Toks[2].getKind());
280   }
281 }
282 
283 // A command marker followed by comment end.
284 TEST_F(CommentLexerTest, DoxygenCommand3) {
285   const char *Sources[] = { "/*\\*/", "/**\\*/" };
286   for (size_t i = 0, e = array_lengthof(Sources); i != e; i++) {
287     std::vector<Token> Toks;
288 
289     lexString(Sources[i], Toks);
290 
291     ASSERT_EQ(3U, Toks.size());
292 
293     ASSERT_EQ(tok::text,           Toks[0].getKind());
294     ASSERT_EQ(StringRef("\\"),     Toks[0].getText());
295 
296     ASSERT_EQ(tok::newline,        Toks[1].getKind());
297     ASSERT_EQ(tok::newline,        Toks[2].getKind());
298   }
299 }
300 
301 // Doxygen escape sequences.
302 TEST_F(CommentLexerTest, DoxygenCommand4) {
303   const char *Sources[] = {
304     "/// \\\\ \\@ \\& \\$ \\# \\< \\> \\% \\\" \\. \\::",
305     "/// @\\ @@ @& @$ @# @< @> @% @\" @. @::"
306   };
307   const char *Text[] = {
308     " ",
309     "\\", " ", "@", " ", "&", " ", "$",  " ", "#", " ",
310     "<",  " ", ">", " ", "%", " ", "\"", " ", ".", " ",
311     "::", ""
312   };
313 
314   for (size_t i = 0, e = array_lengthof(Sources); i != e; i++) {
315     std::vector<Token> Toks;
316 
317     lexString(Sources[i], Toks);
318 
319     ASSERT_EQ(array_lengthof(Text), Toks.size());
320 
321     for (size_t j = 0, e = Toks.size(); j != e; j++) {
322       if(Toks[j].is(tok::text)) {
323         ASSERT_EQ(StringRef(Text[j]), Toks[j].getText())
324           << "index " << i;
325       }
326     }
327   }
328 }
329 
330 // A command marker followed by a non-letter that is not a part of an escape
331 // sequence.
332 TEST_F(CommentLexerTest, DoxygenCommand5) {
333   const char *Source = "/// \\^ \\0";
334   std::vector<Token> Toks;
335 
336   lexString(Source, Toks);
337 
338   ASSERT_EQ(6U, Toks.size());
339 
340   ASSERT_EQ(tok::text,       Toks[0].getKind());
341   ASSERT_EQ(StringRef(" "),  Toks[0].getText());
342 
343   ASSERT_EQ(tok::text,       Toks[1].getKind());
344   ASSERT_EQ(StringRef("\\"), Toks[1].getText());
345 
346   ASSERT_EQ(tok::text,       Toks[2].getKind());
347   ASSERT_EQ(StringRef("^ "), Toks[2].getText());
348 
349   ASSERT_EQ(tok::text,       Toks[3].getKind());
350   ASSERT_EQ(StringRef("\\"), Toks[3].getText());
351 
352   ASSERT_EQ(tok::text,       Toks[4].getKind());
353   ASSERT_EQ(StringRef("0"),  Toks[4].getText());
354 
355   ASSERT_EQ(tok::newline,    Toks[5].getKind());
356 }
357 
358 TEST_F(CommentLexerTest, DoxygenCommand6) {
359   const char *Source = "/// \\brief Aaa.";
360   std::vector<Token> Toks;
361 
362   lexString(Source, Toks);
363 
364   ASSERT_EQ(4U, Toks.size());
365 
366   ASSERT_EQ(tok::text,          Toks[0].getKind());
367   ASSERT_EQ(StringRef(" "),     Toks[0].getText());
368 
369   ASSERT_EQ(tok::backslash_command, Toks[1].getKind());
370   ASSERT_EQ(StringRef("brief"), getCommandName(Toks[1]));
371 
372   ASSERT_EQ(tok::text,          Toks[2].getKind());
373   ASSERT_EQ(StringRef(" Aaa."), Toks[2].getText());
374 
375   ASSERT_EQ(tok::newline,       Toks[3].getKind());
376 }
377 
378 TEST_F(CommentLexerTest, DoxygenCommand7) {
379   const char *Source = "/// \\em\\em \\em\t\\em\n";
380   std::vector<Token> Toks;
381 
382   lexString(Source, Toks);
383 
384   ASSERT_EQ(8U, Toks.size());
385 
386   ASSERT_EQ(tok::text,       Toks[0].getKind());
387   ASSERT_EQ(StringRef(" "),  Toks[0].getText());
388 
389   ASSERT_EQ(tok::backslash_command, Toks[1].getKind());
390   ASSERT_EQ(StringRef("em"), getCommandName(Toks[1]));
391 
392   ASSERT_EQ(tok::backslash_command, Toks[2].getKind());
393   ASSERT_EQ(StringRef("em"), getCommandName(Toks[2]));
394 
395   ASSERT_EQ(tok::text,       Toks[3].getKind());
396   ASSERT_EQ(StringRef(" "),  Toks[3].getText());
397 
398   ASSERT_EQ(tok::backslash_command, Toks[4].getKind());
399   ASSERT_EQ(StringRef("em"), getCommandName(Toks[4]));
400 
401   ASSERT_EQ(tok::text,       Toks[5].getKind());
402   ASSERT_EQ(StringRef("\t"), Toks[5].getText());
403 
404   ASSERT_EQ(tok::backslash_command, Toks[6].getKind());
405   ASSERT_EQ(StringRef("em"), getCommandName(Toks[6]));
406 
407   ASSERT_EQ(tok::newline,    Toks[7].getKind());
408 }
409 
410 TEST_F(CommentLexerTest, DoxygenCommand8) {
411   const char *Source = "/// @em@em @em\t@em\n";
412   std::vector<Token> Toks;
413 
414   lexString(Source, Toks);
415 
416   ASSERT_EQ(8U, Toks.size());
417 
418   ASSERT_EQ(tok::text,       Toks[0].getKind());
419   ASSERT_EQ(StringRef(" "),  Toks[0].getText());
420 
421   ASSERT_EQ(tok::at_command, Toks[1].getKind());
422   ASSERT_EQ(StringRef("em"), getCommandName(Toks[1]));
423 
424   ASSERT_EQ(tok::at_command, Toks[2].getKind());
425   ASSERT_EQ(StringRef("em"), getCommandName(Toks[2]));
426 
427   ASSERT_EQ(tok::text,       Toks[3].getKind());
428   ASSERT_EQ(StringRef(" "),  Toks[3].getText());
429 
430   ASSERT_EQ(tok::at_command, Toks[4].getKind());
431   ASSERT_EQ(StringRef("em"), getCommandName(Toks[4]));
432 
433   ASSERT_EQ(tok::text,       Toks[5].getKind());
434   ASSERT_EQ(StringRef("\t"), Toks[5].getText());
435 
436   ASSERT_EQ(tok::at_command, Toks[6].getKind());
437   ASSERT_EQ(StringRef("em"), getCommandName(Toks[6]));
438 
439   ASSERT_EQ(tok::newline,    Toks[7].getKind());
440 }
441 
442 TEST_F(CommentLexerTest, DoxygenCommand9) {
443   const char *Source = "/// \\aaa\\bbb \\ccc\t\\ddd\n";
444   std::vector<Token> Toks;
445 
446   lexString(Source, Toks);
447 
448   ASSERT_EQ(8U, Toks.size());
449 
450   ASSERT_EQ(tok::text,        Toks[0].getKind());
451   ASSERT_EQ(StringRef(" "),   Toks[0].getText());
452 
453   ASSERT_EQ(tok::unknown_command, Toks[1].getKind());
454   ASSERT_EQ(StringRef("aaa"), Toks[1].getUnknownCommandName());
455 
456   ASSERT_EQ(tok::unknown_command, Toks[2].getKind());
457   ASSERT_EQ(StringRef("bbb"), Toks[2].getUnknownCommandName());
458 
459   ASSERT_EQ(tok::text,        Toks[3].getKind());
460   ASSERT_EQ(StringRef(" "),   Toks[3].getText());
461 
462   ASSERT_EQ(tok::unknown_command, Toks[4].getKind());
463   ASSERT_EQ(StringRef("ccc"), Toks[4].getUnknownCommandName());
464 
465   ASSERT_EQ(tok::text,        Toks[5].getKind());
466   ASSERT_EQ(StringRef("\t"),  Toks[5].getText());
467 
468   ASSERT_EQ(tok::unknown_command, Toks[6].getKind());
469   ASSERT_EQ(StringRef("ddd"), Toks[6].getUnknownCommandName());
470 
471   ASSERT_EQ(tok::newline,     Toks[7].getKind());
472 }
473 
474 TEST_F(CommentLexerTest, DoxygenCommand10) {
475   const char *Source = "// \\c\n";
476   std::vector<Token> Toks;
477 
478   lexString(Source, Toks);
479 
480   ASSERT_EQ(3U, Toks.size());
481 
482   ASSERT_EQ(tok::text,      Toks[0].getKind());
483   ASSERT_EQ(StringRef(" "), Toks[0].getText());
484 
485   ASSERT_EQ(tok::backslash_command, Toks[1].getKind());
486   ASSERT_EQ(StringRef("c"), getCommandName(Toks[1]));
487 
488   ASSERT_EQ(tok::newline,   Toks[2].getKind());
489 }
490 
491 TEST_F(CommentLexerTest, RegisterCustomBlockCommand) {
492   const char *Source =
493     "/// \\NewBlockCommand Aaa.\n"
494     "/// @NewBlockCommand Aaa.\n";
495 
496   Traits.registerBlockCommand(StringRef("NewBlockCommand"));
497 
498   std::vector<Token> Toks;
499 
500   lexString(Source, Toks);
501 
502   ASSERT_EQ(8U, Toks.size());
503 
504   ASSERT_EQ(tok::text,          Toks[0].getKind());
505   ASSERT_EQ(StringRef(" "),     Toks[0].getText());
506 
507   ASSERT_EQ(tok::backslash_command, Toks[1].getKind());
508   ASSERT_EQ(StringRef("NewBlockCommand"), getCommandName(Toks[1]));
509 
510   ASSERT_EQ(tok::text,          Toks[2].getKind());
511   ASSERT_EQ(StringRef(" Aaa."), Toks[2].getText());
512 
513   ASSERT_EQ(tok::newline,       Toks[3].getKind());
514 
515   ASSERT_EQ(tok::text,          Toks[4].getKind());
516   ASSERT_EQ(StringRef(" "),     Toks[4].getText());
517 
518   ASSERT_EQ(tok::at_command,    Toks[5].getKind());
519   ASSERT_EQ(StringRef("NewBlockCommand"), getCommandName(Toks[5]));
520 
521   ASSERT_EQ(tok::text,          Toks[6].getKind());
522   ASSERT_EQ(StringRef(" Aaa."), Toks[6].getText());
523 
524   ASSERT_EQ(tok::newline,       Toks[7].getKind());
525 }
526 
527 TEST_F(CommentLexerTest, RegisterMultipleBlockCommands) {
528   const char *Source =
529     "/// \\Foo\n"
530     "/// \\Bar Baz\n"
531     "/// \\Blech quux=corge\n";
532 
533   Traits.registerBlockCommand(StringRef("Foo"));
534   Traits.registerBlockCommand(StringRef("Bar"));
535   Traits.registerBlockCommand(StringRef("Blech"));
536 
537   std::vector<Token> Toks;
538 
539   lexString(Source, Toks);
540 
541   ASSERT_EQ(11U, Toks.size());
542 
543   ASSERT_EQ(tok::text,      Toks[0].getKind());
544   ASSERT_EQ(StringRef(" "), Toks[0].getText());
545 
546   ASSERT_EQ(tok::backslash_command, Toks[1].getKind());
547   ASSERT_EQ(StringRef("Foo"), getCommandName(Toks[1]));
548 
549   ASSERT_EQ(tok::newline,     Toks[2].getKind());
550 
551   ASSERT_EQ(tok::text,      Toks[3].getKind());
552   ASSERT_EQ(StringRef(" "), Toks[3].getText());
553 
554   ASSERT_EQ(tok::backslash_command, Toks[4].getKind());
555   ASSERT_EQ(StringRef("Bar"), getCommandName(Toks[4]));
556 
557   ASSERT_EQ(tok::text,         Toks[5].getKind());
558   ASSERT_EQ(StringRef(" Baz"), Toks[5].getText());
559 
560   ASSERT_EQ(tok::newline,     Toks[6].getKind());
561 
562   ASSERT_EQ(tok::text,      Toks[7].getKind());
563   ASSERT_EQ(StringRef(" "), Toks[7].getText());
564 
565   ASSERT_EQ(tok::backslash_command, Toks[8].getKind());
566   ASSERT_EQ(StringRef("Blech"), getCommandName(Toks[8]));
567 
568   ASSERT_EQ(tok::text,                Toks[9].getKind());
569   ASSERT_EQ(StringRef(" quux=corge"), Toks[9].getText());
570 
571   ASSERT_EQ(tok::newline,     Toks[10].getKind());
572 }
573 
574 // Empty verbatim block.
575 TEST_F(CommentLexerTest, VerbatimBlock1) {
576   const char *Sources[] = {
577     "/// \\verbatim\\endverbatim\n//",
578     "/** \\verbatim\\endverbatim*/"
579   };
580 
581   for (size_t i = 0, e = array_lengthof(Sources); i != e; i++) {
582     std::vector<Token> Toks;
583 
584     lexString(Sources[i], Toks);
585 
586     ASSERT_EQ(5U, Toks.size());
587 
588     ASSERT_EQ(tok::text,                 Toks[0].getKind());
589     ASSERT_EQ(StringRef(" "),            Toks[0].getText());
590 
591     ASSERT_EQ(tok::verbatim_block_begin, Toks[1].getKind());
592     ASSERT_EQ(StringRef("verbatim"),     getVerbatimBlockName(Toks[1]));
593 
594     ASSERT_EQ(tok::verbatim_block_end,   Toks[2].getKind());
595     ASSERT_EQ(StringRef("endverbatim"),  getVerbatimBlockName(Toks[2]));
596 
597     ASSERT_EQ(tok::newline,              Toks[3].getKind());
598     ASSERT_EQ(tok::newline,              Toks[4].getKind());
599   }
600 }
601 
602 // Empty verbatim block without an end command.
603 TEST_F(CommentLexerTest, VerbatimBlock2) {
604   const char *Source = "/// \\verbatim";
605 
606   std::vector<Token> Toks;
607 
608   lexString(Source, Toks);
609 
610   ASSERT_EQ(3U, Toks.size());
611 
612   ASSERT_EQ(tok::text,                 Toks[0].getKind());
613   ASSERT_EQ(StringRef(" "),            Toks[0].getText());
614 
615   ASSERT_EQ(tok::verbatim_block_begin, Toks[1].getKind());
616   ASSERT_EQ(StringRef("verbatim"),     getVerbatimBlockName(Toks[1]));
617 
618   ASSERT_EQ(tok::newline,              Toks[2].getKind());
619 }
620 
621 // Empty verbatim block without an end command.
622 TEST_F(CommentLexerTest, VerbatimBlock3) {
623   const char *Source = "/** \\verbatim*/";
624 
625   std::vector<Token> Toks;
626 
627   lexString(Source, Toks);
628 
629   ASSERT_EQ(4U, Toks.size());
630 
631   ASSERT_EQ(tok::text,                 Toks[0].getKind());
632   ASSERT_EQ(StringRef(" "),            Toks[0].getText());
633 
634   ASSERT_EQ(tok::verbatim_block_begin, Toks[1].getKind());
635   ASSERT_EQ(StringRef("verbatim"),     getVerbatimBlockName(Toks[1]));
636 
637   ASSERT_EQ(tok::newline,              Toks[2].getKind());
638   ASSERT_EQ(tok::newline,              Toks[3].getKind());
639 }
640 
641 // Single-line verbatim block.
642 TEST_F(CommentLexerTest, VerbatimBlock4) {
643   const char *Sources[] = {
644     "/// Meow \\verbatim aaa \\endverbatim\n//",
645     "/** Meow \\verbatim aaa \\endverbatim*/"
646   };
647 
648   for (size_t i = 0, e = array_lengthof(Sources); i != e; i++) {
649     std::vector<Token> Toks;
650 
651     lexString(Sources[i], Toks);
652 
653     ASSERT_EQ(6U, Toks.size());
654 
655     ASSERT_EQ(tok::text,                 Toks[0].getKind());
656     ASSERT_EQ(StringRef(" Meow "),       Toks[0].getText());
657 
658     ASSERT_EQ(tok::verbatim_block_begin, Toks[1].getKind());
659     ASSERT_EQ(StringRef("verbatim"),     getVerbatimBlockName(Toks[1]));
660 
661     ASSERT_EQ(tok::verbatim_block_line,  Toks[2].getKind());
662     ASSERT_EQ(StringRef(" aaa "),        Toks[2].getVerbatimBlockText());
663 
664     ASSERT_EQ(tok::verbatim_block_end,   Toks[3].getKind());
665     ASSERT_EQ(StringRef("endverbatim"),  getVerbatimBlockName(Toks[3]));
666 
667     ASSERT_EQ(tok::newline,              Toks[4].getKind());
668     ASSERT_EQ(tok::newline,              Toks[5].getKind());
669   }
670 }
671 
672 // Single-line verbatim block without an end command.
673 TEST_F(CommentLexerTest, VerbatimBlock5) {
674   const char *Sources[] = {
675     "/// Meow \\verbatim aaa \n//",
676     "/** Meow \\verbatim aaa */"
677   };
678 
679   for (size_t i = 0, e = array_lengthof(Sources); i != e; i++) {
680     std::vector<Token> Toks;
681 
682     lexString(Sources[i], Toks);
683 
684     ASSERT_EQ(5U, Toks.size());
685 
686     ASSERT_EQ(tok::text,                 Toks[0].getKind());
687     ASSERT_EQ(StringRef(" Meow "),       Toks[0].getText());
688 
689     ASSERT_EQ(tok::verbatim_block_begin, Toks[1].getKind());
690     ASSERT_EQ(StringRef("verbatim"),     getVerbatimBlockName(Toks[1]));
691 
692     ASSERT_EQ(tok::verbatim_block_line,  Toks[2].getKind());
693     ASSERT_EQ(StringRef(" aaa "),        Toks[2].getVerbatimBlockText());
694 
695     ASSERT_EQ(tok::newline,              Toks[3].getKind());
696     ASSERT_EQ(tok::newline,              Toks[4].getKind());
697   }
698 }
699 
700 TEST_F(CommentLexerTest, VerbatimBlock6) {
701   const char *Source =
702     "// \\verbatim\n"
703     "// Aaa\n"
704     "//\n"
705     "// Bbb\n"
706     "// \\endverbatim\n";
707 
708   std::vector<Token> Toks;
709 
710   lexString(Source, Toks);
711 
712   ASSERT_EQ(10U, Toks.size());
713 
714   ASSERT_EQ(tok::text,                 Toks[0].getKind());
715   ASSERT_EQ(StringRef(" "),            Toks[0].getText());
716 
717   ASSERT_EQ(tok::verbatim_block_begin, Toks[1].getKind());
718   ASSERT_EQ(StringRef("verbatim"),     getVerbatimBlockName(Toks[1]));
719 
720   ASSERT_EQ(tok::newline,              Toks[2].getKind());
721 
722   ASSERT_EQ(tok::verbatim_block_line,  Toks[3].getKind());
723   ASSERT_EQ(StringRef(" Aaa"),         Toks[3].getVerbatimBlockText());
724 
725   ASSERT_EQ(tok::newline,              Toks[4].getKind());
726 
727   ASSERT_EQ(tok::newline,              Toks[5].getKind());
728 
729   ASSERT_EQ(tok::verbatim_block_line,  Toks[6].getKind());
730   ASSERT_EQ(StringRef(" Bbb"),         Toks[6].getVerbatimBlockText());
731 
732   ASSERT_EQ(tok::newline,              Toks[7].getKind());
733 
734   ASSERT_EQ(tok::verbatim_block_end,   Toks[8].getKind());
735   ASSERT_EQ(StringRef("endverbatim"),  getVerbatimBlockName(Toks[8]));
736 
737   ASSERT_EQ(tok::newline,              Toks[9].getKind());
738 }
739 
740 TEST_F(CommentLexerTest, VerbatimBlock7) {
741   const char *Source =
742     "/* \\verbatim\n"
743     " * Aaa\n"
744     " *\n"
745     " * Bbb\n"
746     " * \\endverbatim\n"
747     " */";
748 
749   std::vector<Token> Toks;
750 
751   lexString(Source, Toks);
752 
753   ASSERT_EQ(10U, Toks.size());
754 
755   ASSERT_EQ(tok::text,                 Toks[0].getKind());
756   ASSERT_EQ(StringRef(" "),            Toks[0].getText());
757 
758   ASSERT_EQ(tok::verbatim_block_begin, Toks[1].getKind());
759   ASSERT_EQ(StringRef("verbatim"),     getVerbatimBlockName(Toks[1]));
760 
761   ASSERT_EQ(tok::verbatim_block_line,  Toks[2].getKind());
762   ASSERT_EQ(StringRef(" Aaa"),         Toks[2].getVerbatimBlockText());
763 
764   ASSERT_EQ(tok::verbatim_block_line,  Toks[3].getKind());
765   ASSERT_EQ(StringRef(""),             Toks[3].getVerbatimBlockText());
766 
767   ASSERT_EQ(tok::verbatim_block_line,  Toks[4].getKind());
768   ASSERT_EQ(StringRef(" Bbb"),         Toks[4].getVerbatimBlockText());
769 
770   ASSERT_EQ(tok::verbatim_block_end,   Toks[5].getKind());
771   ASSERT_EQ(StringRef("endverbatim"),  getVerbatimBlockName(Toks[5]));
772 
773   ASSERT_EQ(tok::newline,              Toks[6].getKind());
774 
775   ASSERT_EQ(tok::text,                 Toks[7].getKind());
776   ASSERT_EQ(StringRef(" "),            Toks[7].getText());
777 
778   ASSERT_EQ(tok::newline,              Toks[8].getKind());
779   ASSERT_EQ(tok::newline,              Toks[9].getKind());
780 }
781 
782 // Complex test for verbatim blocks.
783 TEST_F(CommentLexerTest, VerbatimBlock8) {
784   const char *Source =
785     "/* Meow \\verbatim aaa\\$\\@\n"
786     "bbb \\endverbati\r"
787     "ccc\r\n"
788     "ddd \\endverbatim Blah \\verbatim eee\n"
789     "\\endverbatim BlahBlah*/";
790   std::vector<Token> Toks;
791 
792   lexString(Source, Toks);
793 
794   ASSERT_EQ(14U, Toks.size());
795 
796   ASSERT_EQ(tok::text,                 Toks[0].getKind());
797   ASSERT_EQ(StringRef(" Meow "),       Toks[0].getText());
798 
799   ASSERT_EQ(tok::verbatim_block_begin, Toks[1].getKind());
800   ASSERT_EQ(StringRef("verbatim"),     getVerbatimBlockName(Toks[1]));
801 
802   ASSERT_EQ(tok::verbatim_block_line,  Toks[2].getKind());
803   ASSERT_EQ(StringRef(" aaa\\$\\@"),   Toks[2].getVerbatimBlockText());
804 
805   ASSERT_EQ(tok::verbatim_block_line,  Toks[3].getKind());
806   ASSERT_EQ(StringRef("bbb \\endverbati"), Toks[3].getVerbatimBlockText());
807 
808   ASSERT_EQ(tok::verbatim_block_line,  Toks[4].getKind());
809   ASSERT_EQ(StringRef("ccc"),          Toks[4].getVerbatimBlockText());
810 
811   ASSERT_EQ(tok::verbatim_block_line,  Toks[5].getKind());
812   ASSERT_EQ(StringRef("ddd "),         Toks[5].getVerbatimBlockText());
813 
814   ASSERT_EQ(tok::verbatim_block_end,   Toks[6].getKind());
815   ASSERT_EQ(StringRef("endverbatim"),  getVerbatimBlockName(Toks[6]));
816 
817   ASSERT_EQ(tok::text,                 Toks[7].getKind());
818   ASSERT_EQ(StringRef(" Blah "),       Toks[7].getText());
819 
820   ASSERT_EQ(tok::verbatim_block_begin, Toks[8].getKind());
821   ASSERT_EQ(StringRef("verbatim"),     getVerbatimBlockName(Toks[8]));
822 
823   ASSERT_EQ(tok::verbatim_block_line,  Toks[9].getKind());
824   ASSERT_EQ(StringRef(" eee"),         Toks[9].getVerbatimBlockText());
825 
826   ASSERT_EQ(tok::verbatim_block_end,   Toks[10].getKind());
827   ASSERT_EQ(StringRef("endverbatim"),  getVerbatimBlockName(Toks[10]));
828 
829   ASSERT_EQ(tok::text,                 Toks[11].getKind());
830   ASSERT_EQ(StringRef(" BlahBlah"),    Toks[11].getText());
831 
832   ASSERT_EQ(tok::newline,              Toks[12].getKind());
833   ASSERT_EQ(tok::newline,              Toks[13].getKind());
834 }
835 
836 // LaTeX verbatim blocks.
837 TEST_F(CommentLexerTest, VerbatimBlock9) {
838   const char *Source =
839     "/// \\f$ Aaa \\f$ \\f[ Bbb \\f] \\f{ Ccc \\f} \\f( Ddd \\f)";
840   std::vector<Token> Toks;
841 
842   lexString(Source, Toks);
843 
844   ASSERT_EQ(17U, Toks.size());
845 
846   ASSERT_EQ(tok::text,                 Toks[0].getKind());
847   ASSERT_EQ(StringRef(" "),            Toks[0].getText());
848 
849   ASSERT_EQ(tok::verbatim_block_begin, Toks[1].getKind());
850   ASSERT_EQ(StringRef("f$"),           getVerbatimBlockName(Toks[1]));
851 
852   ASSERT_EQ(tok::verbatim_block_line,  Toks[2].getKind());
853   ASSERT_EQ(StringRef(" Aaa "),        Toks[2].getVerbatimBlockText());
854 
855   ASSERT_EQ(tok::verbatim_block_end,   Toks[3].getKind());
856   ASSERT_EQ(StringRef("f$"),           getVerbatimBlockName(Toks[3]));
857 
858   ASSERT_EQ(tok::text,                 Toks[4].getKind());
859   ASSERT_EQ(StringRef(" "),            Toks[4].getText());
860 
861   ASSERT_EQ(tok::verbatim_block_begin, Toks[5].getKind());
862   ASSERT_EQ(StringRef("f["),           getVerbatimBlockName(Toks[5]));
863 
864   ASSERT_EQ(tok::verbatim_block_line,  Toks[6].getKind());
865   ASSERT_EQ(StringRef(" Bbb "),        Toks[6].getVerbatimBlockText());
866 
867   ASSERT_EQ(tok::verbatim_block_end,   Toks[7].getKind());
868   ASSERT_EQ(StringRef("f]"),           getVerbatimBlockName(Toks[7]));
869 
870   ASSERT_EQ(tok::text,                 Toks[8].getKind());
871   ASSERT_EQ(StringRef(" "),            Toks[8].getText());
872 
873   ASSERT_EQ(tok::verbatim_block_begin, Toks[9].getKind());
874   ASSERT_EQ(StringRef("f{"),           getVerbatimBlockName(Toks[9]));
875 
876   ASSERT_EQ(tok::verbatim_block_line,  Toks[10].getKind());
877   ASSERT_EQ(StringRef(" Ccc "),        Toks[10].getVerbatimBlockText());
878 
879   ASSERT_EQ(tok::verbatim_block_end,   Toks[11].getKind());
880   ASSERT_EQ(StringRef("f}"),           getVerbatimBlockName(Toks[11]));
881 
882   ASSERT_EQ(tok::text,                 Toks[12].getKind());
883   ASSERT_EQ(StringRef(" "),            Toks[12].getText());
884 
885   ASSERT_EQ(tok::verbatim_block_begin, Toks[13].getKind());
886   ASSERT_EQ(StringRef("f("),           getVerbatimBlockName(Toks[13]));
887 
888   ASSERT_EQ(tok::verbatim_block_line,  Toks[14].getKind());
889   ASSERT_EQ(StringRef(" Ddd "),        Toks[14].getVerbatimBlockText());
890 
891   ASSERT_EQ(tok::verbatim_block_end,   Toks[15].getKind());
892   ASSERT_EQ(StringRef("f)"),           getVerbatimBlockName(Toks[15]));
893 
894   ASSERT_EQ(tok::newline,              Toks[16].getKind());
895 }
896 
897 // Empty verbatim line.
898 TEST_F(CommentLexerTest, VerbatimLine1) {
899   const char *Sources[] = {
900     "/// \\fn\n//",
901     "/** \\fn*/"
902   };
903 
904   for (size_t i = 0, e = array_lengthof(Sources); i != e; i++) {
905     std::vector<Token> Toks;
906 
907     lexString(Sources[i], Toks);
908 
909     ASSERT_EQ(4U, Toks.size());
910 
911     ASSERT_EQ(tok::text,               Toks[0].getKind());
912     ASSERT_EQ(StringRef(" "),          Toks[0].getText());
913 
914     ASSERT_EQ(tok::verbatim_line_name, Toks[1].getKind());
915     ASSERT_EQ(StringRef("fn"),         getVerbatimLineName(Toks[1]));
916 
917     ASSERT_EQ(tok::newline,            Toks[2].getKind());
918     ASSERT_EQ(tok::newline,            Toks[3].getKind());
919   }
920 }
921 
922 // Verbatim line with Doxygen escape sequences, which should not be expanded.
923 TEST_F(CommentLexerTest, VerbatimLine2) {
924   const char *Sources[] = {
925     "/// \\fn void *foo(const char *zzz = \"\\$\");\n//",
926     "/** \\fn void *foo(const char *zzz = \"\\$\");*/"
927   };
928 
929   for (size_t i = 0, e = array_lengthof(Sources); i != e; i++) {
930     std::vector<Token> Toks;
931 
932     lexString(Sources[i], Toks);
933 
934     ASSERT_EQ(5U, Toks.size());
935 
936     ASSERT_EQ(tok::text,               Toks[0].getKind());
937     ASSERT_EQ(StringRef(" "),          Toks[0].getText());
938 
939     ASSERT_EQ(tok::verbatim_line_name, Toks[1].getKind());
940     ASSERT_EQ(StringRef("fn"),         getVerbatimLineName(Toks[1]));
941 
942     ASSERT_EQ(tok::verbatim_line_text, Toks[2].getKind());
943     ASSERT_EQ(StringRef(" void *foo(const char *zzz = \"\\$\");"),
944                                        Toks[2].getVerbatimLineText());
945 
946     ASSERT_EQ(tok::newline,            Toks[3].getKind());
947     ASSERT_EQ(tok::newline,            Toks[4].getKind());
948   }
949 }
950 
951 // Verbatim line should not eat anything from next source line.
952 TEST_F(CommentLexerTest, VerbatimLine3) {
953   const char *Source =
954     "/** \\fn void *foo(const char *zzz = \"\\$\");\n"
955     " * Meow\n"
956     " */";
957 
958   std::vector<Token> Toks;
959 
960   lexString(Source, Toks);
961 
962   ASSERT_EQ(9U, Toks.size());
963 
964   ASSERT_EQ(tok::text,               Toks[0].getKind());
965   ASSERT_EQ(StringRef(" "),          Toks[0].getText());
966 
967   ASSERT_EQ(tok::verbatim_line_name, Toks[1].getKind());
968   ASSERT_EQ(StringRef("fn"),         getVerbatimLineName(Toks[1]));
969 
970   ASSERT_EQ(tok::verbatim_line_text, Toks[2].getKind());
971   ASSERT_EQ(StringRef(" void *foo(const char *zzz = \"\\$\");"),
972                                      Toks[2].getVerbatimLineText());
973   ASSERT_EQ(tok::newline,            Toks[3].getKind());
974 
975   ASSERT_EQ(tok::text,               Toks[4].getKind());
976   ASSERT_EQ(StringRef(" Meow"),      Toks[4].getText());
977   ASSERT_EQ(tok::newline,            Toks[5].getKind());
978 
979   ASSERT_EQ(tok::text,               Toks[6].getKind());
980   ASSERT_EQ(StringRef(" "),          Toks[6].getText());
981 
982   ASSERT_EQ(tok::newline,            Toks[7].getKind());
983   ASSERT_EQ(tok::newline,            Toks[8].getKind());
984 }
985 
986 TEST_F(CommentLexerTest, HTML1) {
987   const char *Source =
988     "// <";
989 
990   std::vector<Token> Toks;
991 
992   lexString(Source, Toks);
993 
994   ASSERT_EQ(3U, Toks.size());
995 
996   ASSERT_EQ(tok::text,      Toks[0].getKind());
997   ASSERT_EQ(StringRef(" "), Toks[0].getText());
998 
999   ASSERT_EQ(tok::text,      Toks[1].getKind());
1000   ASSERT_EQ(StringRef("<"), Toks[1].getText());
1001 
1002   ASSERT_EQ(tok::newline,   Toks[2].getKind());
1003 }
1004 
1005 TEST_F(CommentLexerTest, HTML2) {
1006   const char *Source =
1007     "// a<2";
1008 
1009   std::vector<Token> Toks;
1010 
1011   lexString(Source, Toks);
1012 
1013   ASSERT_EQ(4U, Toks.size());
1014 
1015   ASSERT_EQ(tok::text,       Toks[0].getKind());
1016   ASSERT_EQ(StringRef(" a"), Toks[0].getText());
1017 
1018   ASSERT_EQ(tok::text,       Toks[1].getKind());
1019   ASSERT_EQ(StringRef("<"),  Toks[1].getText());
1020 
1021   ASSERT_EQ(tok::text,       Toks[2].getKind());
1022   ASSERT_EQ(StringRef("2"),  Toks[2].getText());
1023 
1024   ASSERT_EQ(tok::newline,    Toks[3].getKind());
1025 }
1026 
1027 TEST_F(CommentLexerTest, HTML3) {
1028   const char *Source =
1029     "// < img";
1030 
1031   std::vector<Token> Toks;
1032 
1033   lexString(Source, Toks);
1034 
1035   ASSERT_EQ(4U, Toks.size());
1036 
1037   ASSERT_EQ(tok::text,         Toks[0].getKind());
1038   ASSERT_EQ(StringRef(" "),    Toks[0].getText());
1039 
1040   ASSERT_EQ(tok::text,         Toks[1].getKind());
1041   ASSERT_EQ(StringRef("<"),    Toks[1].getText());
1042 
1043   ASSERT_EQ(tok::text,         Toks[2].getKind());
1044   ASSERT_EQ(StringRef(" img"), Toks[2].getText());
1045 
1046   ASSERT_EQ(tok::newline,      Toks[3].getKind());
1047 }
1048 
1049 TEST_F(CommentLexerTest, HTML4) {
1050   const char *Sources[] = {
1051     "// <img",
1052     "// <img "
1053   };
1054 
1055   for (size_t i = 0, e = array_lengthof(Sources); i != e; i++) {
1056     std::vector<Token> Toks;
1057 
1058     lexString(Sources[i], Toks);
1059 
1060     ASSERT_EQ(3U, Toks.size());
1061 
1062     ASSERT_EQ(tok::text,           Toks[0].getKind());
1063     ASSERT_EQ(StringRef(" "),      Toks[0].getText());
1064 
1065     ASSERT_EQ(tok::html_start_tag, Toks[1].getKind());
1066     ASSERT_EQ(StringRef("img"),    Toks[1].getHTMLTagStartName());
1067 
1068     ASSERT_EQ(tok::newline,        Toks[2].getKind());
1069   }
1070 }
1071 
1072 TEST_F(CommentLexerTest, HTML5) {
1073   const char *Source =
1074     "// <img 42";
1075 
1076   std::vector<Token> Toks;
1077 
1078   lexString(Source, Toks);
1079 
1080   ASSERT_EQ(4U, Toks.size());
1081 
1082   ASSERT_EQ(tok::text,           Toks[0].getKind());
1083   ASSERT_EQ(StringRef(" "),      Toks[0].getText());
1084 
1085   ASSERT_EQ(tok::html_start_tag, Toks[1].getKind());
1086   ASSERT_EQ(StringRef("img"),    Toks[1].getHTMLTagStartName());
1087 
1088   ASSERT_EQ(tok::text,           Toks[2].getKind());
1089   ASSERT_EQ(StringRef("42"),     Toks[2].getText());
1090 
1091   ASSERT_EQ(tok::newline,        Toks[3].getKind());
1092 }
1093 
1094 TEST_F(CommentLexerTest, HTML6) {
1095   const char *Source = "// <img> Meow";
1096 
1097   std::vector<Token> Toks;
1098 
1099   lexString(Source, Toks);
1100 
1101   ASSERT_EQ(5U, Toks.size());
1102 
1103   ASSERT_EQ(tok::text,           Toks[0].getKind());
1104   ASSERT_EQ(StringRef(" "),      Toks[0].getText());
1105 
1106   ASSERT_EQ(tok::html_start_tag, Toks[1].getKind());
1107   ASSERT_EQ(StringRef("img"),    Toks[1].getHTMLTagStartName());
1108 
1109   ASSERT_EQ(tok::html_greater,   Toks[2].getKind());
1110 
1111   ASSERT_EQ(tok::text,           Toks[3].getKind());
1112   ASSERT_EQ(StringRef(" Meow"),  Toks[3].getText());
1113 
1114   ASSERT_EQ(tok::newline,        Toks[4].getKind());
1115 }
1116 
1117 TEST_F(CommentLexerTest, HTML7) {
1118   const char *Source = "// <img=";
1119 
1120   std::vector<Token> Toks;
1121 
1122   lexString(Source, Toks);
1123 
1124   ASSERT_EQ(4U, Toks.size());
1125 
1126   ASSERT_EQ(tok::text,           Toks[0].getKind());
1127   ASSERT_EQ(StringRef(" "),      Toks[0].getText());
1128 
1129   ASSERT_EQ(tok::html_start_tag, Toks[1].getKind());
1130   ASSERT_EQ(StringRef("img"),    Toks[1].getHTMLTagStartName());
1131 
1132   ASSERT_EQ(tok::text,           Toks[2].getKind());
1133   ASSERT_EQ(StringRef("="),      Toks[2].getText());
1134 
1135   ASSERT_EQ(tok::newline,        Toks[3].getKind());
1136 }
1137 
1138 TEST_F(CommentLexerTest, HTML8) {
1139   const char *Source = "// <img src=> Meow";
1140 
1141   std::vector<Token> Toks;
1142 
1143   lexString(Source, Toks);
1144 
1145   ASSERT_EQ(7U, Toks.size());
1146 
1147   ASSERT_EQ(tok::text,           Toks[0].getKind());
1148   ASSERT_EQ(StringRef(" "),      Toks[0].getText());
1149 
1150   ASSERT_EQ(tok::html_start_tag, Toks[1].getKind());
1151   ASSERT_EQ(StringRef("img"),    Toks[1].getHTMLTagStartName());
1152 
1153   ASSERT_EQ(tok::html_ident,     Toks[2].getKind());
1154   ASSERT_EQ(StringRef("src"),   Toks[2].getHTMLIdent());
1155 
1156   ASSERT_EQ(tok::html_equals,    Toks[3].getKind());
1157 
1158   ASSERT_EQ(tok::html_greater,   Toks[4].getKind());
1159 
1160   ASSERT_EQ(tok::text,           Toks[5].getKind());
1161   ASSERT_EQ(StringRef(" Meow"),  Toks[5].getText());
1162 
1163   ASSERT_EQ(tok::newline,        Toks[6].getKind());
1164 }
1165 
1166 TEST_F(CommentLexerTest, HTML9) {
1167   const char *Sources[] = {
1168     "// <img src",
1169     "// <img src "
1170   };
1171 
1172   for (size_t i = 0, e = array_lengthof(Sources); i != e; i++) {
1173     std::vector<Token> Toks;
1174 
1175     lexString(Sources[i], Toks);
1176 
1177     ASSERT_EQ(4U, Toks.size());
1178 
1179     ASSERT_EQ(tok::text,           Toks[0].getKind());
1180     ASSERT_EQ(StringRef(" "),      Toks[0].getText());
1181 
1182     ASSERT_EQ(tok::html_start_tag, Toks[1].getKind());
1183     ASSERT_EQ(StringRef("img"),    Toks[1].getHTMLTagStartName());
1184 
1185     ASSERT_EQ(tok::html_ident,     Toks[2].getKind());
1186     ASSERT_EQ(StringRef("src"),    Toks[2].getHTMLIdent());
1187 
1188     ASSERT_EQ(tok::newline,        Toks[3].getKind());
1189   }
1190 }
1191 
1192 TEST_F(CommentLexerTest, HTML10) {
1193   const char *Sources[] = {
1194     "// <img src=",
1195     "// <img src ="
1196   };
1197 
1198   for (size_t i = 0, e = array_lengthof(Sources); i != e; i++) {
1199     std::vector<Token> Toks;
1200 
1201     lexString(Sources[i], Toks);
1202 
1203     ASSERT_EQ(5U, Toks.size());
1204 
1205     ASSERT_EQ(tok::text,           Toks[0].getKind());
1206     ASSERT_EQ(StringRef(" "),      Toks[0].getText());
1207 
1208     ASSERT_EQ(tok::html_start_tag, Toks[1].getKind());
1209     ASSERT_EQ(StringRef("img"),    Toks[1].getHTMLTagStartName());
1210 
1211     ASSERT_EQ(tok::html_ident,     Toks[2].getKind());
1212     ASSERT_EQ(StringRef("src"),    Toks[2].getHTMLIdent());
1213 
1214     ASSERT_EQ(tok::html_equals,    Toks[3].getKind());
1215 
1216     ASSERT_EQ(tok::newline,        Toks[4].getKind());
1217   }
1218 }
1219 
1220 TEST_F(CommentLexerTest, HTML11) {
1221   const char *Sources[] = {
1222     "// <img src=\"",
1223     "// <img src = \"",
1224     "// <img src=\'",
1225     "// <img src = \'"
1226   };
1227 
1228   for (size_t i = 0, e = array_lengthof(Sources); i != e; i++) {
1229     std::vector<Token> Toks;
1230 
1231     lexString(Sources[i], Toks);
1232 
1233     ASSERT_EQ(6U, Toks.size());
1234 
1235     ASSERT_EQ(tok::text,               Toks[0].getKind());
1236     ASSERT_EQ(StringRef(" "),          Toks[0].getText());
1237 
1238     ASSERT_EQ(tok::html_start_tag,     Toks[1].getKind());
1239     ASSERT_EQ(StringRef("img"),        Toks[1].getHTMLTagStartName());
1240 
1241     ASSERT_EQ(tok::html_ident,         Toks[2].getKind());
1242     ASSERT_EQ(StringRef("src"),        Toks[2].getHTMLIdent());
1243 
1244     ASSERT_EQ(tok::html_equals,        Toks[3].getKind());
1245 
1246     ASSERT_EQ(tok::html_quoted_string, Toks[4].getKind());
1247     ASSERT_EQ(StringRef(""),           Toks[4].getHTMLQuotedString());
1248 
1249     ASSERT_EQ(tok::newline,            Toks[5].getKind());
1250   }
1251 }
1252 
1253 TEST_F(CommentLexerTest, HTML12) {
1254   const char *Source = "// <img src=@";
1255 
1256   std::vector<Token> Toks;
1257 
1258   lexString(Source, Toks);
1259 
1260   ASSERT_EQ(6U, Toks.size());
1261 
1262   ASSERT_EQ(tok::text,           Toks[0].getKind());
1263   ASSERT_EQ(StringRef(" "),      Toks[0].getText());
1264 
1265   ASSERT_EQ(tok::html_start_tag, Toks[1].getKind());
1266   ASSERT_EQ(StringRef("img"),    Toks[1].getHTMLTagStartName());
1267 
1268   ASSERT_EQ(tok::html_ident,     Toks[2].getKind());
1269   ASSERT_EQ(StringRef("src"),    Toks[2].getHTMLIdent());
1270 
1271   ASSERT_EQ(tok::html_equals,    Toks[3].getKind());
1272 
1273   ASSERT_EQ(tok::text,           Toks[4].getKind());
1274   ASSERT_EQ(StringRef("@"),      Toks[4].getText());
1275 
1276   ASSERT_EQ(tok::newline,        Toks[5].getKind());
1277 }
1278 
1279 TEST_F(CommentLexerTest, HTML13) {
1280   const char *Sources[] = {
1281     "// <img src=\"val\\\"\\'val",
1282     "// <img src=\"val\\\"\\'val\"",
1283     "// <img src=\'val\\\"\\'val",
1284     "// <img src=\'val\\\"\\'val\'"
1285   };
1286 
1287   for (size_t i = 0, e = array_lengthof(Sources); i != e; i++) {
1288     std::vector<Token> Toks;
1289 
1290     lexString(Sources[i], Toks);
1291 
1292     ASSERT_EQ(6U, Toks.size());
1293 
1294     ASSERT_EQ(tok::text,                  Toks[0].getKind());
1295     ASSERT_EQ(StringRef(" "),             Toks[0].getText());
1296 
1297     ASSERT_EQ(tok::html_start_tag,        Toks[1].getKind());
1298     ASSERT_EQ(StringRef("img"),           Toks[1].getHTMLTagStartName());
1299 
1300     ASSERT_EQ(tok::html_ident,            Toks[2].getKind());
1301     ASSERT_EQ(StringRef("src"),           Toks[2].getHTMLIdent());
1302 
1303     ASSERT_EQ(tok::html_equals,           Toks[3].getKind());
1304 
1305     ASSERT_EQ(tok::html_quoted_string,    Toks[4].getKind());
1306     ASSERT_EQ(StringRef("val\\\"\\'val"), Toks[4].getHTMLQuotedString());
1307 
1308     ASSERT_EQ(tok::newline,               Toks[5].getKind());
1309   }
1310 }
1311 
1312 TEST_F(CommentLexerTest, HTML14) {
1313   const char *Sources[] = {
1314     "// <img src=\"val\\\"\\'val\">",
1315     "// <img src=\'val\\\"\\'val\'>"
1316   };
1317 
1318   for (size_t i = 0, e = array_lengthof(Sources); i != e; i++) {
1319     std::vector<Token> Toks;
1320 
1321     lexString(Sources[i], Toks);
1322 
1323     ASSERT_EQ(7U, Toks.size());
1324 
1325     ASSERT_EQ(tok::text,                  Toks[0].getKind());
1326     ASSERT_EQ(StringRef(" "),             Toks[0].getText());
1327 
1328     ASSERT_EQ(tok::html_start_tag,        Toks[1].getKind());
1329     ASSERT_EQ(StringRef("img"),           Toks[1].getHTMLTagStartName());
1330 
1331     ASSERT_EQ(tok::html_ident,            Toks[2].getKind());
1332     ASSERT_EQ(StringRef("src"),           Toks[2].getHTMLIdent());
1333 
1334     ASSERT_EQ(tok::html_equals,           Toks[3].getKind());
1335 
1336     ASSERT_EQ(tok::html_quoted_string,    Toks[4].getKind());
1337     ASSERT_EQ(StringRef("val\\\"\\'val"), Toks[4].getHTMLQuotedString());
1338 
1339     ASSERT_EQ(tok::html_greater,          Toks[5].getKind());
1340 
1341     ASSERT_EQ(tok::newline,               Toks[6].getKind());
1342   }
1343 }
1344 
1345 TEST_F(CommentLexerTest, HTML15) {
1346   const char *Sources[] = {
1347     "// <img/>",
1348     "// <img />"
1349   };
1350 
1351   for (size_t i = 0, e = array_lengthof(Sources); i != e; i++) {
1352     std::vector<Token> Toks;
1353 
1354     lexString(Sources[i], Toks);
1355 
1356     ASSERT_EQ(4U, Toks.size());
1357 
1358     ASSERT_EQ(tok::text,               Toks[0].getKind());
1359     ASSERT_EQ(StringRef(" "),          Toks[0].getText());
1360 
1361     ASSERT_EQ(tok::html_start_tag,     Toks[1].getKind());
1362     ASSERT_EQ(StringRef("img"),        Toks[1].getHTMLTagStartName());
1363 
1364     ASSERT_EQ(tok::html_slash_greater, Toks[2].getKind());
1365 
1366     ASSERT_EQ(tok::newline,            Toks[3].getKind());
1367   }
1368 }
1369 
1370 TEST_F(CommentLexerTest, HTML16) {
1371   const char *Sources[] = {
1372     "// <img/ Aaa",
1373     "// <img / Aaa"
1374   };
1375 
1376   for (size_t i = 0, e = array_lengthof(Sources); i != e; i++) {
1377     std::vector<Token> Toks;
1378 
1379     lexString(Sources[i], Toks);
1380 
1381     ASSERT_EQ(5U, Toks.size());
1382 
1383     ASSERT_EQ(tok::text,               Toks[0].getKind());
1384     ASSERT_EQ(StringRef(" "),          Toks[0].getText());
1385 
1386     ASSERT_EQ(tok::html_start_tag,     Toks[1].getKind());
1387     ASSERT_EQ(StringRef("img"),        Toks[1].getHTMLTagStartName());
1388 
1389     ASSERT_EQ(tok::text,               Toks[2].getKind());
1390     ASSERT_EQ(StringRef("/"),          Toks[2].getText());
1391 
1392     ASSERT_EQ(tok::text,               Toks[3].getKind());
1393     ASSERT_EQ(StringRef(" Aaa"),       Toks[3].getText());
1394 
1395     ASSERT_EQ(tok::newline,            Toks[4].getKind());
1396   }
1397 }
1398 
1399 TEST_F(CommentLexerTest, HTML17) {
1400   const char *Source = "// </";
1401 
1402   std::vector<Token> Toks;
1403 
1404   lexString(Source, Toks);
1405 
1406   ASSERT_EQ(3U, Toks.size());
1407 
1408   ASSERT_EQ(tok::text,       Toks[0].getKind());
1409   ASSERT_EQ(StringRef(" "),  Toks[0].getText());
1410 
1411   ASSERT_EQ(tok::text,       Toks[1].getKind());
1412   ASSERT_EQ(StringRef("</"), Toks[1].getText());
1413 
1414   ASSERT_EQ(tok::newline,    Toks[2].getKind());
1415 }
1416 
1417 TEST_F(CommentLexerTest, HTML18) {
1418   const char *Source = "// </@";
1419 
1420   std::vector<Token> Toks;
1421 
1422   lexString(Source, Toks);
1423 
1424   ASSERT_EQ(4U, Toks.size());
1425 
1426   ASSERT_EQ(tok::text,       Toks[0].getKind());
1427   ASSERT_EQ(StringRef(" "),  Toks[0].getText());
1428 
1429   ASSERT_EQ(tok::text,       Toks[1].getKind());
1430   ASSERT_EQ(StringRef("</"), Toks[1].getText());
1431 
1432   ASSERT_EQ(tok::text,       Toks[2].getKind());
1433   ASSERT_EQ(StringRef("@"),  Toks[2].getText());
1434 
1435   ASSERT_EQ(tok::newline,    Toks[3].getKind());
1436 }
1437 
1438 TEST_F(CommentLexerTest, HTML19) {
1439   const char *Source = "// </img";
1440 
1441   std::vector<Token> Toks;
1442 
1443   lexString(Source, Toks);
1444 
1445   ASSERT_EQ(3U, Toks.size());
1446 
1447   ASSERT_EQ(tok::text,         Toks[0].getKind());
1448   ASSERT_EQ(StringRef(" "),    Toks[0].getText());
1449 
1450   ASSERT_EQ(tok::html_end_tag, Toks[1].getKind());
1451   ASSERT_EQ(StringRef("img"),  Toks[1].getHTMLTagEndName());
1452 
1453   ASSERT_EQ(tok::newline,      Toks[2].getKind());
1454 }
1455 
1456 TEST_F(CommentLexerTest, NotAKnownHTMLTag1) {
1457   const char *Source = "// <tag>";
1458 
1459   std::vector<Token> Toks;
1460 
1461   lexString(Source, Toks);
1462 
1463   ASSERT_EQ(4U, Toks.size());
1464 
1465   ASSERT_EQ(tok::text,         Toks[0].getKind());
1466   ASSERT_EQ(StringRef(" "),    Toks[0].getText());
1467 
1468   ASSERT_EQ(tok::text,         Toks[1].getKind());
1469   ASSERT_EQ(StringRef("<tag"), Toks[1].getText());
1470 
1471   ASSERT_EQ(tok::text,         Toks[2].getKind());
1472   ASSERT_EQ(StringRef(">"),    Toks[2].getText());
1473 
1474   ASSERT_EQ(tok::newline,      Toks[3].getKind());
1475 }
1476 
1477 TEST_F(CommentLexerTest, NotAKnownHTMLTag2) {
1478   const char *Source = "// </tag>";
1479 
1480   std::vector<Token> Toks;
1481 
1482   lexString(Source, Toks);
1483 
1484   ASSERT_EQ(4U, Toks.size());
1485 
1486   ASSERT_EQ(tok::text,          Toks[0].getKind());
1487   ASSERT_EQ(StringRef(" "),     Toks[0].getText());
1488 
1489   ASSERT_EQ(tok::text,          Toks[1].getKind());
1490   ASSERT_EQ(StringRef("</tag"), Toks[1].getText());
1491 
1492   ASSERT_EQ(tok::text,          Toks[2].getKind());
1493   ASSERT_EQ(StringRef(">"),     Toks[2].getText());
1494 
1495   ASSERT_EQ(tok::newline,       Toks[3].getKind());
1496 }
1497 
1498 TEST_F(CommentLexerTest, HTMLCharacterReferences1) {
1499   const char *Source = "// &";
1500 
1501   std::vector<Token> Toks;
1502 
1503   lexString(Source, Toks);
1504 
1505   ASSERT_EQ(3U, Toks.size());
1506 
1507   ASSERT_EQ(tok::text,         Toks[0].getKind());
1508   ASSERT_EQ(StringRef(" "),    Toks[0].getText());
1509 
1510   ASSERT_EQ(tok::text,         Toks[1].getKind());
1511   ASSERT_EQ(StringRef("&"),    Toks[1].getText());
1512 
1513   ASSERT_EQ(tok::newline,      Toks[2].getKind());
1514 }
1515 
1516 TEST_F(CommentLexerTest, HTMLCharacterReferences2) {
1517   const char *Source = "// &!";
1518 
1519   std::vector<Token> Toks;
1520 
1521   lexString(Source, Toks);
1522 
1523   ASSERT_EQ(4U, Toks.size());
1524 
1525   ASSERT_EQ(tok::text,         Toks[0].getKind());
1526   ASSERT_EQ(StringRef(" "),    Toks[0].getText());
1527 
1528   ASSERT_EQ(tok::text,         Toks[1].getKind());
1529   ASSERT_EQ(StringRef("&"),    Toks[1].getText());
1530 
1531   ASSERT_EQ(tok::text,         Toks[2].getKind());
1532   ASSERT_EQ(StringRef("!"),    Toks[2].getText());
1533 
1534   ASSERT_EQ(tok::newline,      Toks[3].getKind());
1535 }
1536 
1537 TEST_F(CommentLexerTest, HTMLCharacterReferences3) {
1538   const char *Source = "// &amp";
1539 
1540   std::vector<Token> Toks;
1541 
1542   lexString(Source, Toks);
1543 
1544   ASSERT_EQ(3U, Toks.size());
1545 
1546   ASSERT_EQ(tok::text,         Toks[0].getKind());
1547   ASSERT_EQ(StringRef(" "),    Toks[0].getText());
1548 
1549   ASSERT_EQ(tok::text,         Toks[1].getKind());
1550   ASSERT_EQ(StringRef("&amp"), Toks[1].getText());
1551 
1552   ASSERT_EQ(tok::newline,      Toks[2].getKind());
1553 }
1554 
1555 TEST_F(CommentLexerTest, HTMLCharacterReferences4) {
1556   const char *Source = "// &amp!";
1557 
1558   std::vector<Token> Toks;
1559 
1560   lexString(Source, Toks);
1561 
1562   ASSERT_EQ(4U, Toks.size());
1563 
1564   ASSERT_EQ(tok::text,         Toks[0].getKind());
1565   ASSERT_EQ(StringRef(" "),    Toks[0].getText());
1566 
1567   ASSERT_EQ(tok::text,         Toks[1].getKind());
1568   ASSERT_EQ(StringRef("&amp"), Toks[1].getText());
1569 
1570   ASSERT_EQ(tok::text,         Toks[2].getKind());
1571   ASSERT_EQ(StringRef("!"),    Toks[2].getText());
1572 
1573   ASSERT_EQ(tok::newline,      Toks[3].getKind());
1574 }
1575 
1576 TEST_F(CommentLexerTest, HTMLCharacterReferences5) {
1577   const char *Source = "// &#";
1578 
1579   std::vector<Token> Toks;
1580 
1581   lexString(Source, Toks);
1582 
1583   ASSERT_EQ(3U, Toks.size());
1584 
1585   ASSERT_EQ(tok::text,         Toks[0].getKind());
1586   ASSERT_EQ(StringRef(" "),    Toks[0].getText());
1587 
1588   ASSERT_EQ(tok::text,         Toks[1].getKind());
1589   ASSERT_EQ(StringRef("&#"),   Toks[1].getText());
1590 
1591   ASSERT_EQ(tok::newline,      Toks[2].getKind());
1592 }
1593 
1594 TEST_F(CommentLexerTest, HTMLCharacterReferences6) {
1595   const char *Source = "// &#a";
1596 
1597   std::vector<Token> Toks;
1598 
1599   lexString(Source, Toks);
1600 
1601   ASSERT_EQ(4U, Toks.size());
1602 
1603   ASSERT_EQ(tok::text,         Toks[0].getKind());
1604   ASSERT_EQ(StringRef(" "),    Toks[0].getText());
1605 
1606   ASSERT_EQ(tok::text,         Toks[1].getKind());
1607   ASSERT_EQ(StringRef("&#"),   Toks[1].getText());
1608 
1609   ASSERT_EQ(tok::text,         Toks[2].getKind());
1610   ASSERT_EQ(StringRef("a"),    Toks[2].getText());
1611 
1612   ASSERT_EQ(tok::newline,      Toks[3].getKind());
1613 }
1614 
1615 TEST_F(CommentLexerTest, HTMLCharacterReferences7) {
1616   const char *Source = "// &#42";
1617 
1618   std::vector<Token> Toks;
1619 
1620   lexString(Source, Toks);
1621 
1622   ASSERT_EQ(3U, Toks.size());
1623 
1624   ASSERT_EQ(tok::text,         Toks[0].getKind());
1625   ASSERT_EQ(StringRef(" "),    Toks[0].getText());
1626 
1627   ASSERT_EQ(tok::text,         Toks[1].getKind());
1628   ASSERT_EQ(StringRef("&#42"), Toks[1].getText());
1629 
1630   ASSERT_EQ(tok::newline,      Toks[2].getKind());
1631 }
1632 
1633 TEST_F(CommentLexerTest, HTMLCharacterReferences8) {
1634   const char *Source = "// &#42a";
1635 
1636   std::vector<Token> Toks;
1637 
1638   lexString(Source, Toks);
1639 
1640   ASSERT_EQ(4U, Toks.size());
1641 
1642   ASSERT_EQ(tok::text,         Toks[0].getKind());
1643   ASSERT_EQ(StringRef(" "),    Toks[0].getText());
1644 
1645   ASSERT_EQ(tok::text,         Toks[1].getKind());
1646   ASSERT_EQ(StringRef("&#42"), Toks[1].getText());
1647 
1648   ASSERT_EQ(tok::text,         Toks[2].getKind());
1649   ASSERT_EQ(StringRef("a"),    Toks[2].getText());
1650 
1651   ASSERT_EQ(tok::newline,      Toks[3].getKind());
1652 }
1653 
1654 TEST_F(CommentLexerTest, HTMLCharacterReferences9) {
1655   const char *Source = "// &#x";
1656 
1657   std::vector<Token> Toks;
1658 
1659   lexString(Source, Toks);
1660 
1661   ASSERT_EQ(3U, Toks.size());
1662 
1663   ASSERT_EQ(tok::text,         Toks[0].getKind());
1664   ASSERT_EQ(StringRef(" "),    Toks[0].getText());
1665 
1666   ASSERT_EQ(tok::text,         Toks[1].getKind());
1667   ASSERT_EQ(StringRef("&#x"),  Toks[1].getText());
1668 
1669   ASSERT_EQ(tok::newline,      Toks[2].getKind());
1670 }
1671 
1672 TEST_F(CommentLexerTest, HTMLCharacterReferences10) {
1673   const char *Source = "// &#xz";
1674 
1675   std::vector<Token> Toks;
1676 
1677   lexString(Source, Toks);
1678 
1679   ASSERT_EQ(4U, Toks.size());
1680 
1681   ASSERT_EQ(tok::text,         Toks[0].getKind());
1682   ASSERT_EQ(StringRef(" "),    Toks[0].getText());
1683 
1684   ASSERT_EQ(tok::text,         Toks[1].getKind());
1685   ASSERT_EQ(StringRef("&#x"),  Toks[1].getText());
1686 
1687   ASSERT_EQ(tok::text,         Toks[2].getKind());
1688   ASSERT_EQ(StringRef("z"),    Toks[2].getText());
1689 
1690   ASSERT_EQ(tok::newline,      Toks[3].getKind());
1691 }
1692 
1693 TEST_F(CommentLexerTest, HTMLCharacterReferences11) {
1694   const char *Source = "// &#xab";
1695 
1696   std::vector<Token> Toks;
1697 
1698   lexString(Source, Toks);
1699 
1700   ASSERT_EQ(3U, Toks.size());
1701 
1702   ASSERT_EQ(tok::text,          Toks[0].getKind());
1703   ASSERT_EQ(StringRef(" "),     Toks[0].getText());
1704 
1705   ASSERT_EQ(tok::text,          Toks[1].getKind());
1706   ASSERT_EQ(StringRef("&#xab"), Toks[1].getText());
1707 
1708   ASSERT_EQ(tok::newline,       Toks[2].getKind());
1709 }
1710 
1711 TEST_F(CommentLexerTest, HTMLCharacterReferences12) {
1712   const char *Source = "// &#xaBz";
1713 
1714   std::vector<Token> Toks;
1715 
1716   lexString(Source, Toks);
1717 
1718   ASSERT_EQ(4U, Toks.size());
1719 
1720   ASSERT_EQ(tok::text,          Toks[0].getKind());
1721   ASSERT_EQ(StringRef(" "),     Toks[0].getText());
1722 
1723   ASSERT_EQ(tok::text,          Toks[1].getKind());
1724   ASSERT_EQ(StringRef("&#xaB"), Toks[1].getText());
1725 
1726   ASSERT_EQ(tok::text,          Toks[2].getKind());
1727   ASSERT_EQ(StringRef("z"),     Toks[2].getText());
1728 
1729   ASSERT_EQ(tok::newline,       Toks[3].getKind());
1730 }
1731 
1732 TEST_F(CommentLexerTest, HTMLCharacterReferences13) {
1733   const char *Source = "// &amp;";
1734 
1735   std::vector<Token> Toks;
1736 
1737   lexString(Source, Toks);
1738 
1739   ASSERT_EQ(3U, Toks.size());
1740 
1741   ASSERT_EQ(tok::text,          Toks[0].getKind());
1742   ASSERT_EQ(StringRef(" "),     Toks[0].getText());
1743 
1744   ASSERT_EQ(tok::text,          Toks[1].getKind());
1745   ASSERT_EQ(StringRef("&"),     Toks[1].getText());
1746 
1747   ASSERT_EQ(tok::newline,       Toks[2].getKind());
1748 }
1749 
1750 TEST_F(CommentLexerTest, HTMLCharacterReferences14) {
1751   const char *Source = "// &amp;&lt;";
1752 
1753   std::vector<Token> Toks;
1754 
1755   lexString(Source, Toks);
1756 
1757   ASSERT_EQ(4U, Toks.size());
1758 
1759   ASSERT_EQ(tok::text,          Toks[0].getKind());
1760   ASSERT_EQ(StringRef(" "),     Toks[0].getText());
1761 
1762   ASSERT_EQ(tok::text,          Toks[1].getKind());
1763   ASSERT_EQ(StringRef("&"),     Toks[1].getText());
1764 
1765   ASSERT_EQ(tok::text,          Toks[2].getKind());
1766   ASSERT_EQ(StringRef("<"),     Toks[2].getText());
1767 
1768   ASSERT_EQ(tok::newline,       Toks[3].getKind());
1769 }
1770 
1771 TEST_F(CommentLexerTest, HTMLCharacterReferences15) {
1772   const char *Source = "// &amp; meow";
1773 
1774   std::vector<Token> Toks;
1775 
1776   lexString(Source, Toks);
1777 
1778   ASSERT_EQ(4U, Toks.size());
1779 
1780   ASSERT_EQ(tok::text,          Toks[0].getKind());
1781   ASSERT_EQ(StringRef(" "),     Toks[0].getText());
1782 
1783   ASSERT_EQ(tok::text,          Toks[1].getKind());
1784   ASSERT_EQ(StringRef("&"),     Toks[1].getText());
1785 
1786   ASSERT_EQ(tok::text,          Toks[2].getKind());
1787   ASSERT_EQ(StringRef(" meow"), Toks[2].getText());
1788 
1789   ASSERT_EQ(tok::newline,       Toks[3].getKind());
1790 }
1791 
1792 TEST_F(CommentLexerTest, HTMLCharacterReferences16) {
1793   const char *Sources[] = {
1794     "// &#61;",
1795     "// &#x3d;",
1796     "// &#X3d;",
1797     "// &#X3D;"
1798   };
1799 
1800   for (size_t i = 0, e = array_lengthof(Sources); i != e; i++) {
1801     std::vector<Token> Toks;
1802 
1803     lexString(Sources[i], Toks);
1804 
1805     ASSERT_EQ(3U, Toks.size());
1806 
1807     ASSERT_EQ(tok::text,          Toks[0].getKind());
1808     ASSERT_EQ(StringRef(" "),     Toks[0].getText());
1809 
1810     ASSERT_EQ(tok::text,          Toks[1].getKind());
1811     ASSERT_EQ(StringRef("="),     Toks[1].getText());
1812 
1813     ASSERT_EQ(tok::newline,       Toks[2].getKind());
1814   }
1815 }
1816 
1817 TEST_F(CommentLexerTest, MultipleComments) {
1818   const char *Source =
1819     "// Aaa\n"
1820     "/// Bbb\n"
1821     "/* Ccc\n"
1822     " * Ddd*/\n"
1823     "/** Eee*/";
1824 
1825   std::vector<Token> Toks;
1826 
1827   lexString(Source, Toks);
1828 
1829   ASSERT_EQ(12U, Toks.size());
1830 
1831   ASSERT_EQ(tok::text,           Toks[0].getKind());
1832   ASSERT_EQ(StringRef(" Aaa"),   Toks[0].getText());
1833   ASSERT_EQ(tok::newline,        Toks[1].getKind());
1834 
1835   ASSERT_EQ(tok::text,           Toks[2].getKind());
1836   ASSERT_EQ(StringRef(" Bbb"),   Toks[2].getText());
1837   ASSERT_EQ(tok::newline,        Toks[3].getKind());
1838 
1839   ASSERT_EQ(tok::text,           Toks[4].getKind());
1840   ASSERT_EQ(StringRef(" Ccc"),   Toks[4].getText());
1841   ASSERT_EQ(tok::newline,        Toks[5].getKind());
1842 
1843   ASSERT_EQ(tok::text,           Toks[6].getKind());
1844   ASSERT_EQ(StringRef(" Ddd"),   Toks[6].getText());
1845   ASSERT_EQ(tok::newline,        Toks[7].getKind());
1846   ASSERT_EQ(tok::newline,        Toks[8].getKind());
1847 
1848   ASSERT_EQ(tok::text,           Toks[9].getKind());
1849   ASSERT_EQ(StringRef(" Eee"),   Toks[9].getText());
1850 
1851   ASSERT_EQ(tok::newline,        Toks[10].getKind());
1852   ASSERT_EQ(tok::newline,        Toks[11].getKind());
1853 }
1854 
1855 } // end namespace comments
1856 } // end namespace clang
1857 
1858