1 //===- unittests/AST/CommentLexer.cpp ------ Comment lexer tests ----------===//
2 //
3 //                     The LLVM Compiler Infrastructure
4 //
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
7 //
8 //===----------------------------------------------------------------------===//
9 
10 #include "clang/AST/CommentLexer.h"
11 #include "clang/AST/CommentCommandTraits.h"
12 #include "clang/Basic/CommentOptions.h"
13 #include "clang/Basic/Diagnostic.h"
14 #include "clang/Basic/DiagnosticOptions.h"
15 #include "clang/Basic/FileManager.h"
16 #include "clang/Basic/SourceManager.h"
17 #include "llvm/ADT/STLExtras.h"
18 #include "gtest/gtest.h"
19 #include <vector>
20 
21 using namespace llvm;
22 using namespace clang;
23 
24 namespace clang {
25 namespace comments {
26 
27 namespace {
28 class CommentLexerTest : public ::testing::Test {
29 protected:
30   CommentLexerTest()
31     : FileMgr(FileMgrOpts),
32       DiagID(new DiagnosticIDs()),
33       Diags(DiagID, new DiagnosticOptions, new IgnoringDiagConsumer()),
34       SourceMgr(Diags, FileMgr),
35       Traits(Allocator, CommentOptions()) {
36   }
37 
38   FileSystemOptions FileMgrOpts;
39   FileManager FileMgr;
40   IntrusiveRefCntPtr<DiagnosticIDs> DiagID;
41   DiagnosticsEngine Diags;
42   SourceManager SourceMgr;
43   llvm::BumpPtrAllocator Allocator;
44   CommandTraits Traits;
45 
46   void lexString(const char *Source, std::vector<Token> &Toks);
47 
48   StringRef getCommandName(const Token &Tok) {
49     return Traits.getCommandInfo(Tok.getCommandID())->Name;
50   }
51 
52   StringRef getVerbatimBlockName(const Token &Tok) {
53     return Traits.getCommandInfo(Tok.getVerbatimBlockID())->Name;
54   }
55 
56   StringRef getVerbatimLineName(const Token &Tok) {
57     return Traits.getCommandInfo(Tok.getVerbatimLineID())->Name;
58   }
59 };
60 
61 void CommentLexerTest::lexString(const char *Source,
62                                  std::vector<Token> &Toks) {
63   std::unique_ptr<MemoryBuffer> Buf = MemoryBuffer::getMemBuffer(Source);
64   FileID File = SourceMgr.createFileID(std::move(Buf));
65   SourceLocation Begin = SourceMgr.getLocForStartOfFile(File);
66 
67   Lexer L(Allocator, Diags, Traits, Begin, Source, Source + strlen(Source));
68 
69   while (1) {
70     Token Tok;
71     L.lex(Tok);
72     if (Tok.is(tok::eof))
73       break;
74     Toks.push_back(Tok);
75   }
76 }
77 
78 } // unnamed namespace
79 
80 // Empty source range should be handled.
81 TEST_F(CommentLexerTest, Basic1) {
82   const char *Source = "";
83   std::vector<Token> Toks;
84 
85   lexString(Source, Toks);
86 
87   ASSERT_EQ(0U, Toks.size());
88 }
89 
90 // Empty comments should be handled.
91 TEST_F(CommentLexerTest, Basic2) {
92   const char *Sources[] = {
93     "//", "///", "//!", "///<", "//!<"
94   };
95   for (size_t i = 0, e = array_lengthof(Sources); i != e; i++) {
96     std::vector<Token> Toks;
97 
98     lexString(Sources[i], Toks);
99 
100     ASSERT_EQ(1U, Toks.size());
101 
102     ASSERT_EQ(tok::newline, Toks[0].getKind());
103   }
104 }
105 
106 // Empty comments should be handled.
107 TEST_F(CommentLexerTest, Basic3) {
108   const char *Sources[] = {
109     "/**/", "/***/", "/*!*/", "/**<*/", "/*!<*/"
110   };
111   for (size_t i = 0, e = array_lengthof(Sources); i != e; i++) {
112     std::vector<Token> Toks;
113 
114     lexString(Sources[i], Toks);
115 
116     ASSERT_EQ(2U, Toks.size());
117 
118     ASSERT_EQ(tok::newline, Toks[0].getKind());
119     ASSERT_EQ(tok::newline, Toks[1].getKind());
120   }
121 }
122 
123 // Single comment with plain text.
124 TEST_F(CommentLexerTest, Basic4) {
125   const char *Sources[] = {
126     "// Meow",   "/// Meow",    "//! Meow",
127     "// Meow\n", "// Meow\r\n", "//! Meow\r",
128   };
129 
130   for (size_t i = 0, e = array_lengthof(Sources); i != e; i++) {
131     std::vector<Token> Toks;
132 
133     lexString(Sources[i], Toks);
134 
135     ASSERT_EQ(2U, Toks.size());
136 
137     ASSERT_EQ(tok::text,          Toks[0].getKind());
138     ASSERT_EQ(StringRef(" Meow"), Toks[0].getText());
139 
140     ASSERT_EQ(tok::newline,       Toks[1].getKind());
141   }
142 }
143 
144 // Single comment with plain text.
145 TEST_F(CommentLexerTest, Basic5) {
146   const char *Sources[] = {
147     "/* Meow*/", "/** Meow*/",  "/*! Meow*/"
148   };
149 
150   for (size_t i = 0, e = array_lengthof(Sources); i != e; i++) {
151     std::vector<Token> Toks;
152 
153     lexString(Sources[i], Toks);
154 
155     ASSERT_EQ(3U, Toks.size());
156 
157     ASSERT_EQ(tok::text,          Toks[0].getKind());
158     ASSERT_EQ(StringRef(" Meow"), Toks[0].getText());
159 
160     ASSERT_EQ(tok::newline,       Toks[1].getKind());
161     ASSERT_EQ(tok::newline,       Toks[2].getKind());
162   }
163 }
164 
165 // Test newline escaping.
166 TEST_F(CommentLexerTest, Basic6) {
167   const char *Sources[] = {
168     "// Aaa\\\n"   " Bbb\\ \n"   " Ccc?" "?/\n",
169     "// Aaa\\\r\n" " Bbb\\ \r\n" " Ccc?" "?/\r\n",
170     "// Aaa\\\r"   " Bbb\\ \r"   " Ccc?" "?/\r"
171   };
172 
173   for (size_t i = 0, e = array_lengthof(Sources); i != e; i++) {
174     std::vector<Token> Toks;
175 
176     lexString(Sources[i], Toks);
177 
178     ASSERT_EQ(10U, Toks.size());
179 
180     ASSERT_EQ(tok::text,         Toks[0].getKind());
181     ASSERT_EQ(StringRef(" Aaa"), Toks[0].getText());
182     ASSERT_EQ(tok::text,         Toks[1].getKind());
183     ASSERT_EQ(StringRef("\\"),   Toks[1].getText());
184     ASSERT_EQ(tok::newline,      Toks[2].getKind());
185 
186     ASSERT_EQ(tok::text,         Toks[3].getKind());
187     ASSERT_EQ(StringRef(" Bbb"), Toks[3].getText());
188     ASSERT_EQ(tok::text,         Toks[4].getKind());
189     ASSERT_EQ(StringRef("\\"),   Toks[4].getText());
190     ASSERT_EQ(tok::text,         Toks[5].getKind());
191     ASSERT_EQ(StringRef(" "),    Toks[5].getText());
192     ASSERT_EQ(tok::newline,      Toks[6].getKind());
193 
194     ASSERT_EQ(tok::text,         Toks[7].getKind());
195     ASSERT_EQ(StringRef(" Ccc?" "?/"), Toks[7].getText());
196     ASSERT_EQ(tok::newline,      Toks[8].getKind());
197 
198     ASSERT_EQ(tok::newline,      Toks[9].getKind());
199   }
200 }
201 
202 // Check that we skip C-style aligned stars correctly.
203 TEST_F(CommentLexerTest, Basic7) {
204   const char *Source =
205     "/* Aaa\n"
206     " * Bbb\r\n"
207     "\t* Ccc\n"
208     "  ! Ddd\n"
209     "  * Eee\n"
210     "  ** Fff\n"
211     " */";
212   std::vector<Token> Toks;
213 
214   lexString(Source, Toks);
215 
216   ASSERT_EQ(15U, Toks.size());
217 
218   ASSERT_EQ(tok::text,         Toks[0].getKind());
219   ASSERT_EQ(StringRef(" Aaa"), Toks[0].getText());
220   ASSERT_EQ(tok::newline,      Toks[1].getKind());
221 
222   ASSERT_EQ(tok::text,         Toks[2].getKind());
223   ASSERT_EQ(StringRef(" Bbb"), Toks[2].getText());
224   ASSERT_EQ(tok::newline,      Toks[3].getKind());
225 
226   ASSERT_EQ(tok::text,         Toks[4].getKind());
227   ASSERT_EQ(StringRef(" Ccc"), Toks[4].getText());
228   ASSERT_EQ(tok::newline,      Toks[5].getKind());
229 
230   ASSERT_EQ(tok::text,            Toks[6].getKind());
231   ASSERT_EQ(StringRef("  ! Ddd"), Toks[6].getText());
232   ASSERT_EQ(tok::newline,         Toks[7].getKind());
233 
234   ASSERT_EQ(tok::text,         Toks[8].getKind());
235   ASSERT_EQ(StringRef(" Eee"), Toks[8].getText());
236   ASSERT_EQ(tok::newline,      Toks[9].getKind());
237 
238   ASSERT_EQ(tok::text,          Toks[10].getKind());
239   ASSERT_EQ(StringRef("* Fff"), Toks[10].getText());
240   ASSERT_EQ(tok::newline,       Toks[11].getKind());
241 
242   ASSERT_EQ(tok::text,         Toks[12].getKind());
243   ASSERT_EQ(StringRef(" "),    Toks[12].getText());
244 
245   ASSERT_EQ(tok::newline,      Toks[13].getKind());
246   ASSERT_EQ(tok::newline,      Toks[14].getKind());
247 }
248 
249 // A command marker followed by comment end.
250 TEST_F(CommentLexerTest, DoxygenCommand1) {
251   const char *Sources[] = { "//@", "///@", "//!@" };
252   for (size_t i = 0, e = array_lengthof(Sources); i != e; i++) {
253     std::vector<Token> Toks;
254 
255     lexString(Sources[i], Toks);
256 
257     ASSERT_EQ(2U, Toks.size());
258 
259     ASSERT_EQ(tok::text,          Toks[0].getKind());
260     ASSERT_EQ(StringRef("@"),     Toks[0].getText());
261 
262     ASSERT_EQ(tok::newline,       Toks[1].getKind());
263   }
264 }
265 
266 // A command marker followed by comment end.
267 TEST_F(CommentLexerTest, DoxygenCommand2) {
268   const char *Sources[] = { "/*@*/", "/**@*/", "/*!@*/"};
269   for (size_t i = 0, e = array_lengthof(Sources); i != e; i++) {
270     std::vector<Token> Toks;
271 
272     lexString(Sources[i], Toks);
273 
274     ASSERT_EQ(3U, Toks.size());
275 
276     ASSERT_EQ(tok::text,          Toks[0].getKind());
277     ASSERT_EQ(StringRef("@"),     Toks[0].getText());
278 
279     ASSERT_EQ(tok::newline,       Toks[1].getKind());
280     ASSERT_EQ(tok::newline,       Toks[2].getKind());
281   }
282 }
283 
284 // A command marker followed by comment end.
285 TEST_F(CommentLexerTest, DoxygenCommand3) {
286   const char *Sources[] = { "/*\\*/", "/**\\*/" };
287   for (size_t i = 0, e = array_lengthof(Sources); i != e; i++) {
288     std::vector<Token> Toks;
289 
290     lexString(Sources[i], Toks);
291 
292     ASSERT_EQ(3U, Toks.size());
293 
294     ASSERT_EQ(tok::text,           Toks[0].getKind());
295     ASSERT_EQ(StringRef("\\"),     Toks[0].getText());
296 
297     ASSERT_EQ(tok::newline,        Toks[1].getKind());
298     ASSERT_EQ(tok::newline,        Toks[2].getKind());
299   }
300 }
301 
302 // Doxygen escape sequences.
303 TEST_F(CommentLexerTest, DoxygenCommand4) {
304   const char *Sources[] = {
305     "/// \\\\ \\@ \\& \\$ \\# \\< \\> \\% \\\" \\. \\::",
306     "/// @\\ @@ @& @$ @# @< @> @% @\" @. @::"
307   };
308   const char *Text[] = {
309     " ",
310     "\\", " ", "@", " ", "&", " ", "$",  " ", "#", " ",
311     "<",  " ", ">", " ", "%", " ", "\"", " ", ".", " ",
312     "::", ""
313   };
314 
315   for (size_t i = 0, e = array_lengthof(Sources); i != e; i++) {
316     std::vector<Token> Toks;
317 
318     lexString(Sources[i], Toks);
319 
320     ASSERT_EQ(array_lengthof(Text), Toks.size());
321 
322     for (size_t j = 0, e = Toks.size(); j != e; j++) {
323       if(Toks[j].is(tok::text)) {
324         ASSERT_EQ(StringRef(Text[j]), Toks[j].getText())
325           << "index " << i;
326       }
327     }
328   }
329 }
330 
331 // A command marker followed by a non-letter that is not a part of an escape
332 // sequence.
333 TEST_F(CommentLexerTest, DoxygenCommand5) {
334   const char *Source = "/// \\^ \\0";
335   std::vector<Token> Toks;
336 
337   lexString(Source, Toks);
338 
339   ASSERT_EQ(6U, Toks.size());
340 
341   ASSERT_EQ(tok::text,       Toks[0].getKind());
342   ASSERT_EQ(StringRef(" "),  Toks[0].getText());
343 
344   ASSERT_EQ(tok::text,       Toks[1].getKind());
345   ASSERT_EQ(StringRef("\\"), Toks[1].getText());
346 
347   ASSERT_EQ(tok::text,       Toks[2].getKind());
348   ASSERT_EQ(StringRef("^ "), Toks[2].getText());
349 
350   ASSERT_EQ(tok::text,       Toks[3].getKind());
351   ASSERT_EQ(StringRef("\\"), Toks[3].getText());
352 
353   ASSERT_EQ(tok::text,       Toks[4].getKind());
354   ASSERT_EQ(StringRef("0"),  Toks[4].getText());
355 
356   ASSERT_EQ(tok::newline,    Toks[5].getKind());
357 }
358 
359 TEST_F(CommentLexerTest, DoxygenCommand6) {
360   const char *Source = "/// \\brief Aaa.";
361   std::vector<Token> Toks;
362 
363   lexString(Source, Toks);
364 
365   ASSERT_EQ(4U, Toks.size());
366 
367   ASSERT_EQ(tok::text,          Toks[0].getKind());
368   ASSERT_EQ(StringRef(" "),     Toks[0].getText());
369 
370   ASSERT_EQ(tok::backslash_command, Toks[1].getKind());
371   ASSERT_EQ(StringRef("brief"), getCommandName(Toks[1]));
372 
373   ASSERT_EQ(tok::text,          Toks[2].getKind());
374   ASSERT_EQ(StringRef(" Aaa."), Toks[2].getText());
375 
376   ASSERT_EQ(tok::newline,       Toks[3].getKind());
377 }
378 
379 TEST_F(CommentLexerTest, DoxygenCommand7) {
380   const char *Source = "/// \\em\\em \\em\t\\em\n";
381   std::vector<Token> Toks;
382 
383   lexString(Source, Toks);
384 
385   ASSERT_EQ(8U, Toks.size());
386 
387   ASSERT_EQ(tok::text,       Toks[0].getKind());
388   ASSERT_EQ(StringRef(" "),  Toks[0].getText());
389 
390   ASSERT_EQ(tok::backslash_command, Toks[1].getKind());
391   ASSERT_EQ(StringRef("em"), getCommandName(Toks[1]));
392 
393   ASSERT_EQ(tok::backslash_command, Toks[2].getKind());
394   ASSERT_EQ(StringRef("em"), getCommandName(Toks[2]));
395 
396   ASSERT_EQ(tok::text,       Toks[3].getKind());
397   ASSERT_EQ(StringRef(" "),  Toks[3].getText());
398 
399   ASSERT_EQ(tok::backslash_command, Toks[4].getKind());
400   ASSERT_EQ(StringRef("em"), getCommandName(Toks[4]));
401 
402   ASSERT_EQ(tok::text,       Toks[5].getKind());
403   ASSERT_EQ(StringRef("\t"), Toks[5].getText());
404 
405   ASSERT_EQ(tok::backslash_command, Toks[6].getKind());
406   ASSERT_EQ(StringRef("em"), getCommandName(Toks[6]));
407 
408   ASSERT_EQ(tok::newline,    Toks[7].getKind());
409 }
410 
411 TEST_F(CommentLexerTest, DoxygenCommand8) {
412   const char *Source = "/// @em@em @em\t@em\n";
413   std::vector<Token> Toks;
414 
415   lexString(Source, Toks);
416 
417   ASSERT_EQ(8U, Toks.size());
418 
419   ASSERT_EQ(tok::text,       Toks[0].getKind());
420   ASSERT_EQ(StringRef(" "),  Toks[0].getText());
421 
422   ASSERT_EQ(tok::at_command, Toks[1].getKind());
423   ASSERT_EQ(StringRef("em"), getCommandName(Toks[1]));
424 
425   ASSERT_EQ(tok::at_command, Toks[2].getKind());
426   ASSERT_EQ(StringRef("em"), getCommandName(Toks[2]));
427 
428   ASSERT_EQ(tok::text,       Toks[3].getKind());
429   ASSERT_EQ(StringRef(" "),  Toks[3].getText());
430 
431   ASSERT_EQ(tok::at_command, Toks[4].getKind());
432   ASSERT_EQ(StringRef("em"), getCommandName(Toks[4]));
433 
434   ASSERT_EQ(tok::text,       Toks[5].getKind());
435   ASSERT_EQ(StringRef("\t"), Toks[5].getText());
436 
437   ASSERT_EQ(tok::at_command, Toks[6].getKind());
438   ASSERT_EQ(StringRef("em"), getCommandName(Toks[6]));
439 
440   ASSERT_EQ(tok::newline,    Toks[7].getKind());
441 }
442 
443 TEST_F(CommentLexerTest, DoxygenCommand9) {
444   const char *Source = "/// \\aaa\\bbb \\ccc\t\\ddd\n";
445   std::vector<Token> Toks;
446 
447   lexString(Source, Toks);
448 
449   ASSERT_EQ(8U, Toks.size());
450 
451   ASSERT_EQ(tok::text,        Toks[0].getKind());
452   ASSERT_EQ(StringRef(" "),   Toks[0].getText());
453 
454   ASSERT_EQ(tok::unknown_command, Toks[1].getKind());
455   ASSERT_EQ(StringRef("aaa"), Toks[1].getUnknownCommandName());
456 
457   ASSERT_EQ(tok::unknown_command, Toks[2].getKind());
458   ASSERT_EQ(StringRef("bbb"), Toks[2].getUnknownCommandName());
459 
460   ASSERT_EQ(tok::text,        Toks[3].getKind());
461   ASSERT_EQ(StringRef(" "),   Toks[3].getText());
462 
463   ASSERT_EQ(tok::unknown_command, Toks[4].getKind());
464   ASSERT_EQ(StringRef("ccc"), Toks[4].getUnknownCommandName());
465 
466   ASSERT_EQ(tok::text,        Toks[5].getKind());
467   ASSERT_EQ(StringRef("\t"),  Toks[5].getText());
468 
469   ASSERT_EQ(tok::unknown_command, Toks[6].getKind());
470   ASSERT_EQ(StringRef("ddd"), Toks[6].getUnknownCommandName());
471 
472   ASSERT_EQ(tok::newline,     Toks[7].getKind());
473 }
474 
475 TEST_F(CommentLexerTest, DoxygenCommand10) {
476   const char *Source = "// \\c\n";
477   std::vector<Token> Toks;
478 
479   lexString(Source, Toks);
480 
481   ASSERT_EQ(3U, Toks.size());
482 
483   ASSERT_EQ(tok::text,      Toks[0].getKind());
484   ASSERT_EQ(StringRef(" "), Toks[0].getText());
485 
486   ASSERT_EQ(tok::backslash_command, Toks[1].getKind());
487   ASSERT_EQ(StringRef("c"), getCommandName(Toks[1]));
488 
489   ASSERT_EQ(tok::newline,   Toks[2].getKind());
490 }
491 
492 TEST_F(CommentLexerTest, RegisterCustomBlockCommand) {
493   const char *Source =
494     "/// \\NewBlockCommand Aaa.\n"
495     "/// @NewBlockCommand Aaa.\n";
496 
497   Traits.registerBlockCommand(StringRef("NewBlockCommand"));
498 
499   std::vector<Token> Toks;
500 
501   lexString(Source, Toks);
502 
503   ASSERT_EQ(8U, Toks.size());
504 
505   ASSERT_EQ(tok::text,          Toks[0].getKind());
506   ASSERT_EQ(StringRef(" "),     Toks[0].getText());
507 
508   ASSERT_EQ(tok::backslash_command, Toks[1].getKind());
509   ASSERT_EQ(StringRef("NewBlockCommand"), getCommandName(Toks[1]));
510 
511   ASSERT_EQ(tok::text,          Toks[2].getKind());
512   ASSERT_EQ(StringRef(" Aaa."), Toks[2].getText());
513 
514   ASSERT_EQ(tok::newline,       Toks[3].getKind());
515 
516   ASSERT_EQ(tok::text,          Toks[4].getKind());
517   ASSERT_EQ(StringRef(" "),     Toks[4].getText());
518 
519   ASSERT_EQ(tok::at_command,    Toks[5].getKind());
520   ASSERT_EQ(StringRef("NewBlockCommand"), getCommandName(Toks[5]));
521 
522   ASSERT_EQ(tok::text,          Toks[6].getKind());
523   ASSERT_EQ(StringRef(" Aaa."), Toks[6].getText());
524 
525   ASSERT_EQ(tok::newline,       Toks[7].getKind());
526 }
527 
528 TEST_F(CommentLexerTest, RegisterMultipleBlockCommands) {
529   const char *Source =
530     "/// \\Foo\n"
531     "/// \\Bar Baz\n"
532     "/// \\Blech quux=corge\n";
533 
534   Traits.registerBlockCommand(StringRef("Foo"));
535   Traits.registerBlockCommand(StringRef("Bar"));
536   Traits.registerBlockCommand(StringRef("Blech"));
537 
538   std::vector<Token> Toks;
539 
540   lexString(Source, Toks);
541 
542   ASSERT_EQ(11U, Toks.size());
543 
544   ASSERT_EQ(tok::text,      Toks[0].getKind());
545   ASSERT_EQ(StringRef(" "), Toks[0].getText());
546 
547   ASSERT_EQ(tok::backslash_command, Toks[1].getKind());
548   ASSERT_EQ(StringRef("Foo"), getCommandName(Toks[1]));
549 
550   ASSERT_EQ(tok::newline,     Toks[2].getKind());
551 
552   ASSERT_EQ(tok::text,      Toks[3].getKind());
553   ASSERT_EQ(StringRef(" "), Toks[3].getText());
554 
555   ASSERT_EQ(tok::backslash_command, Toks[4].getKind());
556   ASSERT_EQ(StringRef("Bar"), getCommandName(Toks[4]));
557 
558   ASSERT_EQ(tok::text,         Toks[5].getKind());
559   ASSERT_EQ(StringRef(" Baz"), Toks[5].getText());
560 
561   ASSERT_EQ(tok::newline,     Toks[6].getKind());
562 
563   ASSERT_EQ(tok::text,      Toks[7].getKind());
564   ASSERT_EQ(StringRef(" "), Toks[7].getText());
565 
566   ASSERT_EQ(tok::backslash_command, Toks[8].getKind());
567   ASSERT_EQ(StringRef("Blech"), getCommandName(Toks[8]));
568 
569   ASSERT_EQ(tok::text,                Toks[9].getKind());
570   ASSERT_EQ(StringRef(" quux=corge"), Toks[9].getText());
571 
572   ASSERT_EQ(tok::newline,     Toks[10].getKind());
573 }
574 
575 // Empty verbatim block.
576 TEST_F(CommentLexerTest, VerbatimBlock1) {
577   const char *Sources[] = {
578     "/// \\verbatim\\endverbatim\n//",
579     "/** \\verbatim\\endverbatim*/"
580   };
581 
582   for (size_t i = 0, e = array_lengthof(Sources); i != e; i++) {
583     std::vector<Token> Toks;
584 
585     lexString(Sources[i], Toks);
586 
587     ASSERT_EQ(5U, Toks.size());
588 
589     ASSERT_EQ(tok::text,                 Toks[0].getKind());
590     ASSERT_EQ(StringRef(" "),            Toks[0].getText());
591 
592     ASSERT_EQ(tok::verbatim_block_begin, Toks[1].getKind());
593     ASSERT_EQ(StringRef("verbatim"),     getVerbatimBlockName(Toks[1]));
594 
595     ASSERT_EQ(tok::verbatim_block_end,   Toks[2].getKind());
596     ASSERT_EQ(StringRef("endverbatim"),  getVerbatimBlockName(Toks[2]));
597 
598     ASSERT_EQ(tok::newline,              Toks[3].getKind());
599     ASSERT_EQ(tok::newline,              Toks[4].getKind());
600   }
601 }
602 
603 // Empty verbatim block without an end command.
604 TEST_F(CommentLexerTest, VerbatimBlock2) {
605   const char *Source = "/// \\verbatim";
606 
607   std::vector<Token> Toks;
608 
609   lexString(Source, Toks);
610 
611   ASSERT_EQ(3U, Toks.size());
612 
613   ASSERT_EQ(tok::text,                 Toks[0].getKind());
614   ASSERT_EQ(StringRef(" "),            Toks[0].getText());
615 
616   ASSERT_EQ(tok::verbatim_block_begin, Toks[1].getKind());
617   ASSERT_EQ(StringRef("verbatim"),     getVerbatimBlockName(Toks[1]));
618 
619   ASSERT_EQ(tok::newline,              Toks[2].getKind());
620 }
621 
622 // Empty verbatim block without an end command.
623 TEST_F(CommentLexerTest, VerbatimBlock3) {
624   const char *Source = "/** \\verbatim*/";
625 
626   std::vector<Token> Toks;
627 
628   lexString(Source, Toks);
629 
630   ASSERT_EQ(4U, Toks.size());
631 
632   ASSERT_EQ(tok::text,                 Toks[0].getKind());
633   ASSERT_EQ(StringRef(" "),            Toks[0].getText());
634 
635   ASSERT_EQ(tok::verbatim_block_begin, Toks[1].getKind());
636   ASSERT_EQ(StringRef("verbatim"),     getVerbatimBlockName(Toks[1]));
637 
638   ASSERT_EQ(tok::newline,              Toks[2].getKind());
639   ASSERT_EQ(tok::newline,              Toks[3].getKind());
640 }
641 
642 // Single-line verbatim block.
643 TEST_F(CommentLexerTest, VerbatimBlock4) {
644   const char *Sources[] = {
645     "/// Meow \\verbatim aaa \\endverbatim\n//",
646     "/** Meow \\verbatim aaa \\endverbatim*/"
647   };
648 
649   for (size_t i = 0, e = array_lengthof(Sources); i != e; i++) {
650     std::vector<Token> Toks;
651 
652     lexString(Sources[i], Toks);
653 
654     ASSERT_EQ(6U, Toks.size());
655 
656     ASSERT_EQ(tok::text,                 Toks[0].getKind());
657     ASSERT_EQ(StringRef(" Meow "),       Toks[0].getText());
658 
659     ASSERT_EQ(tok::verbatim_block_begin, Toks[1].getKind());
660     ASSERT_EQ(StringRef("verbatim"),     getVerbatimBlockName(Toks[1]));
661 
662     ASSERT_EQ(tok::verbatim_block_line,  Toks[2].getKind());
663     ASSERT_EQ(StringRef(" aaa "),        Toks[2].getVerbatimBlockText());
664 
665     ASSERT_EQ(tok::verbatim_block_end,   Toks[3].getKind());
666     ASSERT_EQ(StringRef("endverbatim"),  getVerbatimBlockName(Toks[3]));
667 
668     ASSERT_EQ(tok::newline,              Toks[4].getKind());
669     ASSERT_EQ(tok::newline,              Toks[5].getKind());
670   }
671 }
672 
673 // Single-line verbatim block without an end command.
674 TEST_F(CommentLexerTest, VerbatimBlock5) {
675   const char *Sources[] = {
676     "/// Meow \\verbatim aaa \n//",
677     "/** Meow \\verbatim aaa */"
678   };
679 
680   for (size_t i = 0, e = array_lengthof(Sources); i != e; i++) {
681     std::vector<Token> Toks;
682 
683     lexString(Sources[i], Toks);
684 
685     ASSERT_EQ(5U, Toks.size());
686 
687     ASSERT_EQ(tok::text,                 Toks[0].getKind());
688     ASSERT_EQ(StringRef(" Meow "),       Toks[0].getText());
689 
690     ASSERT_EQ(tok::verbatim_block_begin, Toks[1].getKind());
691     ASSERT_EQ(StringRef("verbatim"),     getVerbatimBlockName(Toks[1]));
692 
693     ASSERT_EQ(tok::verbatim_block_line,  Toks[2].getKind());
694     ASSERT_EQ(StringRef(" aaa "),        Toks[2].getVerbatimBlockText());
695 
696     ASSERT_EQ(tok::newline,              Toks[3].getKind());
697     ASSERT_EQ(tok::newline,              Toks[4].getKind());
698   }
699 }
700 
701 TEST_F(CommentLexerTest, VerbatimBlock6) {
702   const char *Source =
703     "// \\verbatim\n"
704     "// Aaa\n"
705     "//\n"
706     "// Bbb\n"
707     "// \\endverbatim\n";
708 
709   std::vector<Token> Toks;
710 
711   lexString(Source, Toks);
712 
713   ASSERT_EQ(10U, Toks.size());
714 
715   ASSERT_EQ(tok::text,                 Toks[0].getKind());
716   ASSERT_EQ(StringRef(" "),            Toks[0].getText());
717 
718   ASSERT_EQ(tok::verbatim_block_begin, Toks[1].getKind());
719   ASSERT_EQ(StringRef("verbatim"),     getVerbatimBlockName(Toks[1]));
720 
721   ASSERT_EQ(tok::newline,              Toks[2].getKind());
722 
723   ASSERT_EQ(tok::verbatim_block_line,  Toks[3].getKind());
724   ASSERT_EQ(StringRef(" Aaa"),         Toks[3].getVerbatimBlockText());
725 
726   ASSERT_EQ(tok::newline,              Toks[4].getKind());
727 
728   ASSERT_EQ(tok::newline,              Toks[5].getKind());
729 
730   ASSERT_EQ(tok::verbatim_block_line,  Toks[6].getKind());
731   ASSERT_EQ(StringRef(" Bbb"),         Toks[6].getVerbatimBlockText());
732 
733   ASSERT_EQ(tok::newline,              Toks[7].getKind());
734 
735   ASSERT_EQ(tok::verbatim_block_end,   Toks[8].getKind());
736   ASSERT_EQ(StringRef("endverbatim"),  getVerbatimBlockName(Toks[8]));
737 
738   ASSERT_EQ(tok::newline,              Toks[9].getKind());
739 }
740 
741 TEST_F(CommentLexerTest, VerbatimBlock7) {
742   const char *Source =
743     "/* \\verbatim\n"
744     " * Aaa\n"
745     " *\n"
746     " * Bbb\n"
747     " * \\endverbatim\n"
748     " */";
749 
750   std::vector<Token> Toks;
751 
752   lexString(Source, Toks);
753 
754   ASSERT_EQ(10U, Toks.size());
755 
756   ASSERT_EQ(tok::text,                 Toks[0].getKind());
757   ASSERT_EQ(StringRef(" "),            Toks[0].getText());
758 
759   ASSERT_EQ(tok::verbatim_block_begin, Toks[1].getKind());
760   ASSERT_EQ(StringRef("verbatim"),     getVerbatimBlockName(Toks[1]));
761 
762   ASSERT_EQ(tok::verbatim_block_line,  Toks[2].getKind());
763   ASSERT_EQ(StringRef(" Aaa"),         Toks[2].getVerbatimBlockText());
764 
765   ASSERT_EQ(tok::verbatim_block_line,  Toks[3].getKind());
766   ASSERT_EQ(StringRef(""),             Toks[3].getVerbatimBlockText());
767 
768   ASSERT_EQ(tok::verbatim_block_line,  Toks[4].getKind());
769   ASSERT_EQ(StringRef(" Bbb"),         Toks[4].getVerbatimBlockText());
770 
771   ASSERT_EQ(tok::verbatim_block_end,   Toks[5].getKind());
772   ASSERT_EQ(StringRef("endverbatim"),  getVerbatimBlockName(Toks[5]));
773 
774   ASSERT_EQ(tok::newline,              Toks[6].getKind());
775 
776   ASSERT_EQ(tok::text,                 Toks[7].getKind());
777   ASSERT_EQ(StringRef(" "),            Toks[7].getText());
778 
779   ASSERT_EQ(tok::newline,              Toks[8].getKind());
780   ASSERT_EQ(tok::newline,              Toks[9].getKind());
781 }
782 
783 // Complex test for verbatim blocks.
784 TEST_F(CommentLexerTest, VerbatimBlock8) {
785   const char *Source =
786     "/* Meow \\verbatim aaa\\$\\@\n"
787     "bbb \\endverbati\r"
788     "ccc\r\n"
789     "ddd \\endverbatim Blah \\verbatim eee\n"
790     "\\endverbatim BlahBlah*/";
791   std::vector<Token> Toks;
792 
793   lexString(Source, Toks);
794 
795   ASSERT_EQ(14U, Toks.size());
796 
797   ASSERT_EQ(tok::text,                 Toks[0].getKind());
798   ASSERT_EQ(StringRef(" Meow "),       Toks[0].getText());
799 
800   ASSERT_EQ(tok::verbatim_block_begin, Toks[1].getKind());
801   ASSERT_EQ(StringRef("verbatim"),     getVerbatimBlockName(Toks[1]));
802 
803   ASSERT_EQ(tok::verbatim_block_line,  Toks[2].getKind());
804   ASSERT_EQ(StringRef(" aaa\\$\\@"),   Toks[2].getVerbatimBlockText());
805 
806   ASSERT_EQ(tok::verbatim_block_line,  Toks[3].getKind());
807   ASSERT_EQ(StringRef("bbb \\endverbati"), Toks[3].getVerbatimBlockText());
808 
809   ASSERT_EQ(tok::verbatim_block_line,  Toks[4].getKind());
810   ASSERT_EQ(StringRef("ccc"),          Toks[4].getVerbatimBlockText());
811 
812   ASSERT_EQ(tok::verbatim_block_line,  Toks[5].getKind());
813   ASSERT_EQ(StringRef("ddd "),         Toks[5].getVerbatimBlockText());
814 
815   ASSERT_EQ(tok::verbatim_block_end,   Toks[6].getKind());
816   ASSERT_EQ(StringRef("endverbatim"),  getVerbatimBlockName(Toks[6]));
817 
818   ASSERT_EQ(tok::text,                 Toks[7].getKind());
819   ASSERT_EQ(StringRef(" Blah "),       Toks[7].getText());
820 
821   ASSERT_EQ(tok::verbatim_block_begin, Toks[8].getKind());
822   ASSERT_EQ(StringRef("verbatim"),     getVerbatimBlockName(Toks[8]));
823 
824   ASSERT_EQ(tok::verbatim_block_line,  Toks[9].getKind());
825   ASSERT_EQ(StringRef(" eee"),         Toks[9].getVerbatimBlockText());
826 
827   ASSERT_EQ(tok::verbatim_block_end,   Toks[10].getKind());
828   ASSERT_EQ(StringRef("endverbatim"),  getVerbatimBlockName(Toks[10]));
829 
830   ASSERT_EQ(tok::text,                 Toks[11].getKind());
831   ASSERT_EQ(StringRef(" BlahBlah"),    Toks[11].getText());
832 
833   ASSERT_EQ(tok::newline,              Toks[12].getKind());
834   ASSERT_EQ(tok::newline,              Toks[13].getKind());
835 }
836 
837 // LaTeX verbatim blocks.
838 TEST_F(CommentLexerTest, VerbatimBlock9) {
839   const char *Source =
840     "/// \\f$ Aaa \\f$ \\f[ Bbb \\f] \\f{ Ccc \\f}";
841   std::vector<Token> Toks;
842 
843   lexString(Source, Toks);
844 
845   ASSERT_EQ(13U, Toks.size());
846 
847   ASSERT_EQ(tok::text,                 Toks[0].getKind());
848   ASSERT_EQ(StringRef(" "),            Toks[0].getText());
849 
850   ASSERT_EQ(tok::verbatim_block_begin, Toks[1].getKind());
851   ASSERT_EQ(StringRef("f$"),           getVerbatimBlockName(Toks[1]));
852 
853   ASSERT_EQ(tok::verbatim_block_line,  Toks[2].getKind());
854   ASSERT_EQ(StringRef(" Aaa "),        Toks[2].getVerbatimBlockText());
855 
856   ASSERT_EQ(tok::verbatim_block_end,   Toks[3].getKind());
857   ASSERT_EQ(StringRef("f$"),           getVerbatimBlockName(Toks[3]));
858 
859   ASSERT_EQ(tok::text,                 Toks[4].getKind());
860   ASSERT_EQ(StringRef(" "),            Toks[4].getText());
861 
862   ASSERT_EQ(tok::verbatim_block_begin, Toks[5].getKind());
863   ASSERT_EQ(StringRef("f["),           getVerbatimBlockName(Toks[5]));
864 
865   ASSERT_EQ(tok::verbatim_block_line,  Toks[6].getKind());
866   ASSERT_EQ(StringRef(" Bbb "),        Toks[6].getVerbatimBlockText());
867 
868   ASSERT_EQ(tok::verbatim_block_end,   Toks[7].getKind());
869   ASSERT_EQ(StringRef("f]"),           getVerbatimBlockName(Toks[7]));
870 
871   ASSERT_EQ(tok::text,                 Toks[8].getKind());
872   ASSERT_EQ(StringRef(" "),            Toks[8].getText());
873 
874   ASSERT_EQ(tok::verbatim_block_begin, Toks[9].getKind());
875   ASSERT_EQ(StringRef("f{"),           getVerbatimBlockName(Toks[9]));
876 
877   ASSERT_EQ(tok::verbatim_block_line,  Toks[10].getKind());
878   ASSERT_EQ(StringRef(" Ccc "),        Toks[10].getVerbatimBlockText());
879 
880   ASSERT_EQ(tok::verbatim_block_end,   Toks[11].getKind());
881   ASSERT_EQ(StringRef("f}"),           getVerbatimBlockName(Toks[11]));
882 
883   ASSERT_EQ(tok::newline,              Toks[12].getKind());
884 }
885 
886 // Empty verbatim line.
887 TEST_F(CommentLexerTest, VerbatimLine1) {
888   const char *Sources[] = {
889     "/// \\fn\n//",
890     "/** \\fn*/"
891   };
892 
893   for (size_t i = 0, e = array_lengthof(Sources); i != e; i++) {
894     std::vector<Token> Toks;
895 
896     lexString(Sources[i], Toks);
897 
898     ASSERT_EQ(4U, Toks.size());
899 
900     ASSERT_EQ(tok::text,               Toks[0].getKind());
901     ASSERT_EQ(StringRef(" "),          Toks[0].getText());
902 
903     ASSERT_EQ(tok::verbatim_line_name, Toks[1].getKind());
904     ASSERT_EQ(StringRef("fn"),         getVerbatimLineName(Toks[1]));
905 
906     ASSERT_EQ(tok::newline,            Toks[2].getKind());
907     ASSERT_EQ(tok::newline,            Toks[3].getKind());
908   }
909 }
910 
911 // Verbatim line with Doxygen escape sequences, which should not be expanded.
912 TEST_F(CommentLexerTest, VerbatimLine2) {
913   const char *Sources[] = {
914     "/// \\fn void *foo(const char *zzz = \"\\$\");\n//",
915     "/** \\fn void *foo(const char *zzz = \"\\$\");*/"
916   };
917 
918   for (size_t i = 0, e = array_lengthof(Sources); i != e; i++) {
919     std::vector<Token> Toks;
920 
921     lexString(Sources[i], Toks);
922 
923     ASSERT_EQ(5U, Toks.size());
924 
925     ASSERT_EQ(tok::text,               Toks[0].getKind());
926     ASSERT_EQ(StringRef(" "),          Toks[0].getText());
927 
928     ASSERT_EQ(tok::verbatim_line_name, Toks[1].getKind());
929     ASSERT_EQ(StringRef("fn"),         getVerbatimLineName(Toks[1]));
930 
931     ASSERT_EQ(tok::verbatim_line_text, Toks[2].getKind());
932     ASSERT_EQ(StringRef(" void *foo(const char *zzz = \"\\$\");"),
933                                        Toks[2].getVerbatimLineText());
934 
935     ASSERT_EQ(tok::newline,            Toks[3].getKind());
936     ASSERT_EQ(tok::newline,            Toks[4].getKind());
937   }
938 }
939 
940 // Verbatim line should not eat anything from next source line.
941 TEST_F(CommentLexerTest, VerbatimLine3) {
942   const char *Source =
943     "/** \\fn void *foo(const char *zzz = \"\\$\");\n"
944     " * Meow\n"
945     " */";
946 
947   std::vector<Token> Toks;
948 
949   lexString(Source, Toks);
950 
951   ASSERT_EQ(9U, Toks.size());
952 
953   ASSERT_EQ(tok::text,               Toks[0].getKind());
954   ASSERT_EQ(StringRef(" "),          Toks[0].getText());
955 
956   ASSERT_EQ(tok::verbatim_line_name, Toks[1].getKind());
957   ASSERT_EQ(StringRef("fn"),         getVerbatimLineName(Toks[1]));
958 
959   ASSERT_EQ(tok::verbatim_line_text, Toks[2].getKind());
960   ASSERT_EQ(StringRef(" void *foo(const char *zzz = \"\\$\");"),
961                                      Toks[2].getVerbatimLineText());
962   ASSERT_EQ(tok::newline,            Toks[3].getKind());
963 
964   ASSERT_EQ(tok::text,               Toks[4].getKind());
965   ASSERT_EQ(StringRef(" Meow"),      Toks[4].getText());
966   ASSERT_EQ(tok::newline,            Toks[5].getKind());
967 
968   ASSERT_EQ(tok::text,               Toks[6].getKind());
969   ASSERT_EQ(StringRef(" "),          Toks[6].getText());
970 
971   ASSERT_EQ(tok::newline,            Toks[7].getKind());
972   ASSERT_EQ(tok::newline,            Toks[8].getKind());
973 }
974 
975 TEST_F(CommentLexerTest, HTML1) {
976   const char *Source =
977     "// <";
978 
979   std::vector<Token> Toks;
980 
981   lexString(Source, Toks);
982 
983   ASSERT_EQ(3U, Toks.size());
984 
985   ASSERT_EQ(tok::text,      Toks[0].getKind());
986   ASSERT_EQ(StringRef(" "), Toks[0].getText());
987 
988   ASSERT_EQ(tok::text,      Toks[1].getKind());
989   ASSERT_EQ(StringRef("<"), Toks[1].getText());
990 
991   ASSERT_EQ(tok::newline,   Toks[2].getKind());
992 }
993 
994 TEST_F(CommentLexerTest, HTML2) {
995   const char *Source =
996     "// a<2";
997 
998   std::vector<Token> Toks;
999 
1000   lexString(Source, Toks);
1001 
1002   ASSERT_EQ(4U, Toks.size());
1003 
1004   ASSERT_EQ(tok::text,       Toks[0].getKind());
1005   ASSERT_EQ(StringRef(" a"), Toks[0].getText());
1006 
1007   ASSERT_EQ(tok::text,       Toks[1].getKind());
1008   ASSERT_EQ(StringRef("<"),  Toks[1].getText());
1009 
1010   ASSERT_EQ(tok::text,       Toks[2].getKind());
1011   ASSERT_EQ(StringRef("2"),  Toks[2].getText());
1012 
1013   ASSERT_EQ(tok::newline,    Toks[3].getKind());
1014 }
1015 
1016 TEST_F(CommentLexerTest, HTML3) {
1017   const char *Source =
1018     "// < img";
1019 
1020   std::vector<Token> Toks;
1021 
1022   lexString(Source, Toks);
1023 
1024   ASSERT_EQ(4U, Toks.size());
1025 
1026   ASSERT_EQ(tok::text,         Toks[0].getKind());
1027   ASSERT_EQ(StringRef(" "),    Toks[0].getText());
1028 
1029   ASSERT_EQ(tok::text,         Toks[1].getKind());
1030   ASSERT_EQ(StringRef("<"),    Toks[1].getText());
1031 
1032   ASSERT_EQ(tok::text,         Toks[2].getKind());
1033   ASSERT_EQ(StringRef(" img"), Toks[2].getText());
1034 
1035   ASSERT_EQ(tok::newline,      Toks[3].getKind());
1036 }
1037 
1038 TEST_F(CommentLexerTest, HTML4) {
1039   const char *Sources[] = {
1040     "// <img",
1041     "// <img "
1042   };
1043 
1044   for (size_t i = 0, e = array_lengthof(Sources); i != e; i++) {
1045     std::vector<Token> Toks;
1046 
1047     lexString(Sources[i], Toks);
1048 
1049     ASSERT_EQ(3U, Toks.size());
1050 
1051     ASSERT_EQ(tok::text,           Toks[0].getKind());
1052     ASSERT_EQ(StringRef(" "),      Toks[0].getText());
1053 
1054     ASSERT_EQ(tok::html_start_tag, Toks[1].getKind());
1055     ASSERT_EQ(StringRef("img"),    Toks[1].getHTMLTagStartName());
1056 
1057     ASSERT_EQ(tok::newline,        Toks[2].getKind());
1058   }
1059 }
1060 
1061 TEST_F(CommentLexerTest, HTML5) {
1062   const char *Source =
1063     "// <img 42";
1064 
1065   std::vector<Token> Toks;
1066 
1067   lexString(Source, Toks);
1068 
1069   ASSERT_EQ(4U, Toks.size());
1070 
1071   ASSERT_EQ(tok::text,           Toks[0].getKind());
1072   ASSERT_EQ(StringRef(" "),      Toks[0].getText());
1073 
1074   ASSERT_EQ(tok::html_start_tag, Toks[1].getKind());
1075   ASSERT_EQ(StringRef("img"),    Toks[1].getHTMLTagStartName());
1076 
1077   ASSERT_EQ(tok::text,           Toks[2].getKind());
1078   ASSERT_EQ(StringRef("42"),     Toks[2].getText());
1079 
1080   ASSERT_EQ(tok::newline,        Toks[3].getKind());
1081 }
1082 
1083 TEST_F(CommentLexerTest, HTML6) {
1084   const char *Source = "// <img> Meow";
1085 
1086   std::vector<Token> Toks;
1087 
1088   lexString(Source, Toks);
1089 
1090   ASSERT_EQ(5U, Toks.size());
1091 
1092   ASSERT_EQ(tok::text,           Toks[0].getKind());
1093   ASSERT_EQ(StringRef(" "),      Toks[0].getText());
1094 
1095   ASSERT_EQ(tok::html_start_tag, Toks[1].getKind());
1096   ASSERT_EQ(StringRef("img"),    Toks[1].getHTMLTagStartName());
1097 
1098   ASSERT_EQ(tok::html_greater,   Toks[2].getKind());
1099 
1100   ASSERT_EQ(tok::text,           Toks[3].getKind());
1101   ASSERT_EQ(StringRef(" Meow"),  Toks[3].getText());
1102 
1103   ASSERT_EQ(tok::newline,        Toks[4].getKind());
1104 }
1105 
1106 TEST_F(CommentLexerTest, HTML7) {
1107   const char *Source = "// <img=";
1108 
1109   std::vector<Token> Toks;
1110 
1111   lexString(Source, Toks);
1112 
1113   ASSERT_EQ(4U, Toks.size());
1114 
1115   ASSERT_EQ(tok::text,           Toks[0].getKind());
1116   ASSERT_EQ(StringRef(" "),      Toks[0].getText());
1117 
1118   ASSERT_EQ(tok::html_start_tag, Toks[1].getKind());
1119   ASSERT_EQ(StringRef("img"),    Toks[1].getHTMLTagStartName());
1120 
1121   ASSERT_EQ(tok::text,           Toks[2].getKind());
1122   ASSERT_EQ(StringRef("="),      Toks[2].getText());
1123 
1124   ASSERT_EQ(tok::newline,        Toks[3].getKind());
1125 }
1126 
1127 TEST_F(CommentLexerTest, HTML8) {
1128   const char *Source = "// <img src=> Meow";
1129 
1130   std::vector<Token> Toks;
1131 
1132   lexString(Source, Toks);
1133 
1134   ASSERT_EQ(7U, Toks.size());
1135 
1136   ASSERT_EQ(tok::text,           Toks[0].getKind());
1137   ASSERT_EQ(StringRef(" "),      Toks[0].getText());
1138 
1139   ASSERT_EQ(tok::html_start_tag, Toks[1].getKind());
1140   ASSERT_EQ(StringRef("img"),    Toks[1].getHTMLTagStartName());
1141 
1142   ASSERT_EQ(tok::html_ident,     Toks[2].getKind());
1143   ASSERT_EQ(StringRef("src"),   Toks[2].getHTMLIdent());
1144 
1145   ASSERT_EQ(tok::html_equals,    Toks[3].getKind());
1146 
1147   ASSERT_EQ(tok::html_greater,   Toks[4].getKind());
1148 
1149   ASSERT_EQ(tok::text,           Toks[5].getKind());
1150   ASSERT_EQ(StringRef(" Meow"),  Toks[5].getText());
1151 
1152   ASSERT_EQ(tok::newline,        Toks[6].getKind());
1153 }
1154 
1155 TEST_F(CommentLexerTest, HTML9) {
1156   const char *Sources[] = {
1157     "// <img src",
1158     "// <img src "
1159   };
1160 
1161   for (size_t i = 0, e = array_lengthof(Sources); i != e; i++) {
1162     std::vector<Token> Toks;
1163 
1164     lexString(Sources[i], Toks);
1165 
1166     ASSERT_EQ(4U, Toks.size());
1167 
1168     ASSERT_EQ(tok::text,           Toks[0].getKind());
1169     ASSERT_EQ(StringRef(" "),      Toks[0].getText());
1170 
1171     ASSERT_EQ(tok::html_start_tag, Toks[1].getKind());
1172     ASSERT_EQ(StringRef("img"),    Toks[1].getHTMLTagStartName());
1173 
1174     ASSERT_EQ(tok::html_ident,     Toks[2].getKind());
1175     ASSERT_EQ(StringRef("src"),    Toks[2].getHTMLIdent());
1176 
1177     ASSERT_EQ(tok::newline,        Toks[3].getKind());
1178   }
1179 }
1180 
1181 TEST_F(CommentLexerTest, HTML10) {
1182   const char *Sources[] = {
1183     "// <img src=",
1184     "// <img src ="
1185   };
1186 
1187   for (size_t i = 0, e = array_lengthof(Sources); i != e; i++) {
1188     std::vector<Token> Toks;
1189 
1190     lexString(Sources[i], Toks);
1191 
1192     ASSERT_EQ(5U, Toks.size());
1193 
1194     ASSERT_EQ(tok::text,           Toks[0].getKind());
1195     ASSERT_EQ(StringRef(" "),      Toks[0].getText());
1196 
1197     ASSERT_EQ(tok::html_start_tag, Toks[1].getKind());
1198     ASSERT_EQ(StringRef("img"),    Toks[1].getHTMLTagStartName());
1199 
1200     ASSERT_EQ(tok::html_ident,     Toks[2].getKind());
1201     ASSERT_EQ(StringRef("src"),    Toks[2].getHTMLIdent());
1202 
1203     ASSERT_EQ(tok::html_equals,    Toks[3].getKind());
1204 
1205     ASSERT_EQ(tok::newline,        Toks[4].getKind());
1206   }
1207 }
1208 
1209 TEST_F(CommentLexerTest, HTML11) {
1210   const char *Sources[] = {
1211     "// <img src=\"",
1212     "// <img src = \"",
1213     "// <img src=\'",
1214     "// <img src = \'"
1215   };
1216 
1217   for (size_t i = 0, e = array_lengthof(Sources); i != e; i++) {
1218     std::vector<Token> Toks;
1219 
1220     lexString(Sources[i], Toks);
1221 
1222     ASSERT_EQ(6U, Toks.size());
1223 
1224     ASSERT_EQ(tok::text,               Toks[0].getKind());
1225     ASSERT_EQ(StringRef(" "),          Toks[0].getText());
1226 
1227     ASSERT_EQ(tok::html_start_tag,     Toks[1].getKind());
1228     ASSERT_EQ(StringRef("img"),        Toks[1].getHTMLTagStartName());
1229 
1230     ASSERT_EQ(tok::html_ident,         Toks[2].getKind());
1231     ASSERT_EQ(StringRef("src"),        Toks[2].getHTMLIdent());
1232 
1233     ASSERT_EQ(tok::html_equals,        Toks[3].getKind());
1234 
1235     ASSERT_EQ(tok::html_quoted_string, Toks[4].getKind());
1236     ASSERT_EQ(StringRef(""),           Toks[4].getHTMLQuotedString());
1237 
1238     ASSERT_EQ(tok::newline,            Toks[5].getKind());
1239   }
1240 }
1241 
1242 TEST_F(CommentLexerTest, HTML12) {
1243   const char *Source = "// <img src=@";
1244 
1245   std::vector<Token> Toks;
1246 
1247   lexString(Source, Toks);
1248 
1249   ASSERT_EQ(6U, Toks.size());
1250 
1251   ASSERT_EQ(tok::text,           Toks[0].getKind());
1252   ASSERT_EQ(StringRef(" "),      Toks[0].getText());
1253 
1254   ASSERT_EQ(tok::html_start_tag, Toks[1].getKind());
1255   ASSERT_EQ(StringRef("img"),    Toks[1].getHTMLTagStartName());
1256 
1257   ASSERT_EQ(tok::html_ident,     Toks[2].getKind());
1258   ASSERT_EQ(StringRef("src"),    Toks[2].getHTMLIdent());
1259 
1260   ASSERT_EQ(tok::html_equals,    Toks[3].getKind());
1261 
1262   ASSERT_EQ(tok::text,           Toks[4].getKind());
1263   ASSERT_EQ(StringRef("@"),      Toks[4].getText());
1264 
1265   ASSERT_EQ(tok::newline,        Toks[5].getKind());
1266 }
1267 
1268 TEST_F(CommentLexerTest, HTML13) {
1269   const char *Sources[] = {
1270     "// <img src=\"val\\\"\\'val",
1271     "// <img src=\"val\\\"\\'val\"",
1272     "// <img src=\'val\\\"\\'val",
1273     "// <img src=\'val\\\"\\'val\'"
1274   };
1275 
1276   for (size_t i = 0, e = array_lengthof(Sources); i != e; i++) {
1277     std::vector<Token> Toks;
1278 
1279     lexString(Sources[i], Toks);
1280 
1281     ASSERT_EQ(6U, Toks.size());
1282 
1283     ASSERT_EQ(tok::text,                  Toks[0].getKind());
1284     ASSERT_EQ(StringRef(" "),             Toks[0].getText());
1285 
1286     ASSERT_EQ(tok::html_start_tag,        Toks[1].getKind());
1287     ASSERT_EQ(StringRef("img"),           Toks[1].getHTMLTagStartName());
1288 
1289     ASSERT_EQ(tok::html_ident,            Toks[2].getKind());
1290     ASSERT_EQ(StringRef("src"),           Toks[2].getHTMLIdent());
1291 
1292     ASSERT_EQ(tok::html_equals,           Toks[3].getKind());
1293 
1294     ASSERT_EQ(tok::html_quoted_string,    Toks[4].getKind());
1295     ASSERT_EQ(StringRef("val\\\"\\'val"), Toks[4].getHTMLQuotedString());
1296 
1297     ASSERT_EQ(tok::newline,               Toks[5].getKind());
1298   }
1299 }
1300 
1301 TEST_F(CommentLexerTest, HTML14) {
1302   const char *Sources[] = {
1303     "// <img src=\"val\\\"\\'val\">",
1304     "// <img src=\'val\\\"\\'val\'>"
1305   };
1306 
1307   for (size_t i = 0, e = array_lengthof(Sources); i != e; i++) {
1308     std::vector<Token> Toks;
1309 
1310     lexString(Sources[i], Toks);
1311 
1312     ASSERT_EQ(7U, Toks.size());
1313 
1314     ASSERT_EQ(tok::text,                  Toks[0].getKind());
1315     ASSERT_EQ(StringRef(" "),             Toks[0].getText());
1316 
1317     ASSERT_EQ(tok::html_start_tag,        Toks[1].getKind());
1318     ASSERT_EQ(StringRef("img"),           Toks[1].getHTMLTagStartName());
1319 
1320     ASSERT_EQ(tok::html_ident,            Toks[2].getKind());
1321     ASSERT_EQ(StringRef("src"),           Toks[2].getHTMLIdent());
1322 
1323     ASSERT_EQ(tok::html_equals,           Toks[3].getKind());
1324 
1325     ASSERT_EQ(tok::html_quoted_string,    Toks[4].getKind());
1326     ASSERT_EQ(StringRef("val\\\"\\'val"), Toks[4].getHTMLQuotedString());
1327 
1328     ASSERT_EQ(tok::html_greater,          Toks[5].getKind());
1329 
1330     ASSERT_EQ(tok::newline,               Toks[6].getKind());
1331   }
1332 }
1333 
1334 TEST_F(CommentLexerTest, HTML15) {
1335   const char *Sources[] = {
1336     "// <img/>",
1337     "// <img />"
1338   };
1339 
1340   for (size_t i = 0, e = array_lengthof(Sources); i != e; i++) {
1341     std::vector<Token> Toks;
1342 
1343     lexString(Sources[i], Toks);
1344 
1345     ASSERT_EQ(4U, Toks.size());
1346 
1347     ASSERT_EQ(tok::text,               Toks[0].getKind());
1348     ASSERT_EQ(StringRef(" "),          Toks[0].getText());
1349 
1350     ASSERT_EQ(tok::html_start_tag,     Toks[1].getKind());
1351     ASSERT_EQ(StringRef("img"),        Toks[1].getHTMLTagStartName());
1352 
1353     ASSERT_EQ(tok::html_slash_greater, Toks[2].getKind());
1354 
1355     ASSERT_EQ(tok::newline,            Toks[3].getKind());
1356   }
1357 }
1358 
1359 TEST_F(CommentLexerTest, HTML16) {
1360   const char *Sources[] = {
1361     "// <img/ Aaa",
1362     "// <img / Aaa"
1363   };
1364 
1365   for (size_t i = 0, e = array_lengthof(Sources); i != e; i++) {
1366     std::vector<Token> Toks;
1367 
1368     lexString(Sources[i], Toks);
1369 
1370     ASSERT_EQ(5U, Toks.size());
1371 
1372     ASSERT_EQ(tok::text,               Toks[0].getKind());
1373     ASSERT_EQ(StringRef(" "),          Toks[0].getText());
1374 
1375     ASSERT_EQ(tok::html_start_tag,     Toks[1].getKind());
1376     ASSERT_EQ(StringRef("img"),        Toks[1].getHTMLTagStartName());
1377 
1378     ASSERT_EQ(tok::text,               Toks[2].getKind());
1379     ASSERT_EQ(StringRef("/"),          Toks[2].getText());
1380 
1381     ASSERT_EQ(tok::text,               Toks[3].getKind());
1382     ASSERT_EQ(StringRef(" Aaa"),       Toks[3].getText());
1383 
1384     ASSERT_EQ(tok::newline,            Toks[4].getKind());
1385   }
1386 }
1387 
1388 TEST_F(CommentLexerTest, HTML17) {
1389   const char *Source = "// </";
1390 
1391   std::vector<Token> Toks;
1392 
1393   lexString(Source, Toks);
1394 
1395   ASSERT_EQ(3U, Toks.size());
1396 
1397   ASSERT_EQ(tok::text,       Toks[0].getKind());
1398   ASSERT_EQ(StringRef(" "),  Toks[0].getText());
1399 
1400   ASSERT_EQ(tok::text,       Toks[1].getKind());
1401   ASSERT_EQ(StringRef("</"), Toks[1].getText());
1402 
1403   ASSERT_EQ(tok::newline,    Toks[2].getKind());
1404 }
1405 
1406 TEST_F(CommentLexerTest, HTML18) {
1407   const char *Source = "// </@";
1408 
1409   std::vector<Token> Toks;
1410 
1411   lexString(Source, Toks);
1412 
1413   ASSERT_EQ(4U, Toks.size());
1414 
1415   ASSERT_EQ(tok::text,       Toks[0].getKind());
1416   ASSERT_EQ(StringRef(" "),  Toks[0].getText());
1417 
1418   ASSERT_EQ(tok::text,       Toks[1].getKind());
1419   ASSERT_EQ(StringRef("</"), Toks[1].getText());
1420 
1421   ASSERT_EQ(tok::text,       Toks[2].getKind());
1422   ASSERT_EQ(StringRef("@"),  Toks[2].getText());
1423 
1424   ASSERT_EQ(tok::newline,    Toks[3].getKind());
1425 }
1426 
1427 TEST_F(CommentLexerTest, HTML19) {
1428   const char *Source = "// </img";
1429 
1430   std::vector<Token> Toks;
1431 
1432   lexString(Source, Toks);
1433 
1434   ASSERT_EQ(3U, Toks.size());
1435 
1436   ASSERT_EQ(tok::text,         Toks[0].getKind());
1437   ASSERT_EQ(StringRef(" "),    Toks[0].getText());
1438 
1439   ASSERT_EQ(tok::html_end_tag, Toks[1].getKind());
1440   ASSERT_EQ(StringRef("img"),  Toks[1].getHTMLTagEndName());
1441 
1442   ASSERT_EQ(tok::newline,      Toks[2].getKind());
1443 }
1444 
1445 TEST_F(CommentLexerTest, NotAKnownHTMLTag1) {
1446   const char *Source = "// <tag>";
1447 
1448   std::vector<Token> Toks;
1449 
1450   lexString(Source, Toks);
1451 
1452   ASSERT_EQ(4U, Toks.size());
1453 
1454   ASSERT_EQ(tok::text,         Toks[0].getKind());
1455   ASSERT_EQ(StringRef(" "),    Toks[0].getText());
1456 
1457   ASSERT_EQ(tok::text,         Toks[1].getKind());
1458   ASSERT_EQ(StringRef("<tag"), Toks[1].getText());
1459 
1460   ASSERT_EQ(tok::text,         Toks[2].getKind());
1461   ASSERT_EQ(StringRef(">"),    Toks[2].getText());
1462 
1463   ASSERT_EQ(tok::newline,      Toks[3].getKind());
1464 }
1465 
1466 TEST_F(CommentLexerTest, NotAKnownHTMLTag2) {
1467   const char *Source = "// </tag>";
1468 
1469   std::vector<Token> Toks;
1470 
1471   lexString(Source, Toks);
1472 
1473   ASSERT_EQ(4U, Toks.size());
1474 
1475   ASSERT_EQ(tok::text,          Toks[0].getKind());
1476   ASSERT_EQ(StringRef(" "),     Toks[0].getText());
1477 
1478   ASSERT_EQ(tok::text,          Toks[1].getKind());
1479   ASSERT_EQ(StringRef("</tag"), Toks[1].getText());
1480 
1481   ASSERT_EQ(tok::text,          Toks[2].getKind());
1482   ASSERT_EQ(StringRef(">"),     Toks[2].getText());
1483 
1484   ASSERT_EQ(tok::newline,       Toks[3].getKind());
1485 }
1486 
1487 TEST_F(CommentLexerTest, HTMLCharacterReferences1) {
1488   const char *Source = "// &";
1489 
1490   std::vector<Token> Toks;
1491 
1492   lexString(Source, Toks);
1493 
1494   ASSERT_EQ(3U, Toks.size());
1495 
1496   ASSERT_EQ(tok::text,         Toks[0].getKind());
1497   ASSERT_EQ(StringRef(" "),    Toks[0].getText());
1498 
1499   ASSERT_EQ(tok::text,         Toks[1].getKind());
1500   ASSERT_EQ(StringRef("&"),    Toks[1].getText());
1501 
1502   ASSERT_EQ(tok::newline,      Toks[2].getKind());
1503 }
1504 
1505 TEST_F(CommentLexerTest, HTMLCharacterReferences2) {
1506   const char *Source = "// &!";
1507 
1508   std::vector<Token> Toks;
1509 
1510   lexString(Source, Toks);
1511 
1512   ASSERT_EQ(4U, Toks.size());
1513 
1514   ASSERT_EQ(tok::text,         Toks[0].getKind());
1515   ASSERT_EQ(StringRef(" "),    Toks[0].getText());
1516 
1517   ASSERT_EQ(tok::text,         Toks[1].getKind());
1518   ASSERT_EQ(StringRef("&"),    Toks[1].getText());
1519 
1520   ASSERT_EQ(tok::text,         Toks[2].getKind());
1521   ASSERT_EQ(StringRef("!"),    Toks[2].getText());
1522 
1523   ASSERT_EQ(tok::newline,      Toks[3].getKind());
1524 }
1525 
1526 TEST_F(CommentLexerTest, HTMLCharacterReferences3) {
1527   const char *Source = "// &amp";
1528 
1529   std::vector<Token> Toks;
1530 
1531   lexString(Source, Toks);
1532 
1533   ASSERT_EQ(3U, Toks.size());
1534 
1535   ASSERT_EQ(tok::text,         Toks[0].getKind());
1536   ASSERT_EQ(StringRef(" "),    Toks[0].getText());
1537 
1538   ASSERT_EQ(tok::text,         Toks[1].getKind());
1539   ASSERT_EQ(StringRef("&amp"), Toks[1].getText());
1540 
1541   ASSERT_EQ(tok::newline,      Toks[2].getKind());
1542 }
1543 
1544 TEST_F(CommentLexerTest, HTMLCharacterReferences4) {
1545   const char *Source = "// &amp!";
1546 
1547   std::vector<Token> Toks;
1548 
1549   lexString(Source, Toks);
1550 
1551   ASSERT_EQ(4U, Toks.size());
1552 
1553   ASSERT_EQ(tok::text,         Toks[0].getKind());
1554   ASSERT_EQ(StringRef(" "),    Toks[0].getText());
1555 
1556   ASSERT_EQ(tok::text,         Toks[1].getKind());
1557   ASSERT_EQ(StringRef("&amp"), Toks[1].getText());
1558 
1559   ASSERT_EQ(tok::text,         Toks[2].getKind());
1560   ASSERT_EQ(StringRef("!"),    Toks[2].getText());
1561 
1562   ASSERT_EQ(tok::newline,      Toks[3].getKind());
1563 }
1564 
1565 TEST_F(CommentLexerTest, HTMLCharacterReferences5) {
1566   const char *Source = "// &#";
1567 
1568   std::vector<Token> Toks;
1569 
1570   lexString(Source, Toks);
1571 
1572   ASSERT_EQ(3U, Toks.size());
1573 
1574   ASSERT_EQ(tok::text,         Toks[0].getKind());
1575   ASSERT_EQ(StringRef(" "),    Toks[0].getText());
1576 
1577   ASSERT_EQ(tok::text,         Toks[1].getKind());
1578   ASSERT_EQ(StringRef("&#"),   Toks[1].getText());
1579 
1580   ASSERT_EQ(tok::newline,      Toks[2].getKind());
1581 }
1582 
1583 TEST_F(CommentLexerTest, HTMLCharacterReferences6) {
1584   const char *Source = "// &#a";
1585 
1586   std::vector<Token> Toks;
1587 
1588   lexString(Source, Toks);
1589 
1590   ASSERT_EQ(4U, Toks.size());
1591 
1592   ASSERT_EQ(tok::text,         Toks[0].getKind());
1593   ASSERT_EQ(StringRef(" "),    Toks[0].getText());
1594 
1595   ASSERT_EQ(tok::text,         Toks[1].getKind());
1596   ASSERT_EQ(StringRef("&#"),   Toks[1].getText());
1597 
1598   ASSERT_EQ(tok::text,         Toks[2].getKind());
1599   ASSERT_EQ(StringRef("a"),    Toks[2].getText());
1600 
1601   ASSERT_EQ(tok::newline,      Toks[3].getKind());
1602 }
1603 
1604 TEST_F(CommentLexerTest, HTMLCharacterReferences7) {
1605   const char *Source = "// &#42";
1606 
1607   std::vector<Token> Toks;
1608 
1609   lexString(Source, Toks);
1610 
1611   ASSERT_EQ(3U, Toks.size());
1612 
1613   ASSERT_EQ(tok::text,         Toks[0].getKind());
1614   ASSERT_EQ(StringRef(" "),    Toks[0].getText());
1615 
1616   ASSERT_EQ(tok::text,         Toks[1].getKind());
1617   ASSERT_EQ(StringRef("&#42"), Toks[1].getText());
1618 
1619   ASSERT_EQ(tok::newline,      Toks[2].getKind());
1620 }
1621 
1622 TEST_F(CommentLexerTest, HTMLCharacterReferences8) {
1623   const char *Source = "// &#42a";
1624 
1625   std::vector<Token> Toks;
1626 
1627   lexString(Source, Toks);
1628 
1629   ASSERT_EQ(4U, Toks.size());
1630 
1631   ASSERT_EQ(tok::text,         Toks[0].getKind());
1632   ASSERT_EQ(StringRef(" "),    Toks[0].getText());
1633 
1634   ASSERT_EQ(tok::text,         Toks[1].getKind());
1635   ASSERT_EQ(StringRef("&#42"), Toks[1].getText());
1636 
1637   ASSERT_EQ(tok::text,         Toks[2].getKind());
1638   ASSERT_EQ(StringRef("a"),    Toks[2].getText());
1639 
1640   ASSERT_EQ(tok::newline,      Toks[3].getKind());
1641 }
1642 
1643 TEST_F(CommentLexerTest, HTMLCharacterReferences9) {
1644   const char *Source = "// &#x";
1645 
1646   std::vector<Token> Toks;
1647 
1648   lexString(Source, Toks);
1649 
1650   ASSERT_EQ(3U, Toks.size());
1651 
1652   ASSERT_EQ(tok::text,         Toks[0].getKind());
1653   ASSERT_EQ(StringRef(" "),    Toks[0].getText());
1654 
1655   ASSERT_EQ(tok::text,         Toks[1].getKind());
1656   ASSERT_EQ(StringRef("&#x"),  Toks[1].getText());
1657 
1658   ASSERT_EQ(tok::newline,      Toks[2].getKind());
1659 }
1660 
1661 TEST_F(CommentLexerTest, HTMLCharacterReferences10) {
1662   const char *Source = "// &#xz";
1663 
1664   std::vector<Token> Toks;
1665 
1666   lexString(Source, Toks);
1667 
1668   ASSERT_EQ(4U, Toks.size());
1669 
1670   ASSERT_EQ(tok::text,         Toks[0].getKind());
1671   ASSERT_EQ(StringRef(" "),    Toks[0].getText());
1672 
1673   ASSERT_EQ(tok::text,         Toks[1].getKind());
1674   ASSERT_EQ(StringRef("&#x"),  Toks[1].getText());
1675 
1676   ASSERT_EQ(tok::text,         Toks[2].getKind());
1677   ASSERT_EQ(StringRef("z"),    Toks[2].getText());
1678 
1679   ASSERT_EQ(tok::newline,      Toks[3].getKind());
1680 }
1681 
1682 TEST_F(CommentLexerTest, HTMLCharacterReferences11) {
1683   const char *Source = "// &#xab";
1684 
1685   std::vector<Token> Toks;
1686 
1687   lexString(Source, Toks);
1688 
1689   ASSERT_EQ(3U, Toks.size());
1690 
1691   ASSERT_EQ(tok::text,          Toks[0].getKind());
1692   ASSERT_EQ(StringRef(" "),     Toks[0].getText());
1693 
1694   ASSERT_EQ(tok::text,          Toks[1].getKind());
1695   ASSERT_EQ(StringRef("&#xab"), Toks[1].getText());
1696 
1697   ASSERT_EQ(tok::newline,       Toks[2].getKind());
1698 }
1699 
1700 TEST_F(CommentLexerTest, HTMLCharacterReferences12) {
1701   const char *Source = "// &#xaBz";
1702 
1703   std::vector<Token> Toks;
1704 
1705   lexString(Source, Toks);
1706 
1707   ASSERT_EQ(4U, Toks.size());
1708 
1709   ASSERT_EQ(tok::text,          Toks[0].getKind());
1710   ASSERT_EQ(StringRef(" "),     Toks[0].getText());
1711 
1712   ASSERT_EQ(tok::text,          Toks[1].getKind());
1713   ASSERT_EQ(StringRef("&#xaB"), Toks[1].getText());
1714 
1715   ASSERT_EQ(tok::text,          Toks[2].getKind());
1716   ASSERT_EQ(StringRef("z"),     Toks[2].getText());
1717 
1718   ASSERT_EQ(tok::newline,       Toks[3].getKind());
1719 }
1720 
1721 TEST_F(CommentLexerTest, HTMLCharacterReferences13) {
1722   const char *Source = "// &amp;";
1723 
1724   std::vector<Token> Toks;
1725 
1726   lexString(Source, Toks);
1727 
1728   ASSERT_EQ(3U, Toks.size());
1729 
1730   ASSERT_EQ(tok::text,          Toks[0].getKind());
1731   ASSERT_EQ(StringRef(" "),     Toks[0].getText());
1732 
1733   ASSERT_EQ(tok::text,          Toks[1].getKind());
1734   ASSERT_EQ(StringRef("&"),     Toks[1].getText());
1735 
1736   ASSERT_EQ(tok::newline,       Toks[2].getKind());
1737 }
1738 
1739 TEST_F(CommentLexerTest, HTMLCharacterReferences14) {
1740   const char *Source = "// &amp;&lt;";
1741 
1742   std::vector<Token> Toks;
1743 
1744   lexString(Source, Toks);
1745 
1746   ASSERT_EQ(4U, Toks.size());
1747 
1748   ASSERT_EQ(tok::text,          Toks[0].getKind());
1749   ASSERT_EQ(StringRef(" "),     Toks[0].getText());
1750 
1751   ASSERT_EQ(tok::text,          Toks[1].getKind());
1752   ASSERT_EQ(StringRef("&"),     Toks[1].getText());
1753 
1754   ASSERT_EQ(tok::text,          Toks[2].getKind());
1755   ASSERT_EQ(StringRef("<"),     Toks[2].getText());
1756 
1757   ASSERT_EQ(tok::newline,       Toks[3].getKind());
1758 }
1759 
1760 TEST_F(CommentLexerTest, HTMLCharacterReferences15) {
1761   const char *Source = "// &amp; meow";
1762 
1763   std::vector<Token> Toks;
1764 
1765   lexString(Source, Toks);
1766 
1767   ASSERT_EQ(4U, Toks.size());
1768 
1769   ASSERT_EQ(tok::text,          Toks[0].getKind());
1770   ASSERT_EQ(StringRef(" "),     Toks[0].getText());
1771 
1772   ASSERT_EQ(tok::text,          Toks[1].getKind());
1773   ASSERT_EQ(StringRef("&"),     Toks[1].getText());
1774 
1775   ASSERT_EQ(tok::text,          Toks[2].getKind());
1776   ASSERT_EQ(StringRef(" meow"), Toks[2].getText());
1777 
1778   ASSERT_EQ(tok::newline,       Toks[3].getKind());
1779 }
1780 
1781 TEST_F(CommentLexerTest, HTMLCharacterReferences16) {
1782   const char *Sources[] = {
1783     "// &#61;",
1784     "// &#x3d;",
1785     "// &#X3d;",
1786     "// &#X3D;"
1787   };
1788 
1789   for (size_t i = 0, e = array_lengthof(Sources); i != e; i++) {
1790     std::vector<Token> Toks;
1791 
1792     lexString(Sources[i], Toks);
1793 
1794     ASSERT_EQ(3U, Toks.size());
1795 
1796     ASSERT_EQ(tok::text,          Toks[0].getKind());
1797     ASSERT_EQ(StringRef(" "),     Toks[0].getText());
1798 
1799     ASSERT_EQ(tok::text,          Toks[1].getKind());
1800     ASSERT_EQ(StringRef("="),     Toks[1].getText());
1801 
1802     ASSERT_EQ(tok::newline,       Toks[2].getKind());
1803   }
1804 }
1805 
1806 TEST_F(CommentLexerTest, MultipleComments) {
1807   const char *Source =
1808     "// Aaa\n"
1809     "/// Bbb\n"
1810     "/* Ccc\n"
1811     " * Ddd*/\n"
1812     "/** Eee*/";
1813 
1814   std::vector<Token> Toks;
1815 
1816   lexString(Source, Toks);
1817 
1818   ASSERT_EQ(12U, Toks.size());
1819 
1820   ASSERT_EQ(tok::text,           Toks[0].getKind());
1821   ASSERT_EQ(StringRef(" Aaa"),   Toks[0].getText());
1822   ASSERT_EQ(tok::newline,        Toks[1].getKind());
1823 
1824   ASSERT_EQ(tok::text,           Toks[2].getKind());
1825   ASSERT_EQ(StringRef(" Bbb"),   Toks[2].getText());
1826   ASSERT_EQ(tok::newline,        Toks[3].getKind());
1827 
1828   ASSERT_EQ(tok::text,           Toks[4].getKind());
1829   ASSERT_EQ(StringRef(" Ccc"),   Toks[4].getText());
1830   ASSERT_EQ(tok::newline,        Toks[5].getKind());
1831 
1832   ASSERT_EQ(tok::text,           Toks[6].getKind());
1833   ASSERT_EQ(StringRef(" Ddd"),   Toks[6].getText());
1834   ASSERT_EQ(tok::newline,        Toks[7].getKind());
1835   ASSERT_EQ(tok::newline,        Toks[8].getKind());
1836 
1837   ASSERT_EQ(tok::text,           Toks[9].getKind());
1838   ASSERT_EQ(StringRef(" Eee"),   Toks[9].getText());
1839 
1840   ASSERT_EQ(tok::newline,        Toks[10].getKind());
1841   ASSERT_EQ(tok::newline,        Toks[11].getKind());
1842 }
1843 
1844 } // end namespace comments
1845 } // end namespace clang
1846 
1847