1 //===- unittests/Lex/PPCallbacksTest.cpp - PPCallbacks tests ------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===--------------------------------------------------------------===//
8 
9 #include "clang/Lex/Preprocessor.h"
10 #include "clang/AST/ASTConsumer.h"
11 #include "clang/AST/ASTContext.h"
12 #include "clang/Basic/Diagnostic.h"
13 #include "clang/Basic/DiagnosticOptions.h"
14 #include "clang/Basic/FileManager.h"
15 #include "clang/Basic/LangOptions.h"
16 #include "clang/Basic/SourceManager.h"
17 #include "clang/Basic/TargetInfo.h"
18 #include "clang/Basic/TargetOptions.h"
19 #include "clang/Lex/HeaderSearch.h"
20 #include "clang/Lex/HeaderSearchOptions.h"
21 #include "clang/Lex/ModuleLoader.h"
22 #include "clang/Lex/PreprocessorOptions.h"
23 #include "clang/Parse/Parser.h"
24 #include "clang/Sema/Sema.h"
25 #include "llvm/ADT/SmallString.h"
26 #include "llvm/Support/Path.h"
27 #include "gtest/gtest.h"
28 
29 using namespace clang;
30 
31 namespace {
32 
33 // Stub to collect data from InclusionDirective callbacks.
34 class InclusionDirectiveCallbacks : public PPCallbacks {
35 public:
36   void InclusionDirective(SourceLocation HashLoc, const Token &IncludeTok,
37                           StringRef FileName, bool IsAngled,
38                           CharSourceRange FilenameRange,
39                           Optional<FileEntryRef> File, StringRef SearchPath,
40                           StringRef RelativePath, const Module *Imported,
41                           SrcMgr::CharacteristicKind FileType) override {
42     this->HashLoc = HashLoc;
43     this->IncludeTok = IncludeTok;
44     this->FileName = FileName.str();
45     this->IsAngled = IsAngled;
46     this->FilenameRange = FilenameRange;
47     this->File = File;
48     this->SearchPath = SearchPath.str();
49     this->RelativePath = RelativePath.str();
50     this->Imported = Imported;
51     this->FileType = FileType;
52   }
53 
54   SourceLocation HashLoc;
55   Token IncludeTok;
56   SmallString<16> FileName;
57   bool IsAngled;
58   CharSourceRange FilenameRange;
59   Optional<FileEntryRef> File;
60   SmallString<16> SearchPath;
61   SmallString<16> RelativePath;
62   const Module* Imported;
63   SrcMgr::CharacteristicKind FileType;
64 };
65 
66 class CondDirectiveCallbacks : public PPCallbacks {
67 public:
68   struct Result {
69     SourceRange ConditionRange;
70     ConditionValueKind ConditionValue;
71 
72     Result(SourceRange R, ConditionValueKind K)
73         : ConditionRange(R), ConditionValue(K) {}
74   };
75 
76   std::vector<Result> Results;
77 
78   void If(SourceLocation Loc, SourceRange ConditionRange,
79           ConditionValueKind ConditionValue) override {
80     Results.emplace_back(ConditionRange, ConditionValue);
81   }
82 
83   void Elif(SourceLocation Loc, SourceRange ConditionRange,
84             ConditionValueKind ConditionValue, SourceLocation IfLoc) override {
85     Results.emplace_back(ConditionRange, ConditionValue);
86   }
87 };
88 
89 // Stub to collect data from PragmaOpenCLExtension callbacks.
90 class PragmaOpenCLExtensionCallbacks : public PPCallbacks {
91 public:
92   typedef struct {
93     SmallString<16> Name;
94     unsigned State;
95   } CallbackParameters;
96 
97   PragmaOpenCLExtensionCallbacks() : Name("Not called."), State(99) {}
98 
99   void PragmaOpenCLExtension(clang::SourceLocation NameLoc,
100                              const clang::IdentifierInfo *Name,
101                              clang::SourceLocation StateLoc,
102                              unsigned State) override {
103       this->NameLoc = NameLoc;
104       this->Name = Name->getName();
105       this->StateLoc = StateLoc;
106       this->State = State;
107   }
108 
109   SourceLocation NameLoc;
110   SmallString<16> Name;
111   SourceLocation StateLoc;
112   unsigned State;
113 };
114 
115 class PragmaMarkCallbacks : public PPCallbacks {
116 public:
117   struct Mark {
118     SourceLocation Location;
119     std::string Trivia;
120   };
121 
122   std::vector<Mark> Marks;
123 
124   void PragmaMark(SourceLocation Loc, StringRef Trivia) override {
125     Marks.emplace_back(Mark{Loc, Trivia.str()});
126   }
127 };
128 
129 // PPCallbacks test fixture.
130 class PPCallbacksTest : public ::testing::Test {
131 protected:
132   PPCallbacksTest()
133       : InMemoryFileSystem(new llvm::vfs::InMemoryFileSystem),
134         FileMgr(FileSystemOptions(), InMemoryFileSystem),
135         DiagID(new DiagnosticIDs()), DiagOpts(new DiagnosticOptions()),
136         Diags(DiagID, DiagOpts.get(), new IgnoringDiagConsumer()),
137         SourceMgr(Diags, FileMgr), TargetOpts(new TargetOptions()) {
138     TargetOpts->Triple = "x86_64-apple-darwin11.1.0";
139     Target = TargetInfo::CreateTargetInfo(Diags, TargetOpts);
140   }
141 
142   IntrusiveRefCntPtr<llvm::vfs::InMemoryFileSystem> InMemoryFileSystem;
143   FileManager FileMgr;
144   IntrusiveRefCntPtr<DiagnosticIDs> DiagID;
145   IntrusiveRefCntPtr<DiagnosticOptions> DiagOpts;
146   DiagnosticsEngine Diags;
147   SourceManager SourceMgr;
148   LangOptions LangOpts;
149   std::shared_ptr<TargetOptions> TargetOpts;
150   IntrusiveRefCntPtr<TargetInfo> Target;
151 
152   // Register a header path as a known file and add its location
153   // to search path.
154   void AddFakeHeader(HeaderSearch &HeaderInfo, const char *HeaderPath,
155                      bool IsSystemHeader) {
156     // Tell FileMgr about header.
157     InMemoryFileSystem->addFile(HeaderPath, 0,
158                                 llvm::MemoryBuffer::getMemBuffer("\n"));
159 
160     // Add header's parent path to search path.
161     StringRef SearchPath = llvm::sys::path::parent_path(HeaderPath);
162     auto DE = FileMgr.getOptionalDirectoryRef(SearchPath);
163     DirectoryLookup DL(*DE, SrcMgr::C_User, false);
164     HeaderInfo.AddSearchPath(DL, IsSystemHeader);
165   }
166 
167   // Get the raw source string of the range.
168   StringRef GetSourceString(CharSourceRange Range) {
169     const char* B = SourceMgr.getCharacterData(Range.getBegin());
170     const char* E = SourceMgr.getCharacterData(Range.getEnd());
171 
172     return StringRef(B, E - B);
173   }
174 
175   StringRef GetSourceStringToEnd(CharSourceRange Range) {
176     const char *B = SourceMgr.getCharacterData(Range.getBegin());
177     const char *E = SourceMgr.getCharacterData(Range.getEnd());
178 
179     return StringRef(
180         B,
181         E - B + Lexer::MeasureTokenLength(Range.getEnd(), SourceMgr, LangOpts));
182   }
183 
184   // Run lexer over SourceText and collect FilenameRange from
185   // the InclusionDirective callback.
186   CharSourceRange InclusionDirectiveFilenameRange(const char *SourceText,
187                                                   const char *HeaderPath,
188                                                   bool SystemHeader) {
189     std::unique_ptr<llvm::MemoryBuffer> Buf =
190         llvm::MemoryBuffer::getMemBuffer(SourceText);
191     SourceMgr.setMainFileID(SourceMgr.createFileID(std::move(Buf)));
192 
193     TrivialModuleLoader ModLoader;
194 
195     HeaderSearch HeaderInfo(std::make_shared<HeaderSearchOptions>(), SourceMgr,
196                             Diags, LangOpts, Target.get());
197     AddFakeHeader(HeaderInfo, HeaderPath, SystemHeader);
198 
199     Preprocessor PP(std::make_shared<PreprocessorOptions>(), Diags, LangOpts,
200                     SourceMgr, HeaderInfo, ModLoader,
201                     /*IILookup =*/nullptr,
202                     /*OwnsHeaderSearch =*/false);
203     return InclusionDirectiveCallback(PP)->FilenameRange;
204   }
205 
206   SrcMgr::CharacteristicKind InclusionDirectiveCharacteristicKind(
207       const char *SourceText, const char *HeaderPath, bool SystemHeader) {
208     std::unique_ptr<llvm::MemoryBuffer> Buf =
209         llvm::MemoryBuffer::getMemBuffer(SourceText);
210     SourceMgr.setMainFileID(SourceMgr.createFileID(std::move(Buf)));
211 
212     TrivialModuleLoader ModLoader;
213 
214     HeaderSearch HeaderInfo(std::make_shared<HeaderSearchOptions>(), SourceMgr,
215                             Diags, LangOpts, Target.get());
216     AddFakeHeader(HeaderInfo, HeaderPath, SystemHeader);
217 
218     Preprocessor PP(std::make_shared<PreprocessorOptions>(), Diags, LangOpts,
219                     SourceMgr, HeaderInfo, ModLoader,
220                     /*IILookup =*/nullptr,
221                     /*OwnsHeaderSearch =*/false);
222     return InclusionDirectiveCallback(PP)->FileType;
223   }
224 
225   InclusionDirectiveCallbacks *InclusionDirectiveCallback(Preprocessor &PP) {
226     PP.Initialize(*Target);
227     InclusionDirectiveCallbacks* Callbacks = new InclusionDirectiveCallbacks;
228     PP.addPPCallbacks(std::unique_ptr<PPCallbacks>(Callbacks));
229 
230     // Lex source text.
231     PP.EnterMainSourceFile();
232 
233     while (true) {
234       Token Tok;
235       PP.Lex(Tok);
236       if (Tok.is(tok::eof))
237         break;
238     }
239 
240     // Callbacks have been executed at this point -- return filename range.
241     return Callbacks;
242   }
243 
244   std::vector<CondDirectiveCallbacks::Result>
245   DirectiveExprRange(StringRef SourceText) {
246     TrivialModuleLoader ModLoader;
247     std::unique_ptr<llvm::MemoryBuffer> Buf =
248         llvm::MemoryBuffer::getMemBuffer(SourceText);
249     SourceMgr.setMainFileID(SourceMgr.createFileID(std::move(Buf)));
250     HeaderSearch HeaderInfo(std::make_shared<HeaderSearchOptions>(), SourceMgr,
251                             Diags, LangOpts, Target.get());
252     Preprocessor PP(std::make_shared<PreprocessorOptions>(), Diags, LangOpts,
253                     SourceMgr, HeaderInfo, ModLoader,
254                     /*IILookup =*/nullptr,
255                     /*OwnsHeaderSearch =*/false);
256     PP.Initialize(*Target);
257     auto *Callbacks = new CondDirectiveCallbacks;
258     PP.addPPCallbacks(std::unique_ptr<PPCallbacks>(Callbacks));
259 
260     // Lex source text.
261     PP.EnterMainSourceFile();
262 
263     while (true) {
264       Token Tok;
265       PP.Lex(Tok);
266       if (Tok.is(tok::eof))
267         break;
268     }
269 
270     return Callbacks->Results;
271   }
272 
273   std::vector<PragmaMarkCallbacks::Mark>
274   PragmaMarkCall(const char *SourceText) {
275     std::unique_ptr<llvm::MemoryBuffer> SourceBuf =
276         llvm::MemoryBuffer::getMemBuffer(SourceText, "test.c");
277     SourceMgr.setMainFileID(SourceMgr.createFileID(std::move(SourceBuf)));
278 
279     HeaderSearch HeaderInfo(std::make_shared<HeaderSearchOptions>(), SourceMgr,
280                             Diags, LangOpts, Target.get());
281     TrivialModuleLoader ModLoader;
282 
283     Preprocessor PP(std::make_shared<PreprocessorOptions>(), Diags, LangOpts,
284                     SourceMgr, HeaderInfo, ModLoader, /*IILookup=*/nullptr,
285                     /*OwnsHeaderSearch=*/false);
286     PP.Initialize(*Target);
287 
288     auto *Callbacks = new PragmaMarkCallbacks;
289     PP.addPPCallbacks(std::unique_ptr<PPCallbacks>(Callbacks));
290 
291     // Lex source text.
292     PP.EnterMainSourceFile();
293     while (true) {
294       Token Tok;
295       PP.Lex(Tok);
296       if (Tok.is(tok::eof))
297         break;
298     }
299 
300     return Callbacks->Marks;
301   }
302 
303   PragmaOpenCLExtensionCallbacks::CallbackParameters
304   PragmaOpenCLExtensionCall(const char *SourceText) {
305     LangOptions OpenCLLangOpts;
306     OpenCLLangOpts.OpenCL = 1;
307 
308     std::unique_ptr<llvm::MemoryBuffer> SourceBuf =
309         llvm::MemoryBuffer::getMemBuffer(SourceText, "test.cl");
310     SourceMgr.setMainFileID(SourceMgr.createFileID(std::move(SourceBuf)));
311 
312     TrivialModuleLoader ModLoader;
313     HeaderSearch HeaderInfo(std::make_shared<HeaderSearchOptions>(), SourceMgr,
314                             Diags, OpenCLLangOpts, Target.get());
315 
316     Preprocessor PP(std::make_shared<PreprocessorOptions>(), Diags,
317                     OpenCLLangOpts, SourceMgr, HeaderInfo, ModLoader,
318                     /*IILookup =*/nullptr,
319                     /*OwnsHeaderSearch =*/false);
320     PP.Initialize(*Target);
321 
322     // parser actually sets correct pragma handlers for preprocessor
323     // according to LangOptions, so we init Parser to register opencl
324     // pragma handlers
325     ASTContext Context(OpenCLLangOpts, SourceMgr, PP.getIdentifierTable(),
326                        PP.getSelectorTable(), PP.getBuiltinInfo(), PP.TUKind);
327     Context.InitBuiltinTypes(*Target);
328 
329     ASTConsumer Consumer;
330     Sema S(PP, Context, Consumer);
331     Parser P(PP, S, false);
332     PragmaOpenCLExtensionCallbacks* Callbacks = new PragmaOpenCLExtensionCallbacks;
333     PP.addPPCallbacks(std::unique_ptr<PPCallbacks>(Callbacks));
334 
335     // Lex source text.
336     PP.EnterMainSourceFile();
337     while (true) {
338       Token Tok;
339       PP.Lex(Tok);
340       if (Tok.is(tok::eof))
341         break;
342     }
343 
344     PragmaOpenCLExtensionCallbacks::CallbackParameters RetVal = {
345       Callbacks->Name,
346       Callbacks->State
347     };
348     return RetVal;
349   }
350 };
351 
352 TEST_F(PPCallbacksTest, UserFileCharacteristics) {
353   const char *Source = "#include \"quoted.h\"\n";
354 
355   SrcMgr::CharacteristicKind Kind =
356       InclusionDirectiveCharacteristicKind(Source, "/quoted.h", false);
357 
358   ASSERT_EQ(SrcMgr::CharacteristicKind::C_User, Kind);
359 }
360 
361 TEST_F(PPCallbacksTest, QuotedFilename) {
362   const char* Source =
363     "#include \"quoted.h\"\n";
364 
365   CharSourceRange Range =
366     InclusionDirectiveFilenameRange(Source, "/quoted.h", false);
367 
368   ASSERT_EQ("\"quoted.h\"", GetSourceString(Range));
369 }
370 
371 TEST_F(PPCallbacksTest, AngledFilename) {
372   const char* Source =
373     "#include <angled.h>\n";
374 
375   CharSourceRange Range =
376     InclusionDirectiveFilenameRange(Source, "/angled.h", true);
377 
378   ASSERT_EQ("<angled.h>", GetSourceString(Range));
379 }
380 
381 TEST_F(PPCallbacksTest, QuotedInMacro) {
382   const char* Source =
383     "#define MACRO_QUOTED \"quoted.h\"\n"
384     "#include MACRO_QUOTED\n";
385 
386   CharSourceRange Range =
387     InclusionDirectiveFilenameRange(Source, "/quoted.h", false);
388 
389   ASSERT_EQ("\"quoted.h\"", GetSourceString(Range));
390 }
391 
392 TEST_F(PPCallbacksTest, AngledInMacro) {
393   const char* Source =
394     "#define MACRO_ANGLED <angled.h>\n"
395     "#include MACRO_ANGLED\n";
396 
397   CharSourceRange Range =
398     InclusionDirectiveFilenameRange(Source, "/angled.h", true);
399 
400   ASSERT_EQ("<angled.h>", GetSourceString(Range));
401 }
402 
403 TEST_F(PPCallbacksTest, StringizedMacroArgument) {
404   const char* Source =
405     "#define MACRO_STRINGIZED(x) #x\n"
406     "#include MACRO_STRINGIZED(quoted.h)\n";
407 
408   CharSourceRange Range =
409     InclusionDirectiveFilenameRange(Source, "/quoted.h", false);
410 
411   ASSERT_EQ("\"quoted.h\"", GetSourceString(Range));
412 }
413 
414 TEST_F(PPCallbacksTest, ConcatenatedMacroArgument) {
415   const char* Source =
416     "#define MACRO_ANGLED <angled.h>\n"
417     "#define MACRO_CONCAT(x, y) x ## _ ## y\n"
418     "#include MACRO_CONCAT(MACRO, ANGLED)\n";
419 
420   CharSourceRange Range =
421     InclusionDirectiveFilenameRange(Source, "/angled.h", false);
422 
423   ASSERT_EQ("<angled.h>", GetSourceString(Range));
424 }
425 
426 TEST_F(PPCallbacksTest, TrigraphFilename) {
427   const char* Source =
428     "#include \"tri\?\?-graph.h\"\n";
429 
430   CharSourceRange Range =
431     InclusionDirectiveFilenameRange(Source, "/tri~graph.h", false);
432 
433   ASSERT_EQ("\"tri\?\?-graph.h\"", GetSourceString(Range));
434 }
435 
436 TEST_F(PPCallbacksTest, TrigraphInMacro) {
437   const char* Source =
438     "#define MACRO_TRIGRAPH \"tri\?\?-graph.h\"\n"
439     "#include MACRO_TRIGRAPH\n";
440 
441   CharSourceRange Range =
442     InclusionDirectiveFilenameRange(Source, "/tri~graph.h", false);
443 
444   ASSERT_EQ("\"tri\?\?-graph.h\"", GetSourceString(Range));
445 }
446 
447 TEST_F(PPCallbacksTest, OpenCLExtensionPragmaEnabled) {
448   const char* Source =
449     "#pragma OPENCL EXTENSION cl_khr_fp64 : enable\n";
450 
451   PragmaOpenCLExtensionCallbacks::CallbackParameters Parameters =
452     PragmaOpenCLExtensionCall(Source);
453 
454   ASSERT_EQ("cl_khr_fp64", Parameters.Name);
455   unsigned ExpectedState = 1;
456   ASSERT_EQ(ExpectedState, Parameters.State);
457 }
458 
459 TEST_F(PPCallbacksTest, OpenCLExtensionPragmaDisabled) {
460   const char* Source =
461     "#pragma OPENCL EXTENSION cl_khr_fp16 : disable\n";
462 
463   PragmaOpenCLExtensionCallbacks::CallbackParameters Parameters =
464     PragmaOpenCLExtensionCall(Source);
465 
466   ASSERT_EQ("cl_khr_fp16", Parameters.Name);
467   unsigned ExpectedState = 0;
468   ASSERT_EQ(ExpectedState, Parameters.State);
469 }
470 
471 TEST_F(PPCallbacksTest, CollectMarks) {
472   const char *Source =
473     "#pragma mark\n"
474     "#pragma mark\r\n"
475     "#pragma mark - trivia\n"
476     "#pragma mark - trivia\r\n";
477 
478   auto Marks = PragmaMarkCall(Source);
479 
480   ASSERT_EQ(4u, Marks.size());
481   ASSERT_TRUE(Marks[0].Trivia.empty());
482   ASSERT_TRUE(Marks[1].Trivia.empty());
483   ASSERT_FALSE(Marks[2].Trivia.empty());
484   ASSERT_FALSE(Marks[3].Trivia.empty());
485   ASSERT_EQ(" - trivia", Marks[2].Trivia);
486   ASSERT_EQ(" - trivia", Marks[3].Trivia);
487 }
488 
489 TEST_F(PPCallbacksTest, DirectiveExprRanges) {
490   const auto &Results1 = DirectiveExprRange("#if FLUZZY_FLOOF\n#endif\n");
491   EXPECT_EQ(Results1.size(), 1U);
492   EXPECT_EQ(
493       GetSourceStringToEnd(CharSourceRange(Results1[0].ConditionRange, false)),
494       "FLUZZY_FLOOF");
495 
496   const auto &Results2 = DirectiveExprRange("#if 1 + 4 < 7\n#endif\n");
497   EXPECT_EQ(Results2.size(), 1U);
498   EXPECT_EQ(
499       GetSourceStringToEnd(CharSourceRange(Results2[0].ConditionRange, false)),
500       "1 + 4 < 7");
501 
502   const auto &Results3 = DirectiveExprRange("#if 1 + \\\n  2\n#endif\n");
503   EXPECT_EQ(Results3.size(), 1U);
504   EXPECT_EQ(
505       GetSourceStringToEnd(CharSourceRange(Results3[0].ConditionRange, false)),
506       "1 + \\\n  2");
507 
508   const auto &Results4 = DirectiveExprRange("#if 0\n#elif FLOOFY\n#endif\n");
509   EXPECT_EQ(Results4.size(), 2U);
510   EXPECT_EQ(
511       GetSourceStringToEnd(CharSourceRange(Results4[0].ConditionRange, false)),
512       "0");
513   EXPECT_EQ(
514       GetSourceStringToEnd(CharSourceRange(Results4[1].ConditionRange, false)),
515       "FLOOFY");
516 
517   const auto &Results5 = DirectiveExprRange("#if 1\n#elif FLOOFY\n#endif\n");
518   EXPECT_EQ(Results5.size(), 2U);
519   EXPECT_EQ(
520       GetSourceStringToEnd(CharSourceRange(Results5[0].ConditionRange, false)),
521       "1");
522   EXPECT_EQ(
523       GetSourceStringToEnd(CharSourceRange(Results5[1].ConditionRange, false)),
524       "FLOOFY");
525 
526   const auto &Results6 =
527       DirectiveExprRange("#if defined(FLUZZY_FLOOF)\n#endif\n");
528   EXPECT_EQ(Results6.size(), 1U);
529   EXPECT_EQ(
530       GetSourceStringToEnd(CharSourceRange(Results6[0].ConditionRange, false)),
531       "defined(FLUZZY_FLOOF)");
532 
533   const auto &Results7 =
534       DirectiveExprRange("#if 1\n#elif defined(FLOOFY)\n#endif\n");
535   EXPECT_EQ(Results7.size(), 2U);
536   EXPECT_EQ(
537       GetSourceStringToEnd(CharSourceRange(Results7[0].ConditionRange, false)),
538       "1");
539   EXPECT_EQ(
540       GetSourceStringToEnd(CharSourceRange(Results7[1].ConditionRange, false)),
541       "defined(FLOOFY)");
542 
543   const auto &Results8 =
544       DirectiveExprRange("#define FLOOFY 0\n#if __FILE__ > FLOOFY\n#endif\n");
545   EXPECT_EQ(Results8.size(), 1U);
546   EXPECT_EQ(
547       GetSourceStringToEnd(CharSourceRange(Results8[0].ConditionRange, false)),
548       "__FILE__ > FLOOFY");
549   EXPECT_EQ(
550       Lexer::getSourceText(CharSourceRange(Results8[0].ConditionRange, false),
551                            SourceMgr, LangOpts),
552       "__FILE__ > FLOOFY");
553 }
554 
555 } // namespace
556