1 //===- unittests/Lex/PPCallbacksTest.cpp - PPCallbacks tests ------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===--------------------------------------------------------------===//
8 
9 #include "clang/Lex/Preprocessor.h"
10 #include "clang/AST/ASTConsumer.h"
11 #include "clang/AST/ASTContext.h"
12 #include "clang/Basic/Diagnostic.h"
13 #include "clang/Basic/DiagnosticOptions.h"
14 #include "clang/Basic/FileManager.h"
15 #include "clang/Basic/LangOptions.h"
16 #include "clang/Basic/SourceManager.h"
17 #include "clang/Basic/TargetInfo.h"
18 #include "clang/Basic/TargetOptions.h"
19 #include "clang/Lex/HeaderSearch.h"
20 #include "clang/Lex/HeaderSearchOptions.h"
21 #include "clang/Lex/ModuleLoader.h"
22 #include "clang/Lex/PreprocessorOptions.h"
23 #include "clang/Parse/Parser.h"
24 #include "clang/Sema/Sema.h"
25 #include "llvm/ADT/SmallString.h"
26 #include "llvm/Support/Path.h"
27 #include "gtest/gtest.h"
28 
29 using namespace clang;
30 
31 namespace {
32 
33 // Stub to collect data from InclusionDirective callbacks.
34 class InclusionDirectiveCallbacks : public PPCallbacks {
35 public:
36   void InclusionDirective(SourceLocation HashLoc, const Token &IncludeTok,
37                           StringRef FileName, bool IsAngled,
38                           CharSourceRange FilenameRange, const FileEntry *File,
39                           StringRef SearchPath, StringRef RelativePath,
40                           const Module *Imported,
41                           SrcMgr::CharacteristicKind FileType) override {
42     this->HashLoc = HashLoc;
43     this->IncludeTok = IncludeTok;
44     this->FileName = FileName.str();
45     this->IsAngled = IsAngled;
46     this->FilenameRange = FilenameRange;
47     this->File = File;
48     this->SearchPath = SearchPath.str();
49     this->RelativePath = RelativePath.str();
50     this->Imported = Imported;
51     this->FileType = FileType;
52   }
53 
54   SourceLocation HashLoc;
55   Token IncludeTok;
56   SmallString<16> FileName;
57   bool IsAngled;
58   CharSourceRange FilenameRange;
59   const FileEntry* File;
60   SmallString<16> SearchPath;
61   SmallString<16> RelativePath;
62   const Module* Imported;
63   SrcMgr::CharacteristicKind FileType;
64 };
65 
66 class CondDirectiveCallbacks : public PPCallbacks {
67 public:
68   struct Result {
69     SourceRange ConditionRange;
70     ConditionValueKind ConditionValue;
71 
72     Result(SourceRange R, ConditionValueKind K)
73         : ConditionRange(R), ConditionValue(K) {}
74   };
75 
76   std::vector<Result> Results;
77 
78   void If(SourceLocation Loc, SourceRange ConditionRange,
79           ConditionValueKind ConditionValue) override {
80     Results.emplace_back(ConditionRange, ConditionValue);
81   }
82 
83   void Elif(SourceLocation Loc, SourceRange ConditionRange,
84             ConditionValueKind ConditionValue, SourceLocation IfLoc) override {
85     Results.emplace_back(ConditionRange, ConditionValue);
86   }
87 };
88 
89 // Stub to collect data from PragmaOpenCLExtension callbacks.
90 class PragmaOpenCLExtensionCallbacks : public PPCallbacks {
91 public:
92   typedef struct {
93     SmallString<16> Name;
94     unsigned State;
95   } CallbackParameters;
96 
97   PragmaOpenCLExtensionCallbacks() : Name("Not called."), State(99) {}
98 
99   void PragmaOpenCLExtension(clang::SourceLocation NameLoc,
100                              const clang::IdentifierInfo *Name,
101                              clang::SourceLocation StateLoc,
102                              unsigned State) override {
103       this->NameLoc = NameLoc;
104       this->Name = Name->getName();
105       this->StateLoc = StateLoc;
106       this->State = State;
107   }
108 
109   SourceLocation NameLoc;
110   SmallString<16> Name;
111   SourceLocation StateLoc;
112   unsigned State;
113 };
114 
115 // PPCallbacks test fixture.
116 class PPCallbacksTest : public ::testing::Test {
117 protected:
118   PPCallbacksTest()
119       : InMemoryFileSystem(new llvm::vfs::InMemoryFileSystem),
120         FileMgr(FileSystemOptions(), InMemoryFileSystem),
121         DiagID(new DiagnosticIDs()), DiagOpts(new DiagnosticOptions()),
122         Diags(DiagID, DiagOpts.get(), new IgnoringDiagConsumer()),
123         SourceMgr(Diags, FileMgr), TargetOpts(new TargetOptions()) {
124     TargetOpts->Triple = "x86_64-apple-darwin11.1.0";
125     Target = TargetInfo::CreateTargetInfo(Diags, TargetOpts);
126   }
127 
128   IntrusiveRefCntPtr<llvm::vfs::InMemoryFileSystem> InMemoryFileSystem;
129   FileManager FileMgr;
130   IntrusiveRefCntPtr<DiagnosticIDs> DiagID;
131   IntrusiveRefCntPtr<DiagnosticOptions> DiagOpts;
132   DiagnosticsEngine Diags;
133   SourceManager SourceMgr;
134   LangOptions LangOpts;
135   std::shared_ptr<TargetOptions> TargetOpts;
136   IntrusiveRefCntPtr<TargetInfo> Target;
137 
138   // Register a header path as a known file and add its location
139   // to search path.
140   void AddFakeHeader(HeaderSearch &HeaderInfo, const char *HeaderPath,
141                      bool IsSystemHeader) {
142     // Tell FileMgr about header.
143     InMemoryFileSystem->addFile(HeaderPath, 0,
144                                 llvm::MemoryBuffer::getMemBuffer("\n"));
145 
146     // Add header's parent path to search path.
147     StringRef SearchPath = llvm::sys::path::parent_path(HeaderPath);
148     const DirectoryEntry *DE = FileMgr.getDirectory(SearchPath);
149     DirectoryLookup DL(DE, SrcMgr::C_User, false);
150     HeaderInfo.AddSearchPath(DL, IsSystemHeader);
151   }
152 
153   // Get the raw source string of the range.
154   StringRef GetSourceString(CharSourceRange Range) {
155     const char* B = SourceMgr.getCharacterData(Range.getBegin());
156     const char* E = SourceMgr.getCharacterData(Range.getEnd());
157 
158     return StringRef(B, E - B);
159   }
160 
161   StringRef GetSourceStringToEnd(CharSourceRange Range) {
162     const char *B = SourceMgr.getCharacterData(Range.getBegin());
163     const char *E = SourceMgr.getCharacterData(Range.getEnd());
164 
165     return StringRef(
166         B,
167         E - B + Lexer::MeasureTokenLength(Range.getEnd(), SourceMgr, LangOpts));
168   }
169 
170   // Run lexer over SourceText and collect FilenameRange from
171   // the InclusionDirective callback.
172   CharSourceRange InclusionDirectiveFilenameRange(const char *SourceText,
173                                                   const char *HeaderPath,
174                                                   bool SystemHeader) {
175     std::unique_ptr<llvm::MemoryBuffer> Buf =
176         llvm::MemoryBuffer::getMemBuffer(SourceText);
177     SourceMgr.setMainFileID(SourceMgr.createFileID(std::move(Buf)));
178 
179     TrivialModuleLoader ModLoader;
180 
181     HeaderSearch HeaderInfo(std::make_shared<HeaderSearchOptions>(), SourceMgr,
182                             Diags, LangOpts, Target.get());
183     AddFakeHeader(HeaderInfo, HeaderPath, SystemHeader);
184 
185     Preprocessor PP(std::make_shared<PreprocessorOptions>(), Diags, LangOpts,
186                     SourceMgr, HeaderInfo, ModLoader,
187                     /*IILookup =*/nullptr,
188                     /*OwnsHeaderSearch =*/false);
189     return InclusionDirectiveCallback(PP)->FilenameRange;
190   }
191 
192   SrcMgr::CharacteristicKind InclusionDirectiveCharacteristicKind(
193       const char *SourceText, const char *HeaderPath, bool SystemHeader) {
194     std::unique_ptr<llvm::MemoryBuffer> Buf =
195         llvm::MemoryBuffer::getMemBuffer(SourceText);
196     SourceMgr.setMainFileID(SourceMgr.createFileID(std::move(Buf)));
197 
198     TrivialModuleLoader ModLoader;
199 
200     HeaderSearch HeaderInfo(std::make_shared<HeaderSearchOptions>(), SourceMgr,
201                             Diags, LangOpts, Target.get());
202     AddFakeHeader(HeaderInfo, HeaderPath, SystemHeader);
203 
204     Preprocessor PP(std::make_shared<PreprocessorOptions>(), Diags, LangOpts,
205                     SourceMgr, HeaderInfo, ModLoader,
206                     /*IILookup =*/nullptr,
207                     /*OwnsHeaderSearch =*/false);
208     return InclusionDirectiveCallback(PP)->FileType;
209   }
210 
211   InclusionDirectiveCallbacks *InclusionDirectiveCallback(Preprocessor &PP) {
212     PP.Initialize(*Target);
213     InclusionDirectiveCallbacks* Callbacks = new InclusionDirectiveCallbacks;
214     PP.addPPCallbacks(std::unique_ptr<PPCallbacks>(Callbacks));
215 
216     // Lex source text.
217     PP.EnterMainSourceFile();
218 
219     while (true) {
220       Token Tok;
221       PP.Lex(Tok);
222       if (Tok.is(tok::eof))
223         break;
224     }
225 
226     // Callbacks have been executed at this point -- return filename range.
227     return Callbacks;
228   }
229 
230   std::vector<CondDirectiveCallbacks::Result>
231   DirectiveExprRange(StringRef SourceText) {
232     TrivialModuleLoader ModLoader;
233     std::unique_ptr<llvm::MemoryBuffer> Buf =
234         llvm::MemoryBuffer::getMemBuffer(SourceText);
235     SourceMgr.setMainFileID(SourceMgr.createFileID(std::move(Buf)));
236     HeaderSearch HeaderInfo(std::make_shared<HeaderSearchOptions>(), SourceMgr,
237                             Diags, LangOpts, Target.get());
238     Preprocessor PP(std::make_shared<PreprocessorOptions>(), Diags, LangOpts,
239                     SourceMgr, HeaderInfo, ModLoader,
240                     /*IILookup =*/nullptr,
241                     /*OwnsHeaderSearch =*/false);
242     PP.Initialize(*Target);
243     auto *Callbacks = new CondDirectiveCallbacks;
244     PP.addPPCallbacks(std::unique_ptr<PPCallbacks>(Callbacks));
245 
246     // Lex source text.
247     PP.EnterMainSourceFile();
248 
249     while (true) {
250       Token Tok;
251       PP.Lex(Tok);
252       if (Tok.is(tok::eof))
253         break;
254     }
255 
256     return Callbacks->Results;
257   }
258 
259   PragmaOpenCLExtensionCallbacks::CallbackParameters
260   PragmaOpenCLExtensionCall(const char *SourceText) {
261     LangOptions OpenCLLangOpts;
262     OpenCLLangOpts.OpenCL = 1;
263 
264     std::unique_ptr<llvm::MemoryBuffer> SourceBuf =
265         llvm::MemoryBuffer::getMemBuffer(SourceText, "test.cl");
266     SourceMgr.setMainFileID(SourceMgr.createFileID(std::move(SourceBuf)));
267 
268     TrivialModuleLoader ModLoader;
269     HeaderSearch HeaderInfo(std::make_shared<HeaderSearchOptions>(), SourceMgr,
270                             Diags, OpenCLLangOpts, Target.get());
271 
272     Preprocessor PP(std::make_shared<PreprocessorOptions>(), Diags,
273                     OpenCLLangOpts, SourceMgr, HeaderInfo, ModLoader,
274                     /*IILookup =*/nullptr,
275                     /*OwnsHeaderSearch =*/false);
276     PP.Initialize(*Target);
277 
278     // parser actually sets correct pragma handlers for preprocessor
279     // according to LangOptions, so we init Parser to register opencl
280     // pragma handlers
281     ASTContext Context(OpenCLLangOpts, SourceMgr, PP.getIdentifierTable(),
282                        PP.getSelectorTable(), PP.getBuiltinInfo());
283     Context.InitBuiltinTypes(*Target);
284 
285     ASTConsumer Consumer;
286     Sema S(PP, Context, Consumer);
287     Parser P(PP, S, false);
288     PragmaOpenCLExtensionCallbacks* Callbacks = new PragmaOpenCLExtensionCallbacks;
289     PP.addPPCallbacks(std::unique_ptr<PPCallbacks>(Callbacks));
290 
291     // Lex source text.
292     PP.EnterMainSourceFile();
293     while (true) {
294       Token Tok;
295       PP.Lex(Tok);
296       if (Tok.is(tok::eof))
297         break;
298     }
299 
300     PragmaOpenCLExtensionCallbacks::CallbackParameters RetVal = {
301       Callbacks->Name,
302       Callbacks->State
303     };
304     return RetVal;
305   }
306 };
307 
308 TEST_F(PPCallbacksTest, UserFileCharacteristics) {
309   const char *Source = "#include \"quoted.h\"\n";
310 
311   SrcMgr::CharacteristicKind Kind =
312       InclusionDirectiveCharacteristicKind(Source, "/quoted.h", false);
313 
314   ASSERT_EQ(SrcMgr::CharacteristicKind::C_User, Kind);
315 }
316 
317 TEST_F(PPCallbacksTest, QuotedFilename) {
318   const char* Source =
319     "#include \"quoted.h\"\n";
320 
321   CharSourceRange Range =
322     InclusionDirectiveFilenameRange(Source, "/quoted.h", false);
323 
324   ASSERT_EQ("\"quoted.h\"", GetSourceString(Range));
325 }
326 
327 TEST_F(PPCallbacksTest, AngledFilename) {
328   const char* Source =
329     "#include <angled.h>\n";
330 
331   CharSourceRange Range =
332     InclusionDirectiveFilenameRange(Source, "/angled.h", true);
333 
334   ASSERT_EQ("<angled.h>", GetSourceString(Range));
335 }
336 
337 TEST_F(PPCallbacksTest, QuotedInMacro) {
338   const char* Source =
339     "#define MACRO_QUOTED \"quoted.h\"\n"
340     "#include MACRO_QUOTED\n";
341 
342   CharSourceRange Range =
343     InclusionDirectiveFilenameRange(Source, "/quoted.h", false);
344 
345   ASSERT_EQ("\"quoted.h\"", GetSourceString(Range));
346 }
347 
348 TEST_F(PPCallbacksTest, AngledInMacro) {
349   const char* Source =
350     "#define MACRO_ANGLED <angled.h>\n"
351     "#include MACRO_ANGLED\n";
352 
353   CharSourceRange Range =
354     InclusionDirectiveFilenameRange(Source, "/angled.h", true);
355 
356   ASSERT_EQ("<angled.h>", GetSourceString(Range));
357 }
358 
359 TEST_F(PPCallbacksTest, StringizedMacroArgument) {
360   const char* Source =
361     "#define MACRO_STRINGIZED(x) #x\n"
362     "#include MACRO_STRINGIZED(quoted.h)\n";
363 
364   CharSourceRange Range =
365     InclusionDirectiveFilenameRange(Source, "/quoted.h", false);
366 
367   ASSERT_EQ("\"quoted.h\"", GetSourceString(Range));
368 }
369 
370 TEST_F(PPCallbacksTest, ConcatenatedMacroArgument) {
371   const char* Source =
372     "#define MACRO_ANGLED <angled.h>\n"
373     "#define MACRO_CONCAT(x, y) x ## _ ## y\n"
374     "#include MACRO_CONCAT(MACRO, ANGLED)\n";
375 
376   CharSourceRange Range =
377     InclusionDirectiveFilenameRange(Source, "/angled.h", false);
378 
379   ASSERT_EQ("<angled.h>", GetSourceString(Range));
380 }
381 
382 TEST_F(PPCallbacksTest, TrigraphFilename) {
383   const char* Source =
384     "#include \"tri\?\?-graph.h\"\n";
385 
386   CharSourceRange Range =
387     InclusionDirectiveFilenameRange(Source, "/tri~graph.h", false);
388 
389   ASSERT_EQ("\"tri\?\?-graph.h\"", GetSourceString(Range));
390 }
391 
392 TEST_F(PPCallbacksTest, TrigraphInMacro) {
393   const char* Source =
394     "#define MACRO_TRIGRAPH \"tri\?\?-graph.h\"\n"
395     "#include MACRO_TRIGRAPH\n";
396 
397   CharSourceRange Range =
398     InclusionDirectiveFilenameRange(Source, "/tri~graph.h", false);
399 
400   ASSERT_EQ("\"tri\?\?-graph.h\"", GetSourceString(Range));
401 }
402 
403 TEST_F(PPCallbacksTest, OpenCLExtensionPragmaEnabled) {
404   const char* Source =
405     "#pragma OPENCL EXTENSION cl_khr_fp64 : enable\n";
406 
407   PragmaOpenCLExtensionCallbacks::CallbackParameters Parameters =
408     PragmaOpenCLExtensionCall(Source);
409 
410   ASSERT_EQ("cl_khr_fp64", Parameters.Name);
411   unsigned ExpectedState = 1;
412   ASSERT_EQ(ExpectedState, Parameters.State);
413 }
414 
415 TEST_F(PPCallbacksTest, OpenCLExtensionPragmaDisabled) {
416   const char* Source =
417     "#pragma OPENCL EXTENSION cl_khr_fp16 : disable\n";
418 
419   PragmaOpenCLExtensionCallbacks::CallbackParameters Parameters =
420     PragmaOpenCLExtensionCall(Source);
421 
422   ASSERT_EQ("cl_khr_fp16", Parameters.Name);
423   unsigned ExpectedState = 0;
424   ASSERT_EQ(ExpectedState, Parameters.State);
425 }
426 
427 TEST_F(PPCallbacksTest, DirectiveExprRanges) {
428   const auto &Results1 = DirectiveExprRange("#if FLUZZY_FLOOF\n#endif\n");
429   EXPECT_EQ(Results1.size(), 1U);
430   EXPECT_EQ(
431       GetSourceStringToEnd(CharSourceRange(Results1[0].ConditionRange, false)),
432       "FLUZZY_FLOOF");
433 
434   const auto &Results2 = DirectiveExprRange("#if 1 + 4 < 7\n#endif\n");
435   EXPECT_EQ(Results2.size(), 1U);
436   EXPECT_EQ(
437       GetSourceStringToEnd(CharSourceRange(Results2[0].ConditionRange, false)),
438       "1 + 4 < 7");
439 
440   const auto &Results3 = DirectiveExprRange("#if 1 + \\\n  2\n#endif\n");
441   EXPECT_EQ(Results3.size(), 1U);
442   EXPECT_EQ(
443       GetSourceStringToEnd(CharSourceRange(Results3[0].ConditionRange, false)),
444       "1 + \\\n  2");
445 
446   const auto &Results4 = DirectiveExprRange("#if 0\n#elif FLOOFY\n#endif\n");
447   EXPECT_EQ(Results4.size(), 2U);
448   EXPECT_EQ(
449       GetSourceStringToEnd(CharSourceRange(Results4[0].ConditionRange, false)),
450       "0");
451   EXPECT_EQ(
452       GetSourceStringToEnd(CharSourceRange(Results4[1].ConditionRange, false)),
453       "FLOOFY");
454 
455   const auto &Results5 = DirectiveExprRange("#if 1\n#elif FLOOFY\n#endif\n");
456   EXPECT_EQ(Results5.size(), 2U);
457   EXPECT_EQ(
458       GetSourceStringToEnd(CharSourceRange(Results5[0].ConditionRange, false)),
459       "1");
460   EXPECT_EQ(
461       GetSourceStringToEnd(CharSourceRange(Results5[1].ConditionRange, false)),
462       "FLOOFY");
463 
464   const auto &Results6 =
465       DirectiveExprRange("#if defined(FLUZZY_FLOOF)\n#endif\n");
466   EXPECT_EQ(Results6.size(), 1U);
467   EXPECT_EQ(
468       GetSourceStringToEnd(CharSourceRange(Results6[0].ConditionRange, false)),
469       "defined(FLUZZY_FLOOF)");
470 
471   const auto &Results7 =
472       DirectiveExprRange("#if 1\n#elif defined(FLOOFY)\n#endif\n");
473   EXPECT_EQ(Results7.size(), 2U);
474   EXPECT_EQ(
475       GetSourceStringToEnd(CharSourceRange(Results7[0].ConditionRange, false)),
476       "1");
477   EXPECT_EQ(
478       GetSourceStringToEnd(CharSourceRange(Results7[1].ConditionRange, false)),
479       "defined(FLOOFY)");
480 
481   const auto &Results8 =
482       DirectiveExprRange("#define FLOOFY 0\n#if __FILE__ > FLOOFY\n#endif\n");
483   EXPECT_EQ(Results8.size(), 1U);
484   EXPECT_EQ(
485       GetSourceStringToEnd(CharSourceRange(Results8[0].ConditionRange, false)),
486       "__FILE__ > FLOOFY");
487   EXPECT_EQ(
488       Lexer::getSourceText(CharSourceRange(Results8[0].ConditionRange, false),
489                            SourceMgr, LangOpts),
490       "__FILE__ > FLOOFY");
491 }
492 
493 } // namespace
494