1 //===- unittests/Analysis/MacroExpansionContextTest.cpp - -----------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 
9 #include "clang/Analysis/MacroExpansionContext.h"
10 #include "clang/AST/ASTConsumer.h"
11 #include "clang/AST/ASTContext.h"
12 #include "clang/Basic/Diagnostic.h"
13 #include "clang/Basic/DiagnosticOptions.h"
14 #include "clang/Basic/FileManager.h"
15 #include "clang/Basic/LangOptions.h"
16 #include "clang/Basic/SourceManager.h"
17 #include "clang/Basic/TargetInfo.h"
18 #include "clang/Basic/TargetOptions.h"
19 #include "clang/Lex/HeaderSearch.h"
20 #include "clang/Lex/HeaderSearchOptions.h"
21 #include "clang/Lex/Preprocessor.h"
22 #include "clang/Lex/PreprocessorOptions.h"
23 #include "clang/Parse/Parser.h"
24 #include "llvm/ADT/SmallString.h"
25 #include "gtest/gtest.h"
26 
27 // static bool HACK_EnableDebugInUnitTest = (::llvm::DebugFlag = true);
28 
29 namespace clang {
30 namespace analysis {
31 namespace {
32 
33 class MacroExpansionContextTest : public ::testing::Test {
34 protected:
35   MacroExpansionContextTest()
36       : InMemoryFileSystem(new llvm::vfs::InMemoryFileSystem),
37         FileMgr(FileSystemOptions(), InMemoryFileSystem),
38         DiagID(new DiagnosticIDs()), DiagOpts(new DiagnosticOptions()),
39         Diags(DiagID, DiagOpts.get(), new IgnoringDiagConsumer()),
40         SourceMgr(Diags, FileMgr), TargetOpts(new TargetOptions()) {
41     TargetOpts->Triple = "x86_64-pc-linux-unknown";
42     Target = TargetInfo::CreateTargetInfo(Diags, TargetOpts);
43     LangOpts.CPlusPlus20 = 1; // For __VA_OPT__
44   }
45 
46   IntrusiveRefCntPtr<llvm::vfs::InMemoryFileSystem> InMemoryFileSystem;
47   FileManager FileMgr;
48   IntrusiveRefCntPtr<DiagnosticIDs> DiagID;
49   IntrusiveRefCntPtr<DiagnosticOptions> DiagOpts;
50   DiagnosticsEngine Diags;
51   SourceManager SourceMgr;
52   LangOptions LangOpts;
53   std::shared_ptr<TargetOptions> TargetOpts;
54   IntrusiveRefCntPtr<TargetInfo> Target;
55 
56   std::unique_ptr<MacroExpansionContext>
57   getMacroExpansionContextFor(StringRef SourceText) {
58     std::unique_ptr<llvm::MemoryBuffer> Buf =
59         llvm::MemoryBuffer::getMemBuffer(SourceText);
60     SourceMgr.setMainFileID(SourceMgr.createFileID(std::move(Buf)));
61     TrivialModuleLoader ModLoader;
62     HeaderSearch HeaderInfo(std::make_shared<HeaderSearchOptions>(), SourceMgr,
63                             Diags, LangOpts, Target.get());
64     Preprocessor PP(std::make_shared<PreprocessorOptions>(), Diags, LangOpts,
65                     SourceMgr, HeaderInfo, ModLoader,
66                     /*IILookup =*/nullptr,
67                     /*OwnsHeaderSearch =*/false);
68 
69     PP.Initialize(*Target);
70     auto Ctx = std::make_unique<MacroExpansionContext>(LangOpts);
71     Ctx->registerForPreprocessor(PP);
72 
73     // Lex source text.
74     PP.EnterMainSourceFile();
75 
76     while (true) {
77       Token Tok;
78       PP.Lex(Tok);
79       if (Tok.is(tok::eof))
80         break;
81     }
82 
83     // Callbacks have been executed at this point.
84     return Ctx;
85   }
86 
87   /// Returns the expansion location to main file at the given row and column.
88   SourceLocation at(unsigned row, unsigned col) const {
89     SourceLocation Loc =
90         SourceMgr.translateLineCol(SourceMgr.getMainFileID(), row, col);
91     return SourceMgr.getExpansionLoc(Loc);
92   }
93 
94   static std::string dumpExpandedTexts(const MacroExpansionContext &Ctx) {
95     std::string Buf;
96     llvm::raw_string_ostream OS{Buf};
97     Ctx.dumpExpandedTextsToStream(OS);
98     return Buf;
99   }
100 
101   static std::string dumpExpansionRanges(const MacroExpansionContext &Ctx) {
102     std::string Buf;
103     llvm::raw_string_ostream OS{Buf};
104     Ctx.dumpExpansionRangesToStream(OS);
105     return Buf;
106   }
107 };
108 
109 TEST_F(MacroExpansionContextTest, IgnoresPragmas) {
110   // No-crash during lexing.
111   const auto Ctx = getMacroExpansionContextFor(R"code(
112   _Pragma("pack(push, 1)")
113   _Pragma("pack(pop, 1)")
114       )code");
115   // After preprocessing:
116   // #pragma pack(push, 1)
117   // #pragma pack(pop, 1)
118 
119   EXPECT_EQ("\n=============== ExpandedTokens ===============\n",
120             dumpExpandedTexts(*Ctx));
121   EXPECT_EQ("\n=============== ExpansionRanges ===============\n",
122             dumpExpansionRanges(*Ctx));
123 
124   EXPECT_FALSE(Ctx->getExpandedText(at(2, 1)).hasValue());
125   EXPECT_FALSE(Ctx->getOriginalText(at(2, 1)).hasValue());
126 
127   EXPECT_FALSE(Ctx->getExpandedText(at(2, 3)).hasValue());
128   EXPECT_FALSE(Ctx->getOriginalText(at(2, 3)).hasValue());
129 
130   EXPECT_FALSE(Ctx->getExpandedText(at(3, 3)).hasValue());
131   EXPECT_FALSE(Ctx->getOriginalText(at(3, 3)).hasValue());
132 }
133 
134 TEST_F(MacroExpansionContextTest, NoneForNonExpansionLocations) {
135   const auto Ctx = getMacroExpansionContextFor(R"code(
136   #define EMPTY
137   A b cd EMPTY ef EMPTY gh
138 EMPTY zz
139       )code");
140   // After preprocessing:
141   //  A b cd ef gh
142   //      zz
143 
144   // That's the beginning of the definition of EMPTY.
145   EXPECT_FALSE(Ctx->getExpandedText(at(2, 11)).hasValue());
146   EXPECT_FALSE(Ctx->getOriginalText(at(2, 11)).hasValue());
147 
148   // The space before the first expansion of EMPTY.
149   EXPECT_FALSE(Ctx->getExpandedText(at(3, 9)).hasValue());
150   EXPECT_FALSE(Ctx->getOriginalText(at(3, 9)).hasValue());
151 
152   // The beginning of the first expansion of EMPTY.
153   EXPECT_TRUE(Ctx->getExpandedText(at(3, 10)).hasValue());
154   EXPECT_TRUE(Ctx->getOriginalText(at(3, 10)).hasValue());
155 
156   // Pointing inside of the token EMPTY, but not at the beginning.
157   // FIXME: We only deal with begin locations.
158   EXPECT_FALSE(Ctx->getExpandedText(at(3, 11)).hasValue());
159   EXPECT_FALSE(Ctx->getOriginalText(at(3, 11)).hasValue());
160 
161   // Same here.
162   EXPECT_FALSE(Ctx->getExpandedText(at(3, 12)).hasValue());
163   EXPECT_FALSE(Ctx->getOriginalText(at(3, 12)).hasValue());
164 
165   // The beginning of the last expansion of EMPTY.
166   EXPECT_TRUE(Ctx->getExpandedText(at(4, 1)).hasValue());
167   EXPECT_TRUE(Ctx->getOriginalText(at(4, 1)).hasValue());
168 
169   // Same as for the 3:11 case.
170   EXPECT_FALSE(Ctx->getExpandedText(at(4, 2)).hasValue());
171   EXPECT_FALSE(Ctx->getOriginalText(at(4, 2)).hasValue());
172 }
173 
174 TEST_F(MacroExpansionContextTest, EmptyExpansions) {
175   const auto Ctx = getMacroExpansionContextFor(R"code(
176   #define EMPTY
177   A b cd EMPTY ef EMPTY gh
178 EMPTY zz
179       )code");
180   // After preprocessing:
181   //  A b cd ef gh
182   //      zz
183 
184   EXPECT_EQ("", Ctx->getExpandedText(at(3, 10)).getValue());
185   EXPECT_EQ("EMPTY", Ctx->getOriginalText(at(3, 10)).getValue());
186 
187   EXPECT_EQ("", Ctx->getExpandedText(at(3, 19)).getValue());
188   EXPECT_EQ("EMPTY", Ctx->getOriginalText(at(3, 19)).getValue());
189 
190   EXPECT_EQ("", Ctx->getExpandedText(at(4, 1)).getValue());
191   EXPECT_EQ("EMPTY", Ctx->getOriginalText(at(4, 1)).getValue());
192 }
193 
194 TEST_F(MacroExpansionContextTest, TransitiveExpansions) {
195   const auto Ctx = getMacroExpansionContextFor(R"code(
196   #define EMPTY
197   #define WOOF EMPTY ) EMPTY   1
198   A b cd WOOF ef EMPTY gh
199       )code");
200   // After preprocessing:
201   //  A b cd ) 1 ef gh
202 
203   EXPECT_EQ("WOOF", Ctx->getOriginalText(at(4, 10)).getValue());
204 
205   EXPECT_EQ("", Ctx->getExpandedText(at(4, 18)).getValue());
206   EXPECT_EQ("EMPTY", Ctx->getOriginalText(at(4, 18)).getValue());
207 }
208 
209 TEST_F(MacroExpansionContextTest, MacroFunctions) {
210   const auto Ctx = getMacroExpansionContextFor(R"code(
211   #define EMPTY
212   #define WOOF(x) x(EMPTY ) )  ) EMPTY   1
213   A b cd WOOF($$ ef) EMPTY gh
214   WOOF(WOOF)
215   WOOF(WOOF(bar barr))),,),')
216       )code");
217   // After preprocessing:
218   //  A b cd $$ ef( ) ) ) 1 gh
219   //  WOOF( ) ) ) 1
220   //  bar barr( ) ) ) 1( ) ) ) 1),,),')
221 
222   EXPECT_EQ("$$ ef ()))1", Ctx->getExpandedText(at(4, 10)).getValue());
223   EXPECT_EQ("WOOF($$ ef)", Ctx->getOriginalText(at(4, 10)).getValue());
224 
225   EXPECT_EQ("", Ctx->getExpandedText(at(4, 22)).getValue());
226   EXPECT_EQ("EMPTY", Ctx->getOriginalText(at(4, 22)).getValue());
227 
228   EXPECT_EQ("WOOF ()))1", Ctx->getExpandedText(at(5, 3)).getValue());
229   EXPECT_EQ("WOOF(WOOF)", Ctx->getOriginalText(at(5, 3)).getValue());
230 
231   EXPECT_EQ("bar barr ()))1()))1", Ctx->getExpandedText(at(6, 3)).getValue());
232   EXPECT_EQ("WOOF(WOOF(bar barr))", Ctx->getOriginalText(at(6, 3)).getValue());
233 }
234 
235 TEST_F(MacroExpansionContextTest, VariadicMacros) {
236   // From the GCC website.
237   const auto Ctx = getMacroExpansionContextFor(R"code(
238   #define eprintf(format, ...) fprintf (stderr, format, __VA_ARGS__)
239   eprintf("success!\n", );
240   eprintf("success!\n");
241 
242   #define eprintf2(format, ...) \
243     fprintf (stderr, format __VA_OPT__(,) __VA_ARGS__)
244   eprintf2("success!\n", );
245   eprintf2("success!\n");
246       )code");
247   // After preprocessing:
248   //  fprintf (stderr, "success!\n", );
249   //  fprintf (stderr, "success!\n", );
250   //  fprintf (stderr, "success!\n" );
251   //  fprintf (stderr, "success!\n" );
252 
253   EXPECT_EQ(R"(fprintf (stderr ,"success!\n",))",
254             Ctx->getExpandedText(at(3, 3)).getValue());
255   EXPECT_EQ(R"(eprintf("success!\n", ))",
256             Ctx->getOriginalText(at(3, 3)).getValue());
257 
258   EXPECT_EQ(R"(fprintf (stderr ,"success!\n",))",
259             Ctx->getExpandedText(at(4, 3)).getValue());
260   EXPECT_EQ(R"(eprintf("success!\n"))",
261             Ctx->getOriginalText(at(4, 3)).getValue());
262 
263   EXPECT_EQ(R"(fprintf (stderr ,"success!\n"))",
264             Ctx->getExpandedText(at(8, 3)).getValue());
265   EXPECT_EQ(R"(eprintf2("success!\n", ))",
266             Ctx->getOriginalText(at(8, 3)).getValue());
267 
268   EXPECT_EQ(R"(fprintf (stderr ,"success!\n"))",
269             Ctx->getExpandedText(at(9, 3)).getValue());
270   EXPECT_EQ(R"(eprintf2("success!\n"))",
271             Ctx->getOriginalText(at(9, 3)).getValue());
272 }
273 
274 TEST_F(MacroExpansionContextTest, ConcatenationMacros) {
275   // From the GCC website.
276   const auto Ctx = getMacroExpansionContextFor(R"code(
277   #define COMMAND(NAME)  { #NAME, NAME ## _command }
278   struct command commands[] = {
279     COMMAND(quit),
280     COMMAND(help),
281   };)code");
282   // After preprocessing:
283   //  struct command commands[] = {
284   //    { "quit", quit_command },
285   //    { "help", help_command },
286   //  };
287 
288   EXPECT_EQ(R"({"quit",quit_command })",
289             Ctx->getExpandedText(at(4, 5)).getValue());
290   EXPECT_EQ("COMMAND(quit)", Ctx->getOriginalText(at(4, 5)).getValue());
291 
292   EXPECT_EQ(R"({"help",help_command })",
293             Ctx->getExpandedText(at(5, 5)).getValue());
294   EXPECT_EQ("COMMAND(help)", Ctx->getOriginalText(at(5, 5)).getValue());
295 }
296 
297 TEST_F(MacroExpansionContextTest, StringizingMacros) {
298   // From the GCC website.
299   const auto Ctx = getMacroExpansionContextFor(R"code(
300   #define WARN_IF(EXP) \
301   do { if (EXP) \
302           fprintf (stderr, "Warning: " #EXP "\n"); } \
303   while (0)
304   WARN_IF (x == 0);
305 
306   #define xstr(s) str(s)
307   #define str(s) #s
308   #define foo 4
309   str (foo)
310   xstr (foo)
311       )code");
312   // After preprocessing:
313   //  do { if (x == 0) fprintf (stderr, "Warning: " "x == 0" "\n"); } while (0);
314   //  "foo"
315   //  "4"
316 
317   EXPECT_EQ(
318       R"(do {if (x ==0)fprintf (stderr ,"Warning: ""x == 0""\n");}while (0))",
319       Ctx->getExpandedText(at(6, 3)).getValue());
320   EXPECT_EQ("WARN_IF (x == 0)", Ctx->getOriginalText(at(6, 3)).getValue());
321 
322   EXPECT_EQ(R"("foo")", Ctx->getExpandedText(at(11, 3)).getValue());
323   EXPECT_EQ("str (foo)", Ctx->getOriginalText(at(11, 3)).getValue());
324 
325   EXPECT_EQ(R"("4")", Ctx->getExpandedText(at(12, 3)).getValue());
326   EXPECT_EQ("xstr (foo)", Ctx->getOriginalText(at(12, 3)).getValue());
327 }
328 
329 TEST_F(MacroExpansionContextTest, StringizingVariadicMacros) {
330   const auto Ctx = getMacroExpansionContextFor(R"code(
331   #define xstr(...) str(__VA_ARGS__)
332   #define str(...) #__VA_ARGS__
333   #define RParen2x ) )
334   #define EMPTY
335   #define f(x, ...) __VA_ARGS__ ! x * x
336   #define g(...) zz EMPTY f(__VA_ARGS__ ! x) f() * y
337   #define h(x, G) G(x) G(x ## x RParen2x
338   #define q(G) h(apple, G(apple)) RParen2x
339 
340   q(g)
341   q(xstr)
342   g(RParen2x)
343   f( RParen2x )s
344       )code");
345   // clang-format off
346   // After preprocessing:
347   //  zz ! apple ! x * apple ! x ! * * y(apple) zz ! apple ! x * apple ! x ! * * y(appleapple ) ) ) )
348   //  "apple"(apple) "apple"(appleapple ) ) ) )
349   //  zz ! * ) ! x) ! * * y
350   //  ! ) ) * ) )
351   // clang-format on
352 
353   EXPECT_EQ("zz !apple !x *apple !x !**y (apple )zz !apple !x *apple !x !**y "
354             "(appleapple ))))",
355             Ctx->getExpandedText(at(11, 3)).getValue());
356   EXPECT_EQ("q(g)", Ctx->getOriginalText(at(11, 3)).getValue());
357 
358   EXPECT_EQ(R"res("apple"(apple )"apple"(appleapple )))))res",
359             Ctx->getExpandedText(at(12, 3)).getValue());
360   EXPECT_EQ("q(xstr)", Ctx->getOriginalText(at(12, 3)).getValue());
361 
362   EXPECT_EQ("zz !*)!x )!**y ", Ctx->getExpandedText(at(13, 3)).getValue());
363   EXPECT_EQ("g(RParen2x)", Ctx->getOriginalText(at(13, 3)).getValue());
364 
365   EXPECT_EQ("!))*))", Ctx->getExpandedText(at(14, 3)).getValue());
366   EXPECT_EQ("f( RParen2x )", Ctx->getOriginalText(at(14, 3)).getValue());
367 }
368 
369 TEST_F(MacroExpansionContextTest, RedefUndef) {
370   const auto Ctx = getMacroExpansionContextFor(R"code(
371   #define Hi(x) Welcome x
372   Hi(Adam)
373   #define Hi Willkommen
374   Hi Hans
375   #undef Hi
376   Hi(Hi)
377       )code");
378   // After preprocessing:
379   //  Welcome Adam
380   //  Willkommen Hans
381   //  Hi(Hi)
382 
383   // FIXME: Extra space follows every identifier.
384   EXPECT_EQ("Welcome Adam ", Ctx->getExpandedText(at(3, 3)).getValue());
385   EXPECT_EQ("Hi(Adam)", Ctx->getOriginalText(at(3, 3)).getValue());
386 
387   EXPECT_EQ("Willkommen ", Ctx->getExpandedText(at(5, 3)).getValue());
388   EXPECT_EQ("Hi", Ctx->getOriginalText(at(5, 3)).getValue());
389 
390   // There was no macro expansion at 7:3, we should expect None.
391   EXPECT_FALSE(Ctx->getExpandedText(at(7, 3)).hasValue());
392   EXPECT_FALSE(Ctx->getOriginalText(at(7, 3)).hasValue());
393 }
394 
395 TEST_F(MacroExpansionContextTest, UnbalacedParenthesis) {
396   const auto Ctx = getMacroExpansionContextFor(R"code(
397   #define retArg(x) x
398   #define retArgUnclosed retArg(fun()
399   #define BB CC
400   #define applyInt BB(int)
401   #define CC(x) retArgUnclosed
402 
403   applyInt );
404 
405   #define expandArgUnclosedCommaExpr(x) (x, fun(), 1
406   #define f expandArgUnclosedCommaExpr
407 
408   int x =  f(f(1))  ));
409       )code");
410   // After preprocessing:
411   //  fun();
412   //  int x = ((1, fun(), 1, fun(), 1 ));
413 
414   EXPECT_EQ("fun ()", Ctx->getExpandedText(at(8, 3)).getValue());
415   EXPECT_EQ("applyInt )", Ctx->getOriginalText(at(8, 3)).getValue());
416 
417   EXPECT_EQ("((1,fun (),1,fun (),1",
418             Ctx->getExpandedText(at(13, 12)).getValue());
419   EXPECT_EQ("f(f(1))", Ctx->getOriginalText(at(13, 12)).getValue());
420 }
421 
422 } // namespace
423 } // namespace analysis
424 } // namespace clang
425