1 //===- unittests/Basic/SarifTest.cpp - Test writing SARIF documents -------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 
9 #include "clang/Basic/Sarif.h"
10 #include "clang/Basic/DiagnosticIDs.h"
11 #include "clang/Basic/DiagnosticOptions.h"
12 #include "clang/Basic/FileManager.h"
13 #include "clang/Basic/FileSystemOptions.h"
14 #include "clang/Basic/LangOptions.h"
15 #include "clang/Basic/SourceLocation.h"
16 #include "clang/Basic/SourceManager.h"
17 #include "llvm/ADT/StringRef.h"
18 #include "llvm/Support/FormatVariadic.h"
19 #include "llvm/Support/JSON.h"
20 #include "llvm/Support/MemoryBuffer.h"
21 #include "llvm/Support/VirtualFileSystem.h"
22 #include "llvm/Support/raw_ostream.h"
23 #include "gmock/gmock.h"
24 #include "gtest/gtest.h"
25 
26 #include <algorithm>
27 
28 using namespace clang;
29 
30 namespace {
31 
32 using LineCol = std::pair<unsigned int, unsigned int>;
33 
serializeSarifDocument(llvm::json::Object && Doc)34 static std::string serializeSarifDocument(llvm::json::Object &&Doc) {
35   std::string Output;
36   llvm::json::Value value(std::move(Doc));
37   llvm::raw_string_ostream OS{Output};
38   OS << llvm::formatv("{0}", value);
39   OS.flush();
40   return Output;
41 }
42 
43 class SarifDocumentWriterTest : public ::testing::Test {
44 protected:
SarifDocumentWriterTest()45   SarifDocumentWriterTest()
46       : InMemoryFileSystem(new llvm::vfs::InMemoryFileSystem),
47         FileMgr(FileSystemOptions(), InMemoryFileSystem),
48         DiagID(new DiagnosticIDs()), DiagOpts(new DiagnosticOptions()),
49         Diags(DiagID, DiagOpts.get(), new IgnoringDiagConsumer()),
50         SourceMgr(Diags, FileMgr) {}
51 
52   IntrusiveRefCntPtr<llvm::vfs::InMemoryFileSystem> InMemoryFileSystem;
53   FileManager FileMgr;
54   IntrusiveRefCntPtr<DiagnosticIDs> DiagID;
55   IntrusiveRefCntPtr<DiagnosticOptions> DiagOpts;
56   DiagnosticsEngine Diags;
57   SourceManager SourceMgr;
58   LangOptions LangOpts;
59 
registerSource(llvm::StringRef Name,const char * SourceText,bool IsMainFile=false)60   FileID registerSource(llvm::StringRef Name, const char *SourceText,
61                         bool IsMainFile = false) {
62     std::unique_ptr<llvm::MemoryBuffer> SourceBuf =
63         llvm::MemoryBuffer::getMemBuffer(SourceText);
64     const FileEntry *SourceFile =
65         FileMgr.getVirtualFile(Name, SourceBuf->getBufferSize(), 0);
66     SourceMgr.overrideFileContents(SourceFile, std::move(SourceBuf));
67     FileID FID = SourceMgr.getOrCreateFileID(SourceFile, SrcMgr::C_User);
68     if (IsMainFile)
69       SourceMgr.setMainFileID(FID);
70     return FID;
71   }
72 
getFakeCharSourceRange(FileID FID,LineCol Begin,LineCol End)73   CharSourceRange getFakeCharSourceRange(FileID FID, LineCol Begin,
74                                          LineCol End) {
75     auto BeginLoc = SourceMgr.translateLineCol(FID, Begin.first, Begin.second);
76     auto EndLoc = SourceMgr.translateLineCol(FID, End.first, End.second);
77     return CharSourceRange{SourceRange{BeginLoc, EndLoc}, /* ITR = */ false};
78   }
79 };
80 
TEST_F(SarifDocumentWriterTest,canCreateEmptyDocument)81 TEST_F(SarifDocumentWriterTest, canCreateEmptyDocument) {
82   // GIVEN:
83   SarifDocumentWriter Writer{SourceMgr};
84 
85   // WHEN:
86   const llvm::json::Object &EmptyDoc = Writer.createDocument();
87   std::vector<StringRef> Keys(EmptyDoc.size());
88   std::transform(EmptyDoc.begin(), EmptyDoc.end(), Keys.begin(),
89                  [](auto item) { return item.getFirst(); });
90 
91   // THEN:
92   ASSERT_THAT(Keys, testing::UnorderedElementsAre("$schema", "version"));
93 }
94 
95 // Test that a newly inserted run will associate correct tool names
TEST_F(SarifDocumentWriterTest,canCreateDocumentWithOneRun)96 TEST_F(SarifDocumentWriterTest, canCreateDocumentWithOneRun) {
97   // GIVEN:
98   SarifDocumentWriter Writer{SourceMgr};
99   const char *ShortName = "sariftest";
100   const char *LongName = "sarif writer test";
101 
102   // WHEN:
103   Writer.createRun(ShortName, LongName);
104   Writer.endRun();
105   const llvm::json::Object &Doc = Writer.createDocument();
106   const llvm::json::Array *Runs = Doc.getArray("runs");
107 
108   // THEN:
109   // A run was created
110   ASSERT_THAT(Runs, testing::NotNull());
111 
112   // It is the only run
113   ASSERT_EQ(Runs->size(), 1UL);
114 
115   // The tool associated with the run was the tool
116   const llvm::json::Object *driver =
117       Runs->begin()->getAsObject()->getObject("tool")->getObject("driver");
118   ASSERT_THAT(driver, testing::NotNull());
119 
120   ASSERT_TRUE(driver->getString("name").has_value());
121   ASSERT_TRUE(driver->getString("fullName").has_value());
122   ASSERT_TRUE(driver->getString("language").has_value());
123 
124   EXPECT_EQ(driver->getString("name").value(), ShortName);
125   EXPECT_EQ(driver->getString("fullName").value(), LongName);
126   EXPECT_EQ(driver->getString("language").value(), "en-US");
127 }
128 
TEST_F(SarifDocumentWriterTest,addingResultsWillCrashIfThereIsNoRun)129 TEST_F(SarifDocumentWriterTest, addingResultsWillCrashIfThereIsNoRun) {
130 #if defined(NDEBUG) || !GTEST_HAS_DEATH_TEST
131   GTEST_SKIP() << "This death test is only available for debug builds.";
132 #endif
133   // GIVEN:
134   SarifDocumentWriter Writer{SourceMgr};
135 
136   // WHEN:
137   //  A SarifDocumentWriter::createRun(...) was not called prior to
138   //  SarifDocumentWriter::appendResult(...)
139   // But a rule exists
140   auto RuleIdx = Writer.createRule(SarifRule::create());
141   const SarifResult &EmptyResult = SarifResult::create(RuleIdx);
142 
143   // THEN:
144   auto Matcher = ::testing::AnyOf(
145       ::testing::HasSubstr("create a run first"),
146       ::testing::HasSubstr("no runs associated with the document"));
147   ASSERT_DEATH(Writer.appendResult(EmptyResult), Matcher);
148 }
149 
150 // Test adding rule and result shows up in the final document
TEST_F(SarifDocumentWriterTest,addingResultWithValidRuleAndRunIsOk)151 TEST_F(SarifDocumentWriterTest, addingResultWithValidRuleAndRunIsOk) {
152   // GIVEN:
153   SarifDocumentWriter Writer{SourceMgr};
154   const SarifRule &Rule =
155       SarifRule::create()
156           .setRuleId("clang.unittest")
157           .setDescription("Example rule created during unit tests")
158           .setName("clang unit test");
159 
160   // WHEN:
161   Writer.createRun("sarif test", "sarif test runner");
162   unsigned RuleIdx = Writer.createRule(Rule);
163   const SarifResult &result = SarifResult::create(RuleIdx);
164 
165   Writer.appendResult(result);
166   const llvm::json::Object &Doc = Writer.createDocument();
167 
168   // THEN:
169   // A document with a valid schema and version exists
170   ASSERT_THAT(Doc.get("$schema"), ::testing::NotNull());
171   ASSERT_THAT(Doc.get("version"), ::testing::NotNull());
172   const llvm::json::Array *Runs = Doc.getArray("runs");
173 
174   // A run exists on this document
175   ASSERT_THAT(Runs, ::testing::NotNull());
176   ASSERT_EQ(Runs->size(), 1UL);
177   const llvm::json::Object *TheRun = Runs->back().getAsObject();
178 
179   // The run has slots for tools, results, rules and artifacts
180   ASSERT_THAT(TheRun->get("tool"), ::testing::NotNull());
181   ASSERT_THAT(TheRun->get("results"), ::testing::NotNull());
182   ASSERT_THAT(TheRun->get("artifacts"), ::testing::NotNull());
183   const llvm::json::Object *Driver =
184       TheRun->getObject("tool")->getObject("driver");
185   const llvm::json::Array *Results = TheRun->getArray("results");
186   const llvm::json::Array *Artifacts = TheRun->getArray("artifacts");
187 
188   // The tool is as expected
189   ASSERT_TRUE(Driver->getString("name").has_value());
190   ASSERT_TRUE(Driver->getString("fullName").has_value());
191 
192   EXPECT_EQ(Driver->getString("name").value(), "sarif test");
193   EXPECT_EQ(Driver->getString("fullName").value(), "sarif test runner");
194 
195   // The results are as expected
196   EXPECT_EQ(Results->size(), 1UL);
197 
198   // The artifacts are as expected
199   EXPECT_TRUE(Artifacts->empty());
200 }
201 
TEST_F(SarifDocumentWriterTest,checkSerializingResults)202 TEST_F(SarifDocumentWriterTest, checkSerializingResults) {
203   // GIVEN:
204   const std::string ExpectedOutput =
205       R"({"$schema":"https://docs.oasis-open.org/sarif/sarif/v2.1.0/cos02/schemas/sarif-schema-2.1.0.json","runs":[{"artifacts":[],"columnKind":"unicodeCodePoints","results":[{"message":{"text":""},"ruleId":"clang.unittest","ruleIndex":0}],"tool":{"driver":{"fullName":"sarif test runner","informationUri":"https://clang.llvm.org/docs/UsersManual.html","language":"en-US","name":"sarif test","rules":[{"fullDescription":{"text":"Example rule created during unit tests"},"id":"clang.unittest","name":"clang unit test"}],"version":"1.0.0"}}}],"version":"2.1.0"})";
206 
207   SarifDocumentWriter Writer{SourceMgr};
208   const SarifRule &Rule =
209       SarifRule::create()
210           .setRuleId("clang.unittest")
211           .setDescription("Example rule created during unit tests")
212           .setName("clang unit test");
213 
214   // WHEN: A run contains a result
215   Writer.createRun("sarif test", "sarif test runner", "1.0.0");
216   unsigned ruleIdx = Writer.createRule(Rule);
217   const SarifResult &Result = SarifResult::create(ruleIdx);
218   Writer.appendResult(Result);
219   std::string Output = serializeSarifDocument(Writer.createDocument());
220 
221   // THEN:
222   ASSERT_THAT(Output, ::testing::StrEq(ExpectedOutput));
223 }
224 
225 // Check that serializing artifacts from results produces valid SARIF
TEST_F(SarifDocumentWriterTest,checkSerializingArtifacts)226 TEST_F(SarifDocumentWriterTest, checkSerializingArtifacts) {
227   // GIVEN:
228   const std::string ExpectedOutput =
229       R"({"$schema":"https://docs.oasis-open.org/sarif/sarif/v2.1.0/cos02/schemas/sarif-schema-2.1.0.json","runs":[{"artifacts":[{"length":40,"location":{"index":0,"uri":"file:///main.cpp"},"mimeType":"text/plain","roles":["resultFile"]}],"columnKind":"unicodeCodePoints","results":[{"locations":[{"physicalLocation":{"artifactLocation":{"index":0},"region":{"endColumn":14,"startColumn":14,"startLine":3}}}],"message":{"text":"expected ';' after top level declarator"},"ruleId":"clang.unittest","ruleIndex":0}],"tool":{"driver":{"fullName":"sarif test runner","informationUri":"https://clang.llvm.org/docs/UsersManual.html","language":"en-US","name":"sarif test","rules":[{"fullDescription":{"text":"Example rule created during unit tests"},"id":"clang.unittest","name":"clang unit test"}],"version":"1.0.0"}}}],"version":"2.1.0"})";
230 
231   SarifDocumentWriter Writer{SourceMgr};
232   const SarifRule &Rule =
233       SarifRule::create()
234           .setRuleId("clang.unittest")
235           .setDescription("Example rule created during unit tests")
236           .setName("clang unit test");
237 
238   // WHEN: A result is added with valid source locations for its diagnostics
239   Writer.createRun("sarif test", "sarif test runner", "1.0.0");
240   unsigned RuleIdx = Writer.createRule(Rule);
241 
242   llvm::SmallVector<CharSourceRange, 1> DiagLocs;
243   const char *SourceText = "int foo = 0;\n"
244                            "int bar = 1;\n"
245                            "float x = 0.0\n";
246 
247   FileID MainFileID =
248       registerSource("/main.cpp", SourceText, /* IsMainFile = */ true);
249   CharSourceRange SourceCSR =
250       getFakeCharSourceRange(MainFileID, {3, 14}, {3, 14});
251 
252   DiagLocs.push_back(SourceCSR);
253 
254   const SarifResult &Result =
255       SarifResult::create(RuleIdx).setLocations(DiagLocs).setDiagnosticMessage(
256           "expected ';' after top level declarator");
257   Writer.appendResult(Result);
258   std::string Output = serializeSarifDocument(Writer.createDocument());
259 
260   // THEN: Assert that the serialized SARIF is as expected
261   ASSERT_THAT(Output, ::testing::StrEq(ExpectedOutput));
262 }
263 
TEST_F(SarifDocumentWriterTest,checkSerializingCodeflows)264 TEST_F(SarifDocumentWriterTest, checkSerializingCodeflows) {
265   // GIVEN:
266   const std::string ExpectedOutput =
267       R"({"$schema":"https://docs.oasis-open.org/sarif/sarif/v2.1.0/cos02/schemas/sarif-schema-2.1.0.json","runs":[{"artifacts":[{"length":27,"location":{"index":1,"uri":"file:///test-header-1.h"},"mimeType":"text/plain","roles":["resultFile"]},{"length":30,"location":{"index":2,"uri":"file:///test-header-2.h"},"mimeType":"text/plain","roles":["resultFile"]},{"length":28,"location":{"index":3,"uri":"file:///test-header-3.h"},"mimeType":"text/plain","roles":["resultFile"]},{"length":41,"location":{"index":0,"uri":"file:///main.cpp"},"mimeType":"text/plain","roles":["resultFile"]}],"columnKind":"unicodeCodePoints","results":[{"codeFlows":[{"threadFlows":[{"locations":[{"importance":"essential","location":{"message":{"text":"Message #1"},"physicalLocation":{"artifactLocation":{"index":1},"region":{"endColumn":8,"endLine":2,"startColumn":1,"startLine":1}}}},{"importance":"important","location":{"message":{"text":"Message #2"},"physicalLocation":{"artifactLocation":{"index":2},"region":{"endColumn":8,"endLine":2,"startColumn":1,"startLine":1}}}},{"importance":"unimportant","location":{"message":{"text":"Message #3"},"physicalLocation":{"artifactLocation":{"index":3},"region":{"endColumn":8,"endLine":2,"startColumn":1,"startLine":1}}}}]}]}],"locations":[{"physicalLocation":{"artifactLocation":{"index":0},"region":{"endColumn":8,"endLine":2,"startColumn":5,"startLine":2}}}],"message":{"text":"Redefinition of 'foo'"},"ruleId":"clang.unittest","ruleIndex":0}],"tool":{"driver":{"fullName":"sarif test runner","informationUri":"https://clang.llvm.org/docs/UsersManual.html","language":"en-US","name":"sarif test","rules":[{"fullDescription":{"text":"Example rule created during unit tests"},"id":"clang.unittest","name":"clang unit test"}],"version":"1.0.0"}}}],"version":"2.1.0"})";
268 
269   const char *SourceText = "int foo = 0;\n"
270                            "int foo = 1;\n"
271                            "float x = 0.0;\n";
272   FileID MainFileID =
273       registerSource("/main.cpp", SourceText, /* IsMainFile = */ true);
274   CharSourceRange DiagLoc{getFakeCharSourceRange(MainFileID, {2, 5}, {2, 8})};
275 
276   SarifDocumentWriter Writer{SourceMgr};
277   const SarifRule &Rule =
278       SarifRule::create()
279           .setRuleId("clang.unittest")
280           .setDescription("Example rule created during unit tests")
281           .setName("clang unit test");
282 
283   constexpr unsigned int NUM_CASES = 3;
284   llvm::SmallVector<ThreadFlow, NUM_CASES> Threadflows;
285   const char *HeaderTexts[NUM_CASES]{("#pragma once\n"
286                                       "#include <foo>"),
287                                      ("#ifndef FOO\n"
288                                       "#define FOO\n"
289                                       "#endif"),
290                                      ("#ifdef FOO\n"
291                                       "#undef FOO\n"
292                                       "#endif")};
293   const char *HeaderNames[NUM_CASES]{"/test-header-1.h", "/test-header-2.h",
294                                      "/test-header-3.h"};
295   ThreadFlowImportance Importances[NUM_CASES]{
296       ThreadFlowImportance::Essential, ThreadFlowImportance::Important,
297       ThreadFlowImportance::Unimportant};
298   for (size_t Idx = 0; Idx != NUM_CASES; ++Idx) {
299     FileID FID = registerSource(HeaderNames[Idx], HeaderTexts[Idx]);
300     CharSourceRange &&CSR = getFakeCharSourceRange(FID, {1, 1}, {2, 8});
301     std::string Message = llvm::formatv("Message #{0}", Idx + 1);
302     ThreadFlow Item = ThreadFlow::create()
303                           .setRange(CSR)
304                           .setImportance(Importances[Idx])
305                           .setMessage(Message);
306     Threadflows.push_back(Item);
307   }
308 
309   // WHEN: A result containing code flows and diagnostic locations is added
310   Writer.createRun("sarif test", "sarif test runner", "1.0.0");
311   unsigned RuleIdx = Writer.createRule(Rule);
312   const SarifResult &Result = SarifResult::create(RuleIdx)
313                                   .setLocations({DiagLoc})
314                                   .setDiagnosticMessage("Redefinition of 'foo'")
315                                   .setThreadFlows(Threadflows);
316   Writer.appendResult(Result);
317   std::string Output = serializeSarifDocument(Writer.createDocument());
318 
319   // THEN: Assert that the serialized SARIF is as expected
320   ASSERT_THAT(Output, ::testing::StrEq(ExpectedOutput));
321 }
322 
323 } // namespace
324