clang/unittests/Basic/SarifTest.cpp

   1 //===- unittests/Basic/SarifTest.cpp - Test writing SARIF documents -------===//
   2 //
   3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
   4 // See https://llvm.org/LICENSE.txt for license information.
   5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
   6 //
   7 //===----------------------------------------------------------------------===//
   8
   9 #include "clang/Basic/Sarif.h"
  10 #include "clang/Basic/DiagnosticOptions.h"
  11 #include "clang/Basic/FileManager.h"
  12 #include "clang/Basic/FileSystemOptions.h"
  13 #include "clang/Basic/LangOptions.h"
  14 #include "clang/Basic/SourceLocation.h"
  15 #include "clang/Basic/SourceManager.h"
  16 #include "llvm/ADT/StringRef.h"
  17 #include "llvm/Support/FormatVariadic.h"
  18 #include "llvm/Support/JSON.h"
  19 #include "llvm/Support/MemoryBuffer.h"
  20 #include "llvm/Support/VirtualFileSystem.h"
  21 #include "llvm/Support/raw_ostream.h"
  22 #include "gmock/gmock.h"
  23 #include "gtest/gtest.h"
  24
  25 #include <algorithm>
  26
  27 using namespace clang;
  28
  29 namespace {
  30
  31 using LineCol = std::pair<unsigned int, unsigned int>;
  32
  33 static std::string serializeSarifDocument(llvm::json::Object &&Doc) {
  34   std::string Output;
  35   llvm::json::Value Value(std::move(Doc));
  36   llvm::raw_string_ostream OS{Output};
  37   OS << llvm::formatv("{0}", Value);
  38   OS.flush();
  39   return Output;
  40 }
  41
  42 class SarifDocumentWriterTest : public ::testing::Test {
  43 protected:
  44   SarifDocumentWriterTest()
  45       : InMemoryFileSystem(new llvm::vfs::InMemoryFileSystem),
  46         FileMgr(FileSystemOptions(), InMemoryFileSystem),
  47         DiagID(new DiagnosticIDs()), DiagOpts(new DiagnosticOptions()),
  48         Diags(DiagID, DiagOpts.get(), new IgnoringDiagConsumer()),
  49         SourceMgr(Diags, FileMgr) {}
  50
  51   IntrusiveRefCntPtr<llvm::vfs::InMemoryFileSystem> InMemoryFileSystem;
  52   FileManager FileMgr;
  53   IntrusiveRefCntPtr<DiagnosticIDs> DiagID;
  54   IntrusiveRefCntPtr<DiagnosticOptions> DiagOpts;
  55   DiagnosticsEngine Diags;
  56   SourceManager SourceMgr;
  57   LangOptions LangOpts;
  58
  59   FileID registerSource(llvm::StringRef Name, const char *SourceText,
  60                         bool IsMainFile = false) {
  61     std::unique_ptr<llvm::MemoryBuffer> SourceBuf =
  62         llvm::MemoryBuffer::getMemBuffer(SourceText);
  63     FileEntryRef SourceFile =
  64         FileMgr.getVirtualFileRef(Name, SourceBuf->getBufferSize(), 0);
  65     SourceMgr.overrideFileContents(SourceFile, std::move(SourceBuf));
  66     FileID FID = SourceMgr.getOrCreateFileID(SourceFile, SrcMgr::C_User);
  67     if (IsMainFile)
  68       SourceMgr.setMainFileID(FID);
  69     return FID;
  70   }
  71
  72   CharSourceRange getFakeCharSourceRange(FileID FID, LineCol Begin,
  73                                          LineCol End) {
  74     auto BeginLoc = SourceMgr.translateLineCol(FID, Begin.first, Begin.second);
  75     auto EndLoc = SourceMgr.translateLineCol(FID, End.first, End.second);
  76     return CharSourceRange{SourceRange{BeginLoc, EndLoc}, /* ITR = */ false};
  77   }
  78 };
  79
  80 TEST_F(SarifDocumentWriterTest, canCreateEmptyDocument) {
  81   // GIVEN:
  82   SarifDocumentWriter Writer{SourceMgr};
  83
  84   // WHEN:
  85   const llvm::json::Object &EmptyDoc = Writer.createDocument();
  86   std::vector<StringRef> Keys(EmptyDoc.size());
  87   std::transform(EmptyDoc.begin(), EmptyDoc.end(), Keys.begin(),
  88                  [](auto Item) { return Item.getFirst(); });
  89
  90   // THEN:
  91   ASSERT_THAT(Keys, testing::UnorderedElementsAre("$schema", "version"));
  92 }
  93
  94 // Test that a newly inserted run will associate correct tool names
  95 TEST_F(SarifDocumentWriterTest, canCreateDocumentWithOneRun) {
  96   // GIVEN:
  97   SarifDocumentWriter Writer{SourceMgr};
  98   const char *ShortName = "sariftest";
  99   const char *LongName = "sarif writer test";
 100
 101   // WHEN:
 102   Writer.createRun(ShortName, LongName);
 103   Writer.endRun();
 104   const llvm::json::Object &Doc = Writer.createDocument();
 105   const llvm::json::Array *Runs = Doc.getArray("runs");
 106
 107   // THEN:
 108   // A run was created
 109   ASSERT_THAT(Runs, testing::NotNull());
 110
 111   // It is the only run
 112   ASSERT_EQ(Runs->size(), 1UL);
 113
 114   // The tool associated with the run was the tool
 115   const llvm::json::Object *Driver =
 116       Runs->begin()->getAsObject()->getObject("tool")->getObject("driver");
 117   ASSERT_THAT(Driver, testing::NotNull());
 118
 119   ASSERT_TRUE(Driver->getString("name").has_value());
 120   ASSERT_TRUE(Driver->getString("fullName").has_value());
 121   ASSERT_TRUE(Driver->getString("language").has_value());
 122
 123   EXPECT_EQ(*Driver->getString("name"), ShortName);
 124   EXPECT_EQ(*Driver->getString("fullName"), LongName);
 125   EXPECT_EQ(*Driver->getString("language"), "en-US");
 126 }
 127
 128 TEST_F(SarifDocumentWriterTest, addingResultsWillCrashIfThereIsNoRun) {
 129 #if defined(NDEBUG) || !GTEST_HAS_DEATH_TEST
 130   GTEST_SKIP() << "This death test is only available for debug builds.";
 131 #endif
 132   // GIVEN:
 133   SarifDocumentWriter Writer{SourceMgr};
 134
 135   // WHEN:
 136   //  A SarifDocumentWriter::createRun(...) was not called prior to
 137   //  SarifDocumentWriter::appendResult(...)
 138   // But a rule exists
 139   auto RuleIdx = Writer.createRule(SarifRule::create());
 140   const SarifResult &EmptyResult = SarifResult::create(RuleIdx);
 141
 142   // THEN:
 143   auto Matcher = ::testing::AnyOf(
 144       ::testing::HasSubstr("create a run first"),
 145       ::testing::HasSubstr("no runs associated with the document"));
 146   ASSERT_DEATH(Writer.appendResult(EmptyResult), Matcher);
 147 }
 148
 149 TEST_F(SarifDocumentWriterTest, settingInvalidRankWillCrash) {
 150 #if defined(NDEBUG) || !GTEST_HAS_DEATH_TEST
 151   GTEST_SKIP() << "This death test is only available for debug builds.";
 152 #endif
 153   // GIVEN:
 154   SarifDocumentWriter Writer{SourceMgr};
 155
 156   // WHEN:
 157   // A SarifReportingConfiguration is created with an invalid "rank"
 158   // * Ranks below 0.0 are invalid
 159   // * Ranks above 100.0 are invalid
 160
 161   // THEN: The builder will crash in either case
 162   EXPECT_DEATH(SarifReportingConfiguration::create().setRank(-1.0),
 163                ::testing::HasSubstr("Rule rank cannot be smaller than 0.0"));
 164   EXPECT_DEATH(SarifReportingConfiguration::create().setRank(101.0),
 165                ::testing::HasSubstr("Rule rank cannot be larger than 100.0"));
 166 }
 167
 168 TEST_F(SarifDocumentWriterTest, creatingResultWithDisabledRuleWillCrash) {
 169 #if defined(NDEBUG) || !GTEST_HAS_DEATH_TEST
 170   GTEST_SKIP() << "This death test is only available for debug builds.";
 171 #endif
 172
 173   // GIVEN:
 174   SarifDocumentWriter Writer{SourceMgr};
 175
 176   // WHEN:
 177   // A disabled Rule is created, and a result is create referencing this rule
 178   const auto &Config = SarifReportingConfiguration::create().disable();
 179   auto RuleIdx =
 180       Writer.createRule(SarifRule::create().setDefaultConfiguration(Config));
 181   const SarifResult &Result = SarifResult::create(RuleIdx);
 182
 183   // THEN:
 184   // SarifResult::create(...) will produce a crash
 185   ASSERT_DEATH(
 186       Writer.appendResult(Result),
 187       ::testing::HasSubstr("Cannot add a result referencing a disabled Rule"));
 188 }
 189
 190 // Test adding rule and result shows up in the final document
 191 TEST_F(SarifDocumentWriterTest, addingResultWithValidRuleAndRunIsOk) {
 192   // GIVEN:
 193   SarifDocumentWriter Writer{SourceMgr};
 194   const SarifRule &Rule =
 195       SarifRule::create()
 196           .setRuleId("clang.unittest")
 197           .setDescription("Example rule created during unit tests")
 198           .setName("clang unit test");
 199
 200   // WHEN:
 201   Writer.createRun("sarif test", "sarif test runner");
 202   unsigned RuleIdx = Writer.createRule(Rule);
 203   const SarifResult &Result = SarifResult::create(RuleIdx);
 204
 205   Writer.appendResult(Result);
 206   const llvm::json::Object &Doc = Writer.createDocument();
 207
 208   // THEN:
 209   // A document with a valid schema and version exists
 210   ASSERT_THAT(Doc.get("$schema"), ::testing::NotNull());
 211   ASSERT_THAT(Doc.get("version"), ::testing::NotNull());
 212   const llvm::json::Array *Runs = Doc.getArray("runs");
 213
 214   // A run exists on this document
 215   ASSERT_THAT(Runs, ::testing::NotNull());
 216   ASSERT_EQ(Runs->size(), 1UL);
 217   const llvm::json::Object *TheRun = Runs->back().getAsObject();
 218
 219   // The run has slots for tools, results, rules and artifacts
 220   ASSERT_THAT(TheRun->get("tool"), ::testing::NotNull());
 221   ASSERT_THAT(TheRun->get("results"), ::testing::NotNull());
 222   ASSERT_THAT(TheRun->get("artifacts"), ::testing::NotNull());
 223   const llvm::json::Object *Driver =
 224       TheRun->getObject("tool")->getObject("driver");
 225   const llvm::json::Array *Results = TheRun->getArray("results");
 226   const llvm::json::Array *Artifacts = TheRun->getArray("artifacts");
 227
 228   // The tool is as expected
 229   ASSERT_TRUE(Driver->getString("name").has_value());
 230   ASSERT_TRUE(Driver->getString("fullName").has_value());
 231
 232   EXPECT_EQ(*Driver->getString("name"), "sarif test");
 233   EXPECT_EQ(*Driver->getString("fullName"), "sarif test runner");
 234
 235   // The results are as expected
 236   EXPECT_EQ(Results->size(), 1UL);
 237
 238   // The artifacts are as expected
 239   EXPECT_TRUE(Artifacts->empty());
 240 }
 241
 242 TEST_F(SarifDocumentWriterTest, checkSerializingResultsWithDefaultRuleConfig) {
 243   // GIVEN:
 244   const std::string ExpectedOutput =
 245       R"({"$schema":"https://docs.oasis-open.org/sarif/sarif/v2.1.0/cos02/schemas/sarif-schema-2.1.0.json","runs":[{"artifacts":[],"columnKind":"unicodeCodePoints","results":[{"level":"warning","message":{"text":""},"ruleId":"clang.unittest","ruleIndex":0}],"tool":{"driver":{"fullName":"sarif test runner","informationUri":"https://clang.llvm.org/docs/UsersManual.html","language":"en-US","name":"sarif test","rules":[{"defaultConfiguration":{"enabled":true,"level":"warning","rank":-1},"fullDescription":{"text":"Example rule created during unit tests"},"id":"clang.unittest","name":"clang unit test"}],"version":"1.0.0"}}}],"version":"2.1.0"})";
 246
 247   SarifDocumentWriter Writer{SourceMgr};
 248   const SarifRule &Rule =
 249       SarifRule::create()
 250           .setRuleId("clang.unittest")
 251           .setDescription("Example rule created during unit tests")
 252           .setName("clang unit test");
 253
 254   // WHEN: A run contains a result
 255   Writer.createRun("sarif test", "sarif test runner", "1.0.0");
 256   unsigned RuleIdx = Writer.createRule(Rule);
 257   const SarifResult &Result = SarifResult::create(RuleIdx);
 258   Writer.appendResult(Result);
 259   std::string Output = serializeSarifDocument(Writer.createDocument());
 260
 261   // THEN:
 262   ASSERT_THAT(Output, ::testing::StrEq(ExpectedOutput));
 263 }
 264
 265 TEST_F(SarifDocumentWriterTest, checkSerializingResultsWithCustomRuleConfig) {
 266   // GIVEN:
 267   const std::string ExpectedOutput =
 268       R"({"$schema":"https://docs.oasis-open.org/sarif/sarif/v2.1.0/cos02/schemas/sarif-schema-2.1.0.json","runs":[{"artifacts":[],"columnKind":"unicodeCodePoints","results":[{"level":"error","message":{"text":""},"ruleId":"clang.unittest","ruleIndex":0}],"tool":{"driver":{"fullName":"sarif test runner","informationUri":"https://clang.llvm.org/docs/UsersManual.html","language":"en-US","name":"sarif test","rules":[{"defaultConfiguration":{"enabled":true,"level":"error","rank":35.5},"fullDescription":{"text":"Example rule created during unit tests"},"id":"clang.unittest","name":"clang unit test"}],"version":"1.0.0"}}}],"version":"2.1.0"})";
 269
 270   SarifDocumentWriter Writer{SourceMgr};
 271   const SarifRule &Rule =
 272       SarifRule::create()
 273           .setRuleId("clang.unittest")
 274           .setDescription("Example rule created during unit tests")
 275           .setName("clang unit test")
 276           .setDefaultConfiguration(SarifReportingConfiguration::create()
 277                                        .setLevel(SarifResultLevel::Error)
 278                                        .setRank(35.5));
 279
 280   // WHEN: A run contains a result
 281   Writer.createRun("sarif test", "sarif test runner", "1.0.0");
 282   unsigned RuleIdx = Writer.createRule(Rule);
 283   const SarifResult &Result = SarifResult::create(RuleIdx);
 284   Writer.appendResult(Result);
 285   std::string Output = serializeSarifDocument(Writer.createDocument());
 286
 287   // THEN:
 288   ASSERT_THAT(Output, ::testing::StrEq(ExpectedOutput));
 289 }
 290
 291 // Check that serializing artifacts from results produces valid SARIF
 292 TEST_F(SarifDocumentWriterTest, checkSerializingArtifacts) {
 293   // GIVEN:
 294   const std::string ExpectedOutput =
 295       R"({"$schema":"https://docs.oasis-open.org/sarif/sarif/v2.1.0/cos02/schemas/sarif-schema-2.1.0.json","runs":[{"artifacts":[{"length":40,"location":{"index":0,"uri":"file:///main.cpp"},"mimeType":"text/plain","roles":["resultFile"]}],"columnKind":"unicodeCodePoints","results":[{"level":"error","locations":[{"physicalLocation":{"artifactLocation":{"index":0,"uri":"file:///main.cpp"},"region":{"endColumn":14,"startColumn":14,"startLine":3}}}],"message":{"text":"expected ';' after top level declarator"},"ruleId":"clang.unittest","ruleIndex":0}],"tool":{"driver":{"fullName":"sarif test runner","informationUri":"https://clang.llvm.org/docs/UsersManual.html","language":"en-US","name":"sarif test","rules":[{"defaultConfiguration":{"enabled":true,"level":"warning","rank":-1},"fullDescription":{"text":"Example rule created during unit tests"},"id":"clang.unittest","name":"clang unit test"}],"version":"1.0.0"}}}],"version":"2.1.0"})";
 296
 297   SarifDocumentWriter Writer{SourceMgr};
 298   const SarifRule &Rule =
 299       SarifRule::create()
 300           .setRuleId("clang.unittest")
 301           .setDescription("Example rule created during unit tests")
 302           .setName("clang unit test");
 303
 304   // WHEN: A result is added with valid source locations for its diagnostics
 305   Writer.createRun("sarif test", "sarif test runner", "1.0.0");
 306   unsigned RuleIdx = Writer.createRule(Rule);
 307
 308   llvm::SmallVector<CharSourceRange, 1> DiagLocs;
 309   const char *SourceText = "int foo = 0;\n"
 310                            "int bar = 1;\n"
 311                            "float x = 0.0\n";
 312
 313   FileID MainFileID =
 314       registerSource("/main.cpp", SourceText, /* IsMainFile = */ true);
 315   CharSourceRange SourceCSR =
 316       getFakeCharSourceRange(MainFileID, {3, 14}, {3, 14});
 317
 318   DiagLocs.push_back(SourceCSR);
 319
 320   const SarifResult &Result =
 321       SarifResult::create(RuleIdx)
 322           .setLocations(DiagLocs)
 323           .setDiagnosticMessage("expected ';' after top level declarator")
 324           .setDiagnosticLevel(SarifResultLevel::Error);
 325   Writer.appendResult(Result);
 326   std::string Output = serializeSarifDocument(Writer.createDocument());
 327
 328   // THEN: Assert that the serialized SARIF is as expected
 329   ASSERT_THAT(Output, ::testing::StrEq(ExpectedOutput));
 330 }
 331
 332 TEST_F(SarifDocumentWriterTest, checkSerializingCodeflows) {
 333   // GIVEN:
 334   const std::string ExpectedOutput =
 335       R"({"$schema":"https://docs.oasis-open.org/sarif/sarif/v2.1.0/cos02/schemas/sarif-schema-2.1.0.json","runs":[{"artifacts":[{"length":41,"location":{"index":0,"uri":"file:///main.cpp"},"mimeType":"text/plain","roles":["resultFile"]},{"length":27,"location":{"index":1,"uri":"file:///test-header-1.h"},"mimeType":"text/plain","roles":["resultFile"]},{"length":30,"location":{"index":2,"uri":"file:///test-header-2.h"},"mimeType":"text/plain","roles":["resultFile"]},{"length":28,"location":{"index":3,"uri":"file:///test-header-3.h"},"mimeType":"text/plain","roles":["resultFile"]}],"columnKind":"unicodeCodePoints","results":[{"codeFlows":[{"threadFlows":[{"locations":[{"importance":"essential","location":{"message":{"text":"Message #1"},"physicalLocation":{"artifactLocation":{"index":1,"uri":"file:///test-header-1.h"},"region":{"endColumn":8,"endLine":2,"startColumn":1,"startLine":1}}}},{"importance":"important","location":{"message":{"text":"Message #2"},"physicalLocation":{"artifactLocation":{"index":2,"uri":"file:///test-header-2.h"},"region":{"endColumn":8,"endLine":2,"startColumn":1,"startLine":1}}}},{"importance":"unimportant","location":{"message":{"text":"Message #3"},"physicalLocation":{"artifactLocation":{"index":3,"uri":"file:///test-header-3.h"},"region":{"endColumn":8,"endLine":2,"startColumn":1,"startLine":1}}}}]}]}],"level":"warning","locations":[{"physicalLocation":{"artifactLocation":{"index":0,"uri":"file:///main.cpp"},"region":{"endColumn":8,"endLine":2,"startColumn":5,"startLine":2}}}],"message":{"text":"Redefinition of 'foo'"},"ruleId":"clang.unittest","ruleIndex":0}],"tool":{"driver":{"fullName":"sarif test runner","informationUri":"https://clang.llvm.org/docs/UsersManual.html","language":"en-US","name":"sarif test","rules":[{"defaultConfiguration":{"enabled":true,"level":"warning","rank":-1},"fullDescription":{"text":"Example rule created during unit tests"},"id":"clang.unittest","name":"clang unit test"}],"version":"1.0.0"}}}],"version":"2.1.0"})";
 336
 337   const char *SourceText = "int foo = 0;\n"
 338                            "int foo = 1;\n"
 339                            "float x = 0.0;\n";
 340   FileID MainFileID =
 341       registerSource("/main.cpp", SourceText, /* IsMainFile = */ true);
 342   CharSourceRange DiagLoc{getFakeCharSourceRange(MainFileID, {2, 5}, {2, 8})};
 343
 344   SarifDocumentWriter Writer{SourceMgr};
 345   const SarifRule &Rule =
 346       SarifRule::create()
 347           .setRuleId("clang.unittest")
 348           .setDescription("Example rule created during unit tests")
 349           .setName("clang unit test");
 350
 351   constexpr unsigned int NumCases = 3;
 352   llvm::SmallVector<ThreadFlow, NumCases> Threadflows;
 353   const char *HeaderTexts[NumCases]{("#pragma once\n"
 354                                      "#include <foo>"),
 355                                     ("#ifndef FOO\n"
 356                                      "#define FOO\n"
 357                                      "#endif"),
 358                                     ("#ifdef FOO\n"
 359                                      "#undef FOO\n"
 360                                      "#endif")};
 361   const char *HeaderNames[NumCases]{"/test-header-1.h", "/test-header-2.h",
 362                                     "/test-header-3.h"};
 363   ThreadFlowImportance Importances[NumCases]{ThreadFlowImportance::Essential,
 364                                              ThreadFlowImportance::Important,
 365                                              ThreadFlowImportance::Unimportant};
 366   for (size_t Idx = 0; Idx != NumCases; ++Idx) {
 367     FileID FID = registerSource(HeaderNames[Idx], HeaderTexts[Idx]);
 368     CharSourceRange &&CSR = getFakeCharSourceRange(FID, {1, 1}, {2, 8});
 369     std::string Message = llvm::formatv("Message #{0}", Idx + 1);
 370     ThreadFlow Item = ThreadFlow::create()
 371                           .setRange(CSR)
 372                           .setImportance(Importances[Idx])
 373                           .setMessage(Message);
 374     Threadflows.push_back(Item);
 375   }
 376
 377   // WHEN: A result containing code flows and diagnostic locations is added
 378   Writer.createRun("sarif test", "sarif test runner", "1.0.0");
 379   unsigned RuleIdx = Writer.createRule(Rule);
 380   const SarifResult &Result =
 381       SarifResult::create(RuleIdx)
 382           .setLocations({DiagLoc})
 383           .setDiagnosticMessage("Redefinition of 'foo'")
 384           .setThreadFlows(Threadflows)
 385           .setDiagnosticLevel(SarifResultLevel::Warning);
 386   Writer.appendResult(Result);
 387   std::string Output = serializeSarifDocument(Writer.createDocument());
 388
 389   // THEN: Assert that the serialized SARIF is as expected
 390   ASSERT_THAT(Output, ::testing::StrEq(ExpectedOutput));
 391 }
 392
 393 } // namespace