clang/unittests/Basic/SarifTest.cpp

   1 //===- unittests/Basic/SarifTest.cpp - Test writing SARIF documents -------===//
   2 //
   3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
   4 // See https://llvm.org/LICENSE.txt for license information.
   5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
   6 //
   7 //===----------------------------------------------------------------------===//
   8
   9 #include "clang/Basic/Sarif.h"
  10 #include "clang/Basic/DiagnosticOptions.h"
  11 #include "clang/Basic/FileManager.h"
  12 #include "clang/Basic/FileSystemOptions.h"
  13 #include "clang/Basic/LangOptions.h"
  14 #include "clang/Basic/SourceLocation.h"
  15 #include "clang/Basic/SourceManager.h"
  16 #include "llvm/ADT/StringRef.h"
  17 #include "llvm/Support/FormatVariadic.h"
  18 #include "llvm/Support/JSON.h"
  19 #include "llvm/Support/MemoryBuffer.h"
  20 #include "llvm/Support/VirtualFileSystem.h"
  21 #include "llvm/Support/raw_ostream.h"
  22 #include "gmock/gmock.h"
  23 #include "gtest/gtest.h"
  24
  25 #include <algorithm>
  26
  27 using namespace clang;
  28
  29 namespace {
  30
  31 using LineCol = std::pair<unsigned int, unsigned int>;
  32
  33 static std::string serializeSarifDocument(llvm::json::Object &&Doc) {
  34   std::string Output;
  35   llvm::json::Value Value(std::move(Doc));
  36   llvm::raw_string_ostream OS{Output};
  37   OS << llvm::formatv("{0}", Value);
  38   return Output;
  39 }
  40
  41 class SarifDocumentWriterTest : public ::testing::Test {
  42 protected:
  43   SarifDocumentWriterTest()
  44       : InMemoryFileSystem(new llvm::vfs::InMemoryFileSystem),
  45         FileMgr(FileSystemOptions(), InMemoryFileSystem),
  46         DiagID(new DiagnosticIDs()), DiagOpts(new DiagnosticOptions()),
  47         Diags(DiagID, DiagOpts.get(), new IgnoringDiagConsumer()),
  48         SourceMgr(Diags, FileMgr) {}
  49
  50   IntrusiveRefCntPtr<llvm::vfs::InMemoryFileSystem> InMemoryFileSystem;
  51   FileManager FileMgr;
  52   IntrusiveRefCntPtr<DiagnosticIDs> DiagID;
  53   IntrusiveRefCntPtr<DiagnosticOptions> DiagOpts;
  54   DiagnosticsEngine Diags;
  55   SourceManager SourceMgr;
  56   LangOptions LangOpts;
  57
  58   FileID registerSource(llvm::StringRef Name, const char *SourceText,
  59                         bool IsMainFile = false) {
  60     std::unique_ptr<llvm::MemoryBuffer> SourceBuf =
  61         llvm::MemoryBuffer::getMemBuffer(SourceText);
  62     FileEntryRef SourceFile =
  63         FileMgr.getVirtualFileRef(Name, SourceBuf->getBufferSize(), 0);
  64     SourceMgr.overrideFileContents(SourceFile, std::move(SourceBuf));
  65     FileID FID = SourceMgr.getOrCreateFileID(SourceFile, SrcMgr::C_User);
  66     if (IsMainFile)
  67       SourceMgr.setMainFileID(FID);
  68     return FID;
  69   }
  70
  71   CharSourceRange getFakeCharSourceRange(FileID FID, LineCol Begin,
  72                                          LineCol End) {
  73     auto BeginLoc = SourceMgr.translateLineCol(FID, Begin.first, Begin.second);
  74     auto EndLoc = SourceMgr.translateLineCol(FID, End.first, End.second);
  75     return CharSourceRange{SourceRange{BeginLoc, EndLoc}, /* ITR = */ false};
  76   }
  77 };
  78
  79 TEST_F(SarifDocumentWriterTest, canCreateEmptyDocument) {
  80   // GIVEN:
  81   SarifDocumentWriter Writer{SourceMgr};
  82
  83   // WHEN:
  84   const llvm::json::Object &EmptyDoc = Writer.createDocument();
  85   std::vector<StringRef> Keys(EmptyDoc.size());
  86   std::transform(EmptyDoc.begin(), EmptyDoc.end(), Keys.begin(),
  87                  [](auto Item) { return Item.getFirst(); });
  88
  89   // THEN:
  90   ASSERT_THAT(Keys, testing::UnorderedElementsAre("$schema", "version"));
  91 }
  92
  93 // Test that a newly inserted run will associate correct tool names
  94 TEST_F(SarifDocumentWriterTest, canCreateDocumentWithOneRun) {
  95   // GIVEN:
  96   SarifDocumentWriter Writer{SourceMgr};
  97   const char *ShortName = "sariftest";
  98   const char *LongName = "sarif writer test";
  99
 100   // WHEN:
 101   Writer.createRun(ShortName, LongName);
 102   Writer.endRun();
 103   const llvm::json::Object &Doc = Writer.createDocument();
 104   const llvm::json::Array *Runs = Doc.getArray("runs");
 105
 106   // THEN:
 107   // A run was created
 108   ASSERT_THAT(Runs, testing::NotNull());
 109
 110   // It is the only run
 111   ASSERT_EQ(Runs->size(), 1UL);
 112
 113   // The tool associated with the run was the tool
 114   const llvm::json::Object *Driver =
 115       Runs->begin()->getAsObject()->getObject("tool")->getObject("driver");
 116   ASSERT_THAT(Driver, testing::NotNull());
 117
 118   ASSERT_TRUE(Driver->getString("name").has_value());
 119   ASSERT_TRUE(Driver->getString("fullName").has_value());
 120   ASSERT_TRUE(Driver->getString("language").has_value());
 121
 122   EXPECT_EQ(*Driver->getString("name"), ShortName);
 123   EXPECT_EQ(*Driver->getString("fullName"), LongName);
 124   EXPECT_EQ(*Driver->getString("language"), "en-US");
 125 }
 126
 127 TEST_F(SarifDocumentWriterTest, addingResultsWillCrashIfThereIsNoRun) {
 128 #if defined(NDEBUG) || !GTEST_HAS_DEATH_TEST
 129   GTEST_SKIP() << "This death test is only available for debug builds.";
 130 #endif
 131   // GIVEN:
 132   SarifDocumentWriter Writer{SourceMgr};
 133
 134   // WHEN:
 135   //  A SarifDocumentWriter::createRun(...) was not called prior to
 136   //  SarifDocumentWriter::appendResult(...)
 137   // But a rule exists
 138   auto RuleIdx = Writer.createRule(SarifRule::create());
 139   const SarifResult &EmptyResult = SarifResult::create(RuleIdx);
 140
 141   // THEN:
 142   auto Matcher = ::testing::AnyOf(
 143       ::testing::HasSubstr("create a run first"),
 144       ::testing::HasSubstr("no runs associated with the document"));
 145   ASSERT_DEATH(Writer.appendResult(EmptyResult), Matcher);
 146 }
 147
 148 TEST_F(SarifDocumentWriterTest, settingInvalidRankWillCrash) {
 149 #if defined(NDEBUG) || !GTEST_HAS_DEATH_TEST
 150   GTEST_SKIP() << "This death test is only available for debug builds.";
 151 #endif
 152   // GIVEN:
 153   SarifDocumentWriter Writer{SourceMgr};
 154
 155   // WHEN:
 156   // A SarifReportingConfiguration is created with an invalid "rank"
 157   // * Ranks below 0.0 are invalid
 158   // * Ranks above 100.0 are invalid
 159
 160   // THEN: The builder will crash in either case
 161   EXPECT_DEATH(SarifReportingConfiguration::create().setRank(-1.0),
 162                ::testing::HasSubstr("Rule rank cannot be smaller than 0.0"));
 163   EXPECT_DEATH(SarifReportingConfiguration::create().setRank(101.0),
 164                ::testing::HasSubstr("Rule rank cannot be larger than 100.0"));
 165 }
 166
 167 TEST_F(SarifDocumentWriterTest, creatingResultWithDisabledRuleWillCrash) {
 168 #if defined(NDEBUG) || !GTEST_HAS_DEATH_TEST
 169   GTEST_SKIP() << "This death test is only available for debug builds.";
 170 #endif
 171
 172   // GIVEN:
 173   SarifDocumentWriter Writer{SourceMgr};
 174
 175   // WHEN:
 176   // A disabled Rule is created, and a result is create referencing this rule
 177   const auto &Config = SarifReportingConfiguration::create().disable();
 178   auto RuleIdx =
 179       Writer.createRule(SarifRule::create().setDefaultConfiguration(Config));
 180   const SarifResult &Result = SarifResult::create(RuleIdx);
 181
 182   // THEN:
 183   // SarifResult::create(...) will produce a crash
 184   ASSERT_DEATH(
 185       Writer.appendResult(Result),
 186       ::testing::HasSubstr("Cannot add a result referencing a disabled Rule"));
 187 }
 188
 189 // Test adding rule and result shows up in the final document
 190 TEST_F(SarifDocumentWriterTest, addingResultWithValidRuleAndRunIsOk) {
 191   // GIVEN:
 192   SarifDocumentWriter Writer{SourceMgr};
 193   const SarifRule &Rule =
 194       SarifRule::create()
 195           .setRuleId("clang.unittest")
 196           .setDescription("Example rule created during unit tests")
 197           .setName("clang unit test");
 198
 199   // WHEN:
 200   Writer.createRun("sarif test", "sarif test runner");
 201   unsigned RuleIdx = Writer.createRule(Rule);
 202   const SarifResult &Result = SarifResult::create(RuleIdx);
 203
 204   Writer.appendResult(Result);
 205   const llvm::json::Object &Doc = Writer.createDocument();
 206
 207   // THEN:
 208   // A document with a valid schema and version exists
 209   ASSERT_THAT(Doc.get("$schema"), ::testing::NotNull());
 210   ASSERT_THAT(Doc.get("version"), ::testing::NotNull());
 211   const llvm::json::Array *Runs = Doc.getArray("runs");
 212
 213   // A run exists on this document
 214   ASSERT_THAT(Runs, ::testing::NotNull());
 215   ASSERT_EQ(Runs->size(), 1UL);
 216   const llvm::json::Object *TheRun = Runs->back().getAsObject();
 217
 218   // The run has slots for tools, results, rules and artifacts
 219   ASSERT_THAT(TheRun->get("tool"), ::testing::NotNull());
 220   ASSERT_THAT(TheRun->get("results"), ::testing::NotNull());
 221   ASSERT_THAT(TheRun->get("artifacts"), ::testing::NotNull());
 222   const llvm::json::Object *Driver =
 223       TheRun->getObject("tool")->getObject("driver");
 224   const llvm::json::Array *Results = TheRun->getArray("results");
 225   const llvm::json::Array *Artifacts = TheRun->getArray("artifacts");
 226
 227   // The tool is as expected
 228   ASSERT_TRUE(Driver->getString("name").has_value());
 229   ASSERT_TRUE(Driver->getString("fullName").has_value());
 230
 231   EXPECT_EQ(*Driver->getString("name"), "sarif test");
 232   EXPECT_EQ(*Driver->getString("fullName"), "sarif test runner");
 233
 234   // The results are as expected
 235   EXPECT_EQ(Results->size(), 1UL);
 236
 237   // The artifacts are as expected
 238   EXPECT_TRUE(Artifacts->empty());
 239 }
 240
 241 TEST_F(SarifDocumentWriterTest, checkSerializingResultsWithDefaultRuleConfig) {
 242   // GIVEN:
 243   const std::string ExpectedOutput =
 244       R"({"$schema":"https://docs.oasis-open.org/sarif/sarif/v2.1.0/cos02/schemas/sarif-schema-2.1.0.json","runs":[{"artifacts":[],"columnKind":"unicodeCodePoints","results":[{"level":"warning","message":{"text":""},"ruleId":"clang.unittest","ruleIndex":0}],"tool":{"driver":{"fullName":"sarif test runner","informationUri":"https://clang.llvm.org/docs/UsersManual.html","language":"en-US","name":"sarif test","rules":[{"defaultConfiguration":{"enabled":true,"level":"warning","rank":-1},"fullDescription":{"text":"Example rule created during unit tests"},"id":"clang.unittest","name":"clang unit test"}],"version":"1.0.0"}}}],"version":"2.1.0"})";
 245
 246   SarifDocumentWriter Writer{SourceMgr};
 247   const SarifRule &Rule =
 248       SarifRule::create()
 249           .setRuleId("clang.unittest")
 250           .setDescription("Example rule created during unit tests")
 251           .setName("clang unit test");
 252
 253   // WHEN: A run contains a result
 254   Writer.createRun("sarif test", "sarif test runner", "1.0.0");
 255   unsigned RuleIdx = Writer.createRule(Rule);
 256   const SarifResult &Result = SarifResult::create(RuleIdx);
 257   Writer.appendResult(Result);
 258   std::string Output = serializeSarifDocument(Writer.createDocument());
 259
 260   // THEN:
 261   ASSERT_THAT(Output, ::testing::StrEq(ExpectedOutput));
 262 }
 263
 264 TEST_F(SarifDocumentWriterTest, checkSerializingResultsWithCustomRuleConfig) {
 265   // GIVEN:
 266   const std::string ExpectedOutput =
 267       R"({"$schema":"https://docs.oasis-open.org/sarif/sarif/v2.1.0/cos02/schemas/sarif-schema-2.1.0.json","runs":[{"artifacts":[],"columnKind":"unicodeCodePoints","results":[{"level":"error","message":{"text":""},"ruleId":"clang.unittest","ruleIndex":0}],"tool":{"driver":{"fullName":"sarif test runner","informationUri":"https://clang.llvm.org/docs/UsersManual.html","language":"en-US","name":"sarif test","rules":[{"defaultConfiguration":{"enabled":true,"level":"error","rank":35.5},"fullDescription":{"text":"Example rule created during unit tests"},"id":"clang.unittest","name":"clang unit test"}],"version":"1.0.0"}}}],"version":"2.1.0"})";
 268
 269   SarifDocumentWriter Writer{SourceMgr};
 270   const SarifRule &Rule =
 271       SarifRule::create()
 272           .setRuleId("clang.unittest")
 273           .setDescription("Example rule created during unit tests")
 274           .setName("clang unit test")
 275           .setDefaultConfiguration(SarifReportingConfiguration::create()
 276                                        .setLevel(SarifResultLevel::Error)
 277                                        .setRank(35.5));
 278
 279   // WHEN: A run contains a result
 280   Writer.createRun("sarif test", "sarif test runner", "1.0.0");
 281   unsigned RuleIdx = Writer.createRule(Rule);
 282   const SarifResult &Result = SarifResult::create(RuleIdx);
 283   Writer.appendResult(Result);
 284   std::string Output = serializeSarifDocument(Writer.createDocument());
 285
 286   // THEN:
 287   ASSERT_THAT(Output, ::testing::StrEq(ExpectedOutput));
 288 }
 289
 290 // Check that serializing artifacts from results produces valid SARIF
 291 TEST_F(SarifDocumentWriterTest, checkSerializingArtifacts) {
 292   // GIVEN:
 293   const std::string ExpectedOutput =
 294       R"({"$schema":"https://docs.oasis-open.org/sarif/sarif/v2.1.0/cos02/schemas/sarif-schema-2.1.0.json","runs":[{"artifacts":[{"length":40,"location":{"index":0,"uri":"file:///main.cpp"},"mimeType":"text/plain","roles":["resultFile"]}],"columnKind":"unicodeCodePoints","results":[{"level":"error","locations":[{"physicalLocation":{"artifactLocation":{"index":0,"uri":"file:///main.cpp"},"region":{"endColumn":14,"startColumn":14,"startLine":3}}}],"message":{"text":"expected ';' after top level declarator"},"ruleId":"clang.unittest","ruleIndex":0}],"tool":{"driver":{"fullName":"sarif test runner","informationUri":"https://clang.llvm.org/docs/UsersManual.html","language":"en-US","name":"sarif test","rules":[{"defaultConfiguration":{"enabled":true,"level":"warning","rank":-1},"fullDescription":{"text":"Example rule created during unit tests"},"id":"clang.unittest","name":"clang unit test"}],"version":"1.0.0"}}}],"version":"2.1.0"})";
 295
 296   SarifDocumentWriter Writer{SourceMgr};
 297   const SarifRule &Rule =
 298       SarifRule::create()
 299           .setRuleId("clang.unittest")
 300           .setDescription("Example rule created during unit tests")
 301           .setName("clang unit test");
 302
 303   // WHEN: A result is added with valid source locations for its diagnostics
 304   Writer.createRun("sarif test", "sarif test runner", "1.0.0");
 305   unsigned RuleIdx = Writer.createRule(Rule);
 306
 307   llvm::SmallVector<CharSourceRange, 1> DiagLocs;
 308   const char *SourceText = "int foo = 0;\n"
 309                            "int bar = 1;\n"
 310                            "float x = 0.0\n";
 311
 312   FileID MainFileID =
 313       registerSource("/main.cpp", SourceText, /* IsMainFile = */ true);
 314   CharSourceRange SourceCSR =
 315       getFakeCharSourceRange(MainFileID, {3, 14}, {3, 14});
 316
 317   DiagLocs.push_back(SourceCSR);
 318
 319   const SarifResult &Result =
 320       SarifResult::create(RuleIdx)
 321           .setLocations(DiagLocs)
 322           .setDiagnosticMessage("expected ';' after top level declarator")
 323           .setDiagnosticLevel(SarifResultLevel::Error);
 324   Writer.appendResult(Result);
 325   std::string Output = serializeSarifDocument(Writer.createDocument());
 326
 327   // THEN: Assert that the serialized SARIF is as expected
 328   ASSERT_THAT(Output, ::testing::StrEq(ExpectedOutput));
 329 }
 330
 331 TEST_F(SarifDocumentWriterTest, checkSerializingCodeflows) {
 332   // GIVEN:
 333   const std::string ExpectedOutput =
 334       R"({"$schema":"https://docs.oasis-open.org/sarif/sarif/v2.1.0/cos02/schemas/sarif-schema-2.1.0.json","runs":[{"artifacts":[{"length":41,"location":{"index":0,"uri":"file:///main.cpp"},"mimeType":"text/plain","roles":["resultFile"]},{"length":27,"location":{"index":1,"uri":"file:///test-header-1.h"},"mimeType":"text/plain","roles":["resultFile"]},{"length":30,"location":{"index":2,"uri":"file:///test-header-2.h"},"mimeType":"text/plain","roles":["resultFile"]},{"length":28,"location":{"index":3,"uri":"file:///test-header-3.h"},"mimeType":"text/plain","roles":["resultFile"]}],"columnKind":"unicodeCodePoints","results":[{"codeFlows":[{"threadFlows":[{"locations":[{"importance":"essential","location":{"message":{"text":"Message #1"},"physicalLocation":{"artifactLocation":{"index":1,"uri":"file:///test-header-1.h"},"region":{"endColumn":8,"endLine":2,"startColumn":1,"startLine":1}}}},{"importance":"important","location":{"message":{"text":"Message #2"},"physicalLocation":{"artifactLocation":{"index":2,"uri":"file:///test-header-2.h"},"region":{"endColumn":8,"endLine":2,"startColumn":1,"startLine":1}}}},{"importance":"unimportant","location":{"message":{"text":"Message #3"},"physicalLocation":{"artifactLocation":{"index":3,"uri":"file:///test-header-3.h"},"region":{"endColumn":8,"endLine":2,"startColumn":1,"startLine":1}}}}]}]}],"level":"warning","locations":[{"physicalLocation":{"artifactLocation":{"index":0,"uri":"file:///main.cpp"},"region":{"endColumn":8,"endLine":2,"startColumn":5,"startLine":2}}}],"message":{"text":"Redefinition of 'foo'"},"ruleId":"clang.unittest","ruleIndex":0}],"tool":{"driver":{"fullName":"sarif test runner","informationUri":"https://clang.llvm.org/docs/UsersManual.html","language":"en-US","name":"sarif test","rules":[{"defaultConfiguration":{"enabled":true,"level":"warning","rank":-1},"fullDescription":{"text":"Example rule created during unit tests"},"id":"clang.unittest","name":"clang unit test"}],"version":"1.0.0"}}}],"version":"2.1.0"})";
 335
 336   const char *SourceText = "int foo = 0;\n"
 337                            "int foo = 1;\n"
 338                            "float x = 0.0;\n";
 339   FileID MainFileID =
 340       registerSource("/main.cpp", SourceText, /* IsMainFile = */ true);
 341   CharSourceRange DiagLoc{getFakeCharSourceRange(MainFileID, {2, 5}, {2, 8})};
 342
 343   SarifDocumentWriter Writer{SourceMgr};
 344   const SarifRule &Rule =
 345       SarifRule::create()
 346           .setRuleId("clang.unittest")
 347           .setDescription("Example rule created during unit tests")
 348           .setName("clang unit test");
 349
 350   constexpr unsigned int NumCases = 3;
 351   llvm::SmallVector<ThreadFlow, NumCases> Threadflows;
 352   const char *HeaderTexts[NumCases]{("#pragma once\n"
 353                                      "#include <foo>"),
 354                                     ("#ifndef FOO\n"
 355                                      "#define FOO\n"
 356                                      "#endif"),
 357                                     ("#ifdef FOO\n"
 358                                      "#undef FOO\n"
 359                                      "#endif")};
 360   const char *HeaderNames[NumCases]{"/test-header-1.h", "/test-header-2.h",
 361                                     "/test-header-3.h"};
 362   ThreadFlowImportance Importances[NumCases]{ThreadFlowImportance::Essential,
 363                                              ThreadFlowImportance::Important,
 364                                              ThreadFlowImportance::Unimportant};
 365   for (size_t Idx = 0; Idx != NumCases; ++Idx) {
 366     FileID FID = registerSource(HeaderNames[Idx], HeaderTexts[Idx]);
 367     CharSourceRange &&CSR = getFakeCharSourceRange(FID, {1, 1}, {2, 8});
 368     std::string Message = llvm::formatv("Message #{0}", Idx + 1);
 369     ThreadFlow Item = ThreadFlow::create()
 370                           .setRange(CSR)
 371                           .setImportance(Importances[Idx])
 372                           .setMessage(Message);
 373     Threadflows.push_back(Item);
 374   }
 375
 376   // WHEN: A result containing code flows and diagnostic locations is added
 377   Writer.createRun("sarif test", "sarif test runner", "1.0.0");
 378   unsigned RuleIdx = Writer.createRule(Rule);
 379   const SarifResult &Result =
 380       SarifResult::create(RuleIdx)
 381           .setLocations({DiagLoc})
 382           .setDiagnosticMessage("Redefinition of 'foo'")
 383           .setThreadFlows(Threadflows)
 384           .setDiagnosticLevel(SarifResultLevel::Warning);
 385   Writer.appendResult(Result);
 386   std::string Output = serializeSarifDocument(Writer.createDocument());
 387
 388   // THEN: Assert that the serialized SARIF is as expected
 389   ASSERT_THAT(Output, ::testing::StrEq(ExpectedOutput));
 390 }
 391
 392 } // namespace