libc/benchmarks/automemcpy/unittests/ResultAnalyzerTest.cpp

   1 //===--  Automemcpy Json Results Analyzer Test ----------------------------===//
   2 //
   3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
   4 // See https://llvm.org/LICENSE.txt for license information.
   5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
   6 //
   7 //===----------------------------------------------------------------------===//
   8
   9 #include "automemcpy/ResultAnalyzer.h"
  10 #include "gmock/gmock.h"
  11 #include "gtest/gtest.h"
  12
  13 using testing::DoubleNear;
  14 using testing::ElementsAre;
  15 using testing::Pair;
  16 using testing::SizeIs;
  17
  18 namespace llvm {
  19 namespace automemcpy {
  20 namespace {
  21
  22 TEST(AutomemcpyJsonResultsAnalyzer, getThroughputsOneSample) {
  23   static constexpr FunctionId Foo1 = {"memcpy1", FunctionType::MEMCPY};
  24   static constexpr DistributionId DistA = {{"A"}};
  25   static constexpr SampleId Id = {Foo1, DistA};
  26   static constexpr Sample kSamples[] = {
  27       Sample{Id, SampleType::ITERATION, 4},
  28       Sample{Id, SampleType::AGGREGATE, -1}, // Aggegates gets discarded
  29   };
  30
  31   const std::vector<FunctionData> Data = getThroughputs(kSamples);
  32   EXPECT_THAT(Data, SizeIs(1));
  33   EXPECT_THAT(Data[0].Id, Foo1);
  34   EXPECT_THAT(Data[0].PerDistributionData, SizeIs(1));
  35   // A single value is provided.
  36   const auto &DistributionData = Data[0].PerDistributionData.lookup(DistA.Name);
  37   EXPECT_THAT(DistributionData.BytesPerSecondMedian, 4);
  38   EXPECT_THAT(DistributionData.BytesPerSecondMean, 4);
  39   EXPECT_THAT(DistributionData.BytesPerSecondVariance, 0);
  40 }
  41
  42 TEST(AutomemcpyJsonResultsAnalyzer, getThroughputsManySamplesSameBucket) {
  43   static constexpr FunctionId Foo1 = {"memcpy1", FunctionType::MEMCPY};
  44   static constexpr DistributionId DistA = {{"A"}};
  45   static constexpr SampleId Id = {Foo1, DistA};
  46   static constexpr Sample kSamples[] = {Sample{Id, SampleType::ITERATION, 4},
  47                                         Sample{Id, SampleType::ITERATION, 5},
  48                                         Sample{Id, SampleType::ITERATION, 5}};
  49
  50   const std::vector<FunctionData> Data = getThroughputs(kSamples);
  51   EXPECT_THAT(Data, SizeIs(1));
  52   EXPECT_THAT(Data[0].Id, Foo1);
  53   EXPECT_THAT(Data[0].PerDistributionData, SizeIs(1));
  54   // When multiple values are provided we pick the median one (here median of 4,
  55   // 5, 5).
  56   const auto &DistributionData = Data[0].PerDistributionData.lookup(DistA.Name);
  57   EXPECT_THAT(DistributionData.BytesPerSecondMedian, 5);
  58   EXPECT_THAT(DistributionData.BytesPerSecondMean, DoubleNear(4.6, 0.1));
  59   EXPECT_THAT(DistributionData.BytesPerSecondVariance, DoubleNear(0.33, 0.01));
  60 }
  61
  62 TEST(AutomemcpyJsonResultsAnalyzer, getThroughputsServeralFunctionAndDist) {
  63   static constexpr FunctionId Foo1 = {"memcpy1", FunctionType::MEMCPY};
  64   static constexpr DistributionId DistA = {{"A"}};
  65   static constexpr FunctionId Foo2 = {"memcpy2", FunctionType::MEMCPY};
  66   static constexpr DistributionId DistB = {{"B"}};
  67   static constexpr Sample kSamples[] = {
  68       Sample{{Foo1, DistA}, SampleType::ITERATION, 1},
  69       Sample{{Foo1, DistB}, SampleType::ITERATION, 2},
  70       Sample{{Foo2, DistA}, SampleType::ITERATION, 3},
  71       Sample{{Foo2, DistB}, SampleType::ITERATION, 4}};
  72   // Data is aggregated per function.
  73   const std::vector<FunctionData> Data = getThroughputs(kSamples);
  74   EXPECT_THAT(Data, SizeIs(2)); // 2 functions Foo1 and Foo2.
  75   // Each function has data for both distributions DistA and DistB.
  76   EXPECT_THAT(Data[0].PerDistributionData, SizeIs(2));
  77   EXPECT_THAT(Data[1].PerDistributionData, SizeIs(2));
  78 }
  79
  80 TEST(AutomemcpyJsonResultsAnalyzer, getScore) {
  81   static constexpr FunctionId Foo1 = {"memcpy1", FunctionType::MEMCPY};
  82   static constexpr FunctionId Foo2 = {"memcpy2", FunctionType::MEMCPY};
  83   static constexpr FunctionId Foo3 = {"memcpy3", FunctionType::MEMCPY};
  84   static constexpr DistributionId Dist = {{"A"}};
  85   static constexpr Sample kSamples[] = {
  86       Sample{{Foo1, Dist}, SampleType::ITERATION, 1},
  87       Sample{{Foo2, Dist}, SampleType::ITERATION, 2},
  88       Sample{{Foo3, Dist}, SampleType::ITERATION, 3}};
  89
  90   // Data is aggregated per function.
  91   std::vector<FunctionData> Data = getThroughputs(kSamples);
  92
  93   // Sort Data by function name so we can test them.
  94   std::sort(
  95       Data.begin(), Data.end(),
  96       [](const FunctionData &A, const FunctionData &B) { return A.Id < B.Id; });
  97
  98   EXPECT_THAT(Data[0].Id, Foo1);
  99   EXPECT_THAT(Data[0].PerDistributionData.lookup("A").BytesPerSecondMedian, 1);
 100   EXPECT_THAT(Data[1].Id, Foo2);
 101   EXPECT_THAT(Data[1].PerDistributionData.lookup("A").BytesPerSecondMedian, 2);
 102   EXPECT_THAT(Data[2].Id, Foo3);
 103   EXPECT_THAT(Data[2].PerDistributionData.lookup("A").BytesPerSecondMedian, 3);
 104
 105   // Normalizes throughput per distribution.
 106   fillScores(Data);
 107   EXPECT_THAT(Data[0].PerDistributionData.lookup("A").Score, 0);
 108   EXPECT_THAT(Data[1].PerDistributionData.lookup("A").Score, 0.5);
 109   EXPECT_THAT(Data[2].PerDistributionData.lookup("A").Score, 1);
 110 }
 111
 112 TEST(AutomemcpyJsonResultsAnalyzer, castVotes) {
 113   static constexpr double kAbsErr = 0.01;
 114
 115   static constexpr FunctionId Foo1 = {"memcpy1", FunctionType::MEMCPY};
 116   static constexpr FunctionId Foo2 = {"memcpy2", FunctionType::MEMCPY};
 117   static constexpr FunctionId Foo3 = {"memcpy3", FunctionType::MEMCPY};
 118   static constexpr DistributionId DistA = {{"A"}};
 119   static constexpr DistributionId DistB = {{"B"}};
 120   static constexpr Sample kSamples[] = {
 121       Sample{{Foo1, DistA}, SampleType::ITERATION, 0},
 122       Sample{{Foo1, DistB}, SampleType::ITERATION, 30},
 123       Sample{{Foo2, DistA}, SampleType::ITERATION, 1},
 124       Sample{{Foo2, DistB}, SampleType::ITERATION, 100},
 125       Sample{{Foo3, DistA}, SampleType::ITERATION, 7},
 126       Sample{{Foo3, DistB}, SampleType::ITERATION, 100},
 127   };
 128
 129   // DistA Thoughput ranges from 0 to 7.
 130   // DistB Thoughput ranges from 30 to 100.
 131
 132   // Data is aggregated per function.
 133   std::vector<FunctionData> Data = getThroughputs(kSamples);
 134
 135   // Sort Data by function name so we can test them.
 136   std::sort(
 137       Data.begin(), Data.end(),
 138       [](const FunctionData &A, const FunctionData &B) { return A.Id < B.Id; });
 139
 140   // Normalizes throughput per distribution.
 141   fillScores(Data);
 142
 143   // Cast votes
 144   castVotes(Data);
 145
 146   EXPECT_THAT(Data[0].Id, Foo1);
 147   EXPECT_THAT(Data[1].Id, Foo2);
 148   EXPECT_THAT(Data[2].Id, Foo3);
 149
 150   const auto GetDistData = [&Data](size_t Index, StringRef Name) {
 151     return Data[Index].PerDistributionData.lookup(Name);
 152   };
 153
 154   // Distribution A
 155   // Throughput is 0, 1 and 7, so normalized scores are 0, 1/7 and 1.
 156   EXPECT_THAT(GetDistData(0, "A").Score, DoubleNear(0, kAbsErr));
 157   EXPECT_THAT(GetDistData(1, "A").Score, DoubleNear(1. / 7, kAbsErr));
 158   EXPECT_THAT(GetDistData(2, "A").Score, DoubleNear(1, kAbsErr));
 159   // which are turned into grades BAD,  MEDIOCRE and EXCELLENT.
 160   EXPECT_THAT(GetDistData(0, "A").Grade, Grade::BAD);
 161   EXPECT_THAT(GetDistData(1, "A").Grade, Grade::MEDIOCRE);
 162   EXPECT_THAT(GetDistData(2, "A").Grade, Grade::EXCELLENT);
 163
 164   // Distribution B
 165   // Throughput is 30, 100 and 100, so normalized scores are 0, 1 and 1.
 166   EXPECT_THAT(GetDistData(0, "B").Score, DoubleNear(0, kAbsErr));
 167   EXPECT_THAT(GetDistData(1, "B").Score, DoubleNear(1, kAbsErr));
 168   EXPECT_THAT(GetDistData(2, "B").Score, DoubleNear(1, kAbsErr));
 169   // which are turned into grades BAD, EXCELLENT and EXCELLENT.
 170   EXPECT_THAT(GetDistData(0, "B").Grade, Grade::BAD);
 171   EXPECT_THAT(GetDistData(1, "B").Grade, Grade::EXCELLENT);
 172   EXPECT_THAT(GetDistData(2, "B").Grade, Grade::EXCELLENT);
 173
 174   // Now looking from the functions point of view.
 175   EXPECT_THAT(Data[0].ScoresGeoMean, DoubleNear(0, kAbsErr));
 176   EXPECT_THAT(Data[1].ScoresGeoMean, DoubleNear(1. * (1. / 7), kAbsErr));
 177   EXPECT_THAT(Data[2].ScoresGeoMean, DoubleNear(1, kAbsErr));
 178
 179   // Note the array is indexed by GradeEnum values (EXCELLENT=0 / BAD = 6)
 180   EXPECT_THAT(Data[0].GradeHisto, ElementsAre(0, 0, 0, 0, 0, 0, 2));
 181   EXPECT_THAT(Data[1].GradeHisto, ElementsAre(1, 0, 0, 0, 0, 1, 0));
 182   EXPECT_THAT(Data[2].GradeHisto, ElementsAre(2, 0, 0, 0, 0, 0, 0));
 183
 184   EXPECT_THAT(Data[0].FinalGrade, Grade::BAD);
 185   EXPECT_THAT(Data[1].FinalGrade, Grade::MEDIOCRE);
 186   EXPECT_THAT(Data[2].FinalGrade, Grade::EXCELLENT);
 187 }
 188
 189 } // namespace
 190 } // namespace automemcpy
 191 } // namespace llvm