gpu/config/gpu_test_expectations_parser_unittest.cc

   1 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
   2 // Use of this source code is governed by a BSD-style license that can be
   3 // found in the LICENSE file.
   4
   5 #include "base/logging.h"
   6 #include "gpu/config/gpu_test_expectations_parser.h"
   7 #include "testing/gtest/include/gtest/gtest.h"
   8
   9 namespace gpu {
  10
  11 class GPUTestExpectationsParserTest : public testing::Test {
  12  public:
  13   GPUTestExpectationsParserTest() { }
  14
  15   ~GPUTestExpectationsParserTest() override {}
  16
  17   const GPUTestBotConfig& bot_config() const {
  18     return bot_config_;
  19   }
  20
  21  protected:
  22   void SetUp() override {
  23     bot_config_.set_os(GPUTestConfig::kOsWin7);
  24     bot_config_.set_build_type(GPUTestConfig::kBuildTypeRelease);
  25     bot_config_.AddGPUVendor(0x10de);
  26     bot_config_.set_gpu_device_id(0x0640);
  27     ASSERT_TRUE(bot_config_.IsValid());
  28   }
  29
  30   void TearDown() override {}
  31
  32  private:
  33   GPUTestBotConfig bot_config_;
  34 };
  35
  36 TEST_F(GPUTestExpectationsParserTest, CommentOnly) {
  37   const std::string text =
  38       "  \n"
  39       "// This is just some comment\n"
  40       "";
  41   GPUTestExpectationsParser parser;
  42   EXPECT_TRUE(parser.LoadTestExpectations(text));
  43   EXPECT_EQ(0u, parser.GetErrorMessages().size());
  44   EXPECT_EQ(GPUTestExpectationsParser::kGpuTestPass,
  45             parser.GetTestExpectation("some_test", bot_config()));
  46 }
  47
  48 TEST_F(GPUTestExpectationsParserTest, ValidFullEntry) {
  49   const std::string text =
  50       "BUG12345 WIN7 RELEASE NVIDIA 0x0640 : MyTest = FAIL";
  51
  52   GPUTestExpectationsParser parser;
  53   EXPECT_TRUE(parser.LoadTestExpectations(text));
  54   EXPECT_EQ(0u, parser.GetErrorMessages().size());
  55   EXPECT_EQ(GPUTestExpectationsParser::kGpuTestFail,
  56             parser.GetTestExpectation("MyTest", bot_config()));
  57 }
  58
  59 TEST_F(GPUTestExpectationsParserTest, ValidPartialEntry) {
  60   const std::string text =
  61       "BUG12345 WIN NVIDIA : MyTest = TIMEOUT";
  62
  63   GPUTestExpectationsParser parser;
  64   EXPECT_TRUE(parser.LoadTestExpectations(text));
  65   EXPECT_EQ(0u, parser.GetErrorMessages().size());
  66   EXPECT_EQ(GPUTestExpectationsParser::kGpuTestTimeout,
  67             parser.GetTestExpectation("MyTest", bot_config()));
  68 }
  69
  70 TEST_F(GPUTestExpectationsParserTest, ValidUnrelatedOsEntry) {
  71   const std::string text =
  72       "BUG12345 LEOPARD : MyTest = TIMEOUT";
  73
  74   GPUTestExpectationsParser parser;
  75   EXPECT_TRUE(parser.LoadTestExpectations(text));
  76   EXPECT_EQ(0u, parser.GetErrorMessages().size());
  77   EXPECT_EQ(GPUTestExpectationsParser::kGpuTestPass,
  78             parser.GetTestExpectation("MyTest", bot_config()));
  79 }
  80
  81 TEST_F(GPUTestExpectationsParserTest, ValidUnrelatedTestEntry) {
  82   const std::string text =
  83       "BUG12345 WIN7 RELEASE NVIDIA 0x0640 : AnotherTest = FAIL";
  84
  85   GPUTestExpectationsParser parser;
  86   EXPECT_TRUE(parser.LoadTestExpectations(text));
  87   EXPECT_EQ(0u, parser.GetErrorMessages().size());
  88   EXPECT_EQ(GPUTestExpectationsParser::kGpuTestPass,
  89             parser.GetTestExpectation("MyTest", bot_config()));
  90 }
  91
  92 TEST_F(GPUTestExpectationsParserTest, AllModifiers) {
  93   const std::string text =
  94       "BUG12345 XP VISTA WIN7 WIN8 LEOPARD SNOWLEOPARD LION MOUNTAINLION "
  95       "MAVERICKS LINUX CHROMEOS ANDROID "
  96       "NVIDIA INTEL AMD VMWARE RELEASE DEBUG : MyTest = "
  97       "PASS FAIL FLAKY TIMEOUT SKIP";
  98
  99   GPUTestExpectationsParser parser;
 100   EXPECT_TRUE(parser.LoadTestExpectations(text));
 101   EXPECT_EQ(0u, parser.GetErrorMessages().size());
 102   EXPECT_EQ(GPUTestExpectationsParser::kGpuTestPass |
 103             GPUTestExpectationsParser::kGpuTestFail |
 104             GPUTestExpectationsParser::kGpuTestFlaky |
 105             GPUTestExpectationsParser::kGpuTestTimeout |
 106             GPUTestExpectationsParser::kGpuTestSkip,
 107             parser.GetTestExpectation("MyTest", bot_config()));
 108 }
 109
 110 TEST_F(GPUTestExpectationsParserTest, DuplicateModifiers) {
 111   const std::string text =
 112       "BUG12345 WIN7 WIN7 RELEASE NVIDIA 0x0640 : MyTest = FAIL";
 113
 114   GPUTestExpectationsParser parser;
 115   EXPECT_FALSE(parser.LoadTestExpectations(text));
 116   EXPECT_NE(0u, parser.GetErrorMessages().size());
 117 }
 118
 119 TEST_F(GPUTestExpectationsParserTest, AllModifiersLowerCase) {
 120   const std::string text =
 121       "BUG12345 xp vista win7 leopard snowleopard lion linux chromeos android "
 122       "nvidia intel amd vmware release debug : MyTest = "
 123       "pass fail flaky timeout skip";
 124
 125   GPUTestExpectationsParser parser;
 126   EXPECT_TRUE(parser.LoadTestExpectations(text));
 127   EXPECT_EQ(0u, parser.GetErrorMessages().size());
 128   EXPECT_EQ(GPUTestExpectationsParser::kGpuTestPass |
 129             GPUTestExpectationsParser::kGpuTestFail |
 130             GPUTestExpectationsParser::kGpuTestFlaky |
 131             GPUTestExpectationsParser::kGpuTestTimeout |
 132             GPUTestExpectationsParser::kGpuTestSkip,
 133             parser.GetTestExpectation("MyTest", bot_config()));
 134 }
 135
 136 TEST_F(GPUTestExpectationsParserTest, MissingColon) {
 137   const std::string text =
 138       "BUG12345 XP MyTest = FAIL";
 139
 140   GPUTestExpectationsParser parser;
 141   EXPECT_FALSE(parser.LoadTestExpectations(text));
 142   EXPECT_NE(0u, parser.GetErrorMessages().size());
 143 }
 144
 145 TEST_F(GPUTestExpectationsParserTest, MissingEqual) {
 146   const std::string text =
 147       "BUG12345 XP : MyTest FAIL";
 148
 149   GPUTestExpectationsParser parser;
 150   EXPECT_FALSE(parser.LoadTestExpectations(text));
 151   EXPECT_NE(0u, parser.GetErrorMessages().size());
 152 }
 153
 154 TEST_F(GPUTestExpectationsParserTest, IllegalModifier) {
 155   const std::string text =
 156       "BUG12345 XP XXX : MyTest = FAIL";
 157
 158   GPUTestExpectationsParser parser;
 159   EXPECT_FALSE(parser.LoadTestExpectations(text));
 160   EXPECT_NE(0u, parser.GetErrorMessages().size());
 161 }
 162
 163 TEST_F(GPUTestExpectationsParserTest, OsConflicts) {
 164   const std::string text =
 165       "BUG12345 XP WIN : MyTest = FAIL";
 166
 167   GPUTestExpectationsParser parser;
 168   EXPECT_FALSE(parser.LoadTestExpectations(text));
 169   EXPECT_NE(0u, parser.GetErrorMessages().size());
 170 }
 171
 172 TEST_F(GPUTestExpectationsParserTest, InvalidModifierCombination) {
 173   const std::string text =
 174       "BUG12345 XP NVIDIA INTEL 0x0640 : MyTest = FAIL";
 175
 176   GPUTestExpectationsParser parser;
 177   EXPECT_FALSE(parser.LoadTestExpectations(text));
 178   EXPECT_NE(0u, parser.GetErrorMessages().size());
 179 }
 180
 181 TEST_F(GPUTestExpectationsParserTest, BadGpuDeviceID) {
 182   const std::string text =
 183       "BUG12345 XP NVIDIA 0xU07X : MyTest = FAIL";
 184
 185   GPUTestExpectationsParser parser;
 186   EXPECT_FALSE(parser.LoadTestExpectations(text));
 187   EXPECT_NE(0u, parser.GetErrorMessages().size());
 188 }
 189
 190 TEST_F(GPUTestExpectationsParserTest, MoreThanOneGpuDeviceID) {
 191   const std::string text =
 192       "BUG12345 XP NVIDIA 0x0640 0x0641 : MyTest = FAIL";
 193
 194   GPUTestExpectationsParser parser;
 195   EXPECT_FALSE(parser.LoadTestExpectations(text));
 196   EXPECT_NE(0u, parser.GetErrorMessages().size());
 197 }
 198
 199 TEST_F(GPUTestExpectationsParserTest, MultipleEntriesConflicts) {
 200   const std::string text =
 201       "BUG12345 WIN7 RELEASE NVIDIA 0x0640 : MyTest = FAIL\n"
 202       "BUG12345 WIN : MyTest = FAIL";
 203
 204   GPUTestExpectationsParser parser;
 205   EXPECT_FALSE(parser.LoadTestExpectations(text));
 206   EXPECT_NE(0u, parser.GetErrorMessages().size());
 207 }
 208
 209 TEST_F(GPUTestExpectationsParserTest, MultipleTests) {
 210   const std::string text =
 211       "BUG12345 WIN7 RELEASE NVIDIA 0x0640 : MyTest = FAIL\n"
 212       "BUG12345 WIN : AnotherTest = FAIL";
 213
 214   GPUTestExpectationsParser parser;
 215   EXPECT_TRUE(parser.LoadTestExpectations(text));
 216   EXPECT_EQ(0u, parser.GetErrorMessages().size());
 217 }
 218
 219 TEST_F(GPUTestExpectationsParserTest, ValidMultipleEntries) {
 220   const std::string text =
 221       "BUG12345 WIN7 RELEASE NVIDIA 0x0640 : MyTest = FAIL\n"
 222       "BUG12345 LINUX : MyTest = TIMEOUT";
 223
 224   GPUTestExpectationsParser parser;
 225   EXPECT_TRUE(parser.LoadTestExpectations(text));
 226   EXPECT_EQ(0u, parser.GetErrorMessages().size());
 227   EXPECT_EQ(GPUTestExpectationsParser::kGpuTestFail,
 228             parser.GetTestExpectation("MyTest", bot_config()));
 229 }
 230
 231 TEST_F(GPUTestExpectationsParserTest, StarMatching) {
 232   const std::string text =
 233       "BUG12345 WIN7 RELEASE NVIDIA 0x0640 : MyTest* = FAIL";
 234
 235   GPUTestExpectationsParser parser;
 236   EXPECT_TRUE(parser.LoadTestExpectations(text));
 237   EXPECT_EQ(0u, parser.GetErrorMessages().size());
 238   EXPECT_EQ(GPUTestExpectationsParser::kGpuTestFail,
 239             parser.GetTestExpectation("MyTest0", bot_config()));
 240   EXPECT_EQ(GPUTestExpectationsParser::kGpuTestPass,
 241             parser.GetTestExpectation("OtherTest", bot_config()));
 242 }
 243
 244 }  // namespace gpu
 245