gpu/config/gpu_test_expectations_parser_unittest.cc

   1 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
   2 // Use of this source code is governed by a BSD-style license that can be
   3 // found in the LICENSE file.
   4
   5 #include "base/format_macros.h"
   6 #include "base/logging.h"
   7 #include "base/strings/stringprintf.h"
   8 #include "gpu/config/gpu_test_expectations_parser.h"
   9 #include "testing/gtest/include/gtest/gtest.h"
  10
  11 namespace gpu {
  12
  13 struct TestOSEntry {
  14   const char* name;
  15   GPUTestConfig::OS os;
  16 };
  17
  18 static const TestOSEntry kOsFamilyWin = { "WIN", GPUTestConfig::kOsWin };
  19 static const TestOSEntry kOsFamilyMac = { "MAC", GPUTestConfig::kOsMac };
  20
  21 static const struct TestOsWithFamily {
  22   TestOSEntry version;
  23   TestOSEntry family;
  24 } kOSVersionsWithFamily[] = {
  25   { { "XP", GPUTestConfig::kOsWinXP }, kOsFamilyWin },
  26   { { "VISTA", GPUTestConfig::kOsWinVista }, kOsFamilyWin },
  27   { { "WIN7", GPUTestConfig::kOsWin7 }, kOsFamilyWin },
  28   { { "WIN8", GPUTestConfig::kOsWin8 }, kOsFamilyWin },
  29   { { "LEOPARD", GPUTestConfig::kOsMacLeopard }, kOsFamilyMac },
  30   { { "SNOWLEOPARD", GPUTestConfig::kOsMacSnowLeopard }, kOsFamilyMac },
  31   { { "LION", GPUTestConfig::kOsMacLion }, kOsFamilyMac },
  32   { { "MOUNTAINLION", GPUTestConfig::kOsMacMountainLion }, kOsFamilyMac },
  33   { { "MAVERICKS", GPUTestConfig::kOsMacMavericks }, kOsFamilyMac },
  34   { { "YOSEMITE", GPUTestConfig::kOsMacYosemite }, kOsFamilyMac },
  35   { { "LINUX", GPUTestConfig::kOsLinux },
  36     { "LINUX", GPUTestConfig::kOsLinux } },
  37   { { "CHROMEOS", GPUTestConfig::kOsChromeOS },
  38     { "CHROMEOS", GPUTestConfig::kOsChromeOS } },
  39   { { "ANDROID", GPUTestConfig::kOsAndroid },
  40     { "ANDROID", GPUTestConfig::kOsAndroid } }
  41 };
  42
  43 TestOSEntry GetUnrelatedOS(const TestOSEntry& os) {
  44   return (os.os & kOsFamilyWin.os) ? kOsFamilyMac : kOsFamilyWin;
  45 }
  46
  47 // Prints test parameter details.
  48 std::ostream& operator << (std::ostream& out, const TestOsWithFamily& os) {
  49   out << "{ os_name: \"" << os.version.name
  50       << "\", os_flag: " << os.version.os
  51       << ", os_family: \"" << os.family.name
  52       << "\", os_family_flag: " << os.family.os
  53       << " }";
  54   return out;
  55 }
  56
  57 class GPUTestExpectationsParserTest : public testing::Test {
  58  public:
  59   GPUTestExpectationsParserTest() { }
  60
  61   ~GPUTestExpectationsParserTest() override {}
  62
  63   const GPUTestBotConfig& bot_config() const {
  64     return bot_config_;
  65   }
  66
  67  protected:
  68   void SetUp() override {
  69     bot_config_.set_os(GPUTestConfig::kOsWin7);
  70     bot_config_.set_build_type(GPUTestConfig::kBuildTypeRelease);
  71     bot_config_.AddGPUVendor(0x10de);
  72     bot_config_.set_gpu_device_id(0x0640);
  73     ASSERT_TRUE(bot_config_.IsValid());
  74   }
  75
  76   void TearDown() override {}
  77
  78   void set_os(int32 os) {
  79     bot_config_.set_os(os);
  80     ASSERT_TRUE(bot_config_.IsValid());
  81   }
  82
  83  private:
  84   GPUTestBotConfig bot_config_;
  85 };
  86
  87 class GPUTestExpectationsParserParamTest
  88     : public GPUTestExpectationsParserTest,
  89       public testing::WithParamInterface<TestOsWithFamily> {
  90  public:
  91   GPUTestExpectationsParserParamTest() { }
  92
  93   ~GPUTestExpectationsParserParamTest() override {}
  94
  95  protected:
  96   const GPUTestBotConfig& GetBotConfig() {
  97     set_os(GetParam().version.os);
  98     return bot_config();
  99   }
 100
 101  private:
 102   // Restrict access to bot_config() function.
 103   // GetBotConfig() should be used instead.
 104   using GPUTestExpectationsParserTest::bot_config;
 105 };
 106
 107 TEST_F(GPUTestExpectationsParserTest, CommentOnly) {
 108   const std::string text =
 109       "  \n"
 110       "// This is just some comment\n"
 111       "";
 112   GPUTestExpectationsParser parser;
 113   EXPECT_TRUE(parser.LoadTestExpectations(text));
 114   EXPECT_EQ(0u, parser.GetErrorMessages().size());
 115   EXPECT_EQ(GPUTestExpectationsParser::kGpuTestPass,
 116             parser.GetTestExpectation("some_test", bot_config()));
 117 }
 118
 119 TEST_P(GPUTestExpectationsParserParamTest, ValidFullEntry) {
 120   const std::string text =
 121       base::StringPrintf("BUG12345 %s RELEASE NVIDIA 0x0640 : MyTest = FAIL",
 122                          GetParam().version.name);
 123
 124   GPUTestExpectationsParser parser;
 125   EXPECT_TRUE(parser.LoadTestExpectations(text));
 126   EXPECT_EQ(0u, parser.GetErrorMessages().size());
 127   EXPECT_EQ(GPUTestExpectationsParser::kGpuTestFail,
 128             parser.GetTestExpectation("MyTest", GetBotConfig()));
 129 }
 130
 131 TEST_P(GPUTestExpectationsParserParamTest, ValidPartialEntry) {
 132   const std::string text =
 133       base::StringPrintf("BUG12345 %s NVIDIA : MyTest = TIMEOUT",
 134                          GetParam().family.name);
 135
 136   GPUTestExpectationsParser parser;
 137   EXPECT_TRUE(parser.LoadTestExpectations(text));
 138   EXPECT_EQ(0u, parser.GetErrorMessages().size());
 139   EXPECT_EQ(GPUTestExpectationsParser::kGpuTestTimeout,
 140             parser.GetTestExpectation("MyTest", GetBotConfig()));
 141 }
 142
 143 TEST_P(GPUTestExpectationsParserParamTest, ValidUnrelatedOsEntry) {
 144   const std::string text = base::StringPrintf(
 145       "BUG12345 %s : MyTest = TIMEOUT",
 146       GetUnrelatedOS(GetParam().version).name);
 147
 148   GPUTestExpectationsParser parser;
 149   EXPECT_TRUE(parser.LoadTestExpectations(text));
 150   EXPECT_EQ(0u, parser.GetErrorMessages().size());
 151   EXPECT_EQ(GPUTestExpectationsParser::kGpuTestPass,
 152             parser.GetTestExpectation("MyTest", GetBotConfig()));
 153 }
 154
 155 TEST_F(GPUTestExpectationsParserTest, ValidUnrelatedTestEntry) {
 156   const std::string text =
 157       "BUG12345 WIN7 RELEASE NVIDIA 0x0640 : AnotherTest = FAIL";
 158
 159   GPUTestExpectationsParser parser;
 160   EXPECT_TRUE(parser.LoadTestExpectations(text));
 161   EXPECT_EQ(0u, parser.GetErrorMessages().size());
 162   EXPECT_EQ(GPUTestExpectationsParser::kGpuTestPass,
 163             parser.GetTestExpectation("MyTest", bot_config()));
 164 }
 165
 166 TEST_F(GPUTestExpectationsParserTest, AllModifiers) {
 167   const std::string text =
 168       "BUG12345 XP VISTA WIN7 WIN8 LEOPARD SNOWLEOPARD LION MOUNTAINLION "
 169       "MAVERICKS LINUX CHROMEOS ANDROID "
 170       "NVIDIA INTEL AMD VMWARE RELEASE DEBUG : MyTest = "
 171       "PASS FAIL FLAKY TIMEOUT SKIP";
 172
 173   GPUTestExpectationsParser parser;
 174   EXPECT_TRUE(parser.LoadTestExpectations(text));
 175   EXPECT_EQ(0u, parser.GetErrorMessages().size());
 176   EXPECT_EQ(GPUTestExpectationsParser::kGpuTestPass |
 177             GPUTestExpectationsParser::kGpuTestFail |
 178             GPUTestExpectationsParser::kGpuTestFlaky |
 179             GPUTestExpectationsParser::kGpuTestTimeout |
 180             GPUTestExpectationsParser::kGpuTestSkip,
 181             parser.GetTestExpectation("MyTest", bot_config()));
 182 }
 183
 184 TEST_P(GPUTestExpectationsParserParamTest, DuplicateModifiers) {
 185   const std::string text = base::StringPrintf(
 186       "BUG12345 %s %s RELEASE NVIDIA 0x0640 : MyTest = FAIL",
 187       GetParam().version.name,
 188       GetParam().version.name);
 189
 190   GPUTestExpectationsParser parser;
 191   EXPECT_FALSE(parser.LoadTestExpectations(text));
 192   EXPECT_NE(0u, parser.GetErrorMessages().size());
 193 }
 194
 195 TEST_F(GPUTestExpectationsParserTest, AllModifiersLowerCase) {
 196   const std::string text =
 197       "BUG12345 xp vista win7 leopard snowleopard lion linux chromeos android "
 198       "nvidia intel amd vmware release debug : MyTest = "
 199       "pass fail flaky timeout skip";
 200
 201   GPUTestExpectationsParser parser;
 202   EXPECT_TRUE(parser.LoadTestExpectations(text));
 203   EXPECT_EQ(0u, parser.GetErrorMessages().size());
 204   EXPECT_EQ(GPUTestExpectationsParser::kGpuTestPass |
 205             GPUTestExpectationsParser::kGpuTestFail |
 206             GPUTestExpectationsParser::kGpuTestFlaky |
 207             GPUTestExpectationsParser::kGpuTestTimeout |
 208             GPUTestExpectationsParser::kGpuTestSkip,
 209             parser.GetTestExpectation("MyTest", bot_config()));
 210 }
 211
 212 TEST_F(GPUTestExpectationsParserTest, MissingColon) {
 213   const std::string text =
 214       "BUG12345 XP MyTest = FAIL";
 215
 216   GPUTestExpectationsParser parser;
 217   EXPECT_FALSE(parser.LoadTestExpectations(text));
 218   EXPECT_NE(0u, parser.GetErrorMessages().size());
 219 }
 220
 221 TEST_F(GPUTestExpectationsParserTest, MissingEqual) {
 222   const std::string text =
 223       "BUG12345 XP : MyTest FAIL";
 224
 225   GPUTestExpectationsParser parser;
 226   EXPECT_FALSE(parser.LoadTestExpectations(text));
 227   EXPECT_NE(0u, parser.GetErrorMessages().size());
 228 }
 229
 230 TEST_F(GPUTestExpectationsParserTest, IllegalModifier) {
 231   const std::string text =
 232       "BUG12345 XP XXX : MyTest = FAIL";
 233
 234   GPUTestExpectationsParser parser;
 235   EXPECT_FALSE(parser.LoadTestExpectations(text));
 236   EXPECT_NE(0u, parser.GetErrorMessages().size());
 237 }
 238
 239 TEST_P(GPUTestExpectationsParserParamTest, OsConflicts) {
 240   const std::string text = base::StringPrintf("BUG12345 %s %s : MyTest = FAIL",
 241                                               GetParam().version.name,
 242                                               GetParam().family.name);
 243
 244   GPUTestExpectationsParser parser;
 245   EXPECT_FALSE(parser.LoadTestExpectations(text));
 246   EXPECT_NE(0u, parser.GetErrorMessages().size());
 247 }
 248
 249 TEST_F(GPUTestExpectationsParserTest, InvalidModifierCombination) {
 250   const std::string text =
 251       "BUG12345 XP NVIDIA INTEL 0x0640 : MyTest = FAIL";
 252
 253   GPUTestExpectationsParser parser;
 254   EXPECT_FALSE(parser.LoadTestExpectations(text));
 255   EXPECT_NE(0u, parser.GetErrorMessages().size());
 256 }
 257
 258 TEST_F(GPUTestExpectationsParserTest, BadGpuDeviceID) {
 259   const std::string text =
 260       "BUG12345 XP NVIDIA 0xU07X : MyTest = FAIL";
 261
 262   GPUTestExpectationsParser parser;
 263   EXPECT_FALSE(parser.LoadTestExpectations(text));
 264   EXPECT_NE(0u, parser.GetErrorMessages().size());
 265 }
 266
 267 TEST_F(GPUTestExpectationsParserTest, MoreThanOneGpuDeviceID) {
 268   const std::string text =
 269       "BUG12345 XP NVIDIA 0x0640 0x0641 : MyTest = FAIL";
 270
 271   GPUTestExpectationsParser parser;
 272   EXPECT_FALSE(parser.LoadTestExpectations(text));
 273   EXPECT_NE(0u, parser.GetErrorMessages().size());
 274 }
 275
 276 TEST_P(GPUTestExpectationsParserParamTest, MultipleEntriesConflicts) {
 277   const std::string text = base::StringPrintf(
 278       "BUG12345 %s RELEASE NVIDIA 0x0640 : MyTest = FAIL\n"
 279       "BUG12345 %s : MyTest = FAIL",
 280       GetParam().version.name,
 281       GetParam().family.name);
 282
 283   GPUTestExpectationsParser parser;
 284   EXPECT_FALSE(parser.LoadTestExpectations(text));
 285   EXPECT_NE(0u, parser.GetErrorMessages().size());
 286 }
 287
 288 TEST_F(GPUTestExpectationsParserTest, MultipleTests) {
 289   const std::string text =
 290       "BUG12345 WIN7 RELEASE NVIDIA 0x0640 : MyTest = FAIL\n"
 291       "BUG12345 WIN : AnotherTest = FAIL";
 292
 293   GPUTestExpectationsParser parser;
 294   EXPECT_TRUE(parser.LoadTestExpectations(text));
 295   EXPECT_EQ(0u, parser.GetErrorMessages().size());
 296 }
 297
 298 TEST_F(GPUTestExpectationsParserTest, ValidMultipleEntries) {
 299   const std::string text =
 300       "BUG12345 WIN7 RELEASE NVIDIA 0x0640 : MyTest = FAIL\n"
 301       "BUG12345 LINUX : MyTest = TIMEOUT";
 302
 303   GPUTestExpectationsParser parser;
 304   EXPECT_TRUE(parser.LoadTestExpectations(text));
 305   EXPECT_EQ(0u, parser.GetErrorMessages().size());
 306   EXPECT_EQ(GPUTestExpectationsParser::kGpuTestFail,
 307             parser.GetTestExpectation("MyTest", bot_config()));
 308 }
 309
 310 TEST_F(GPUTestExpectationsParserTest, StarMatching) {
 311   const std::string text =
 312       "BUG12345 WIN7 RELEASE NVIDIA 0x0640 : MyTest* = FAIL";
 313
 314   GPUTestExpectationsParser parser;
 315   EXPECT_TRUE(parser.LoadTestExpectations(text));
 316   EXPECT_EQ(0u, parser.GetErrorMessages().size());
 317   EXPECT_EQ(GPUTestExpectationsParser::kGpuTestFail,
 318             parser.GetTestExpectation("MyTest0", bot_config()));
 319   EXPECT_EQ(GPUTestExpectationsParser::kGpuTestPass,
 320             parser.GetTestExpectation("OtherTest", bot_config()));
 321 }
 322
 323 INSTANTIATE_TEST_CASE_P(GPUTestExpectationsParser,
 324                         GPUTestExpectationsParserParamTest,
 325                         ::testing::ValuesIn(kOSVersionsWithFamily));
 326
 327 }  // namespace gpu
 328