gpu/config/gpu_test_expectations_parser_unittest.cc

   1 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
   2 // Use of this source code is governed by a BSD-style license that can be
   3 // found in the LICENSE file.
   4
   5 #include "base/format_macros.h"
   6 #include "base/logging.h"
   7 #include "base/strings/stringprintf.h"
   8 #include "gpu/config/gpu_test_expectations_parser.h"
   9 #include "testing/gtest/include/gtest/gtest.h"
  10
  11 namespace gpu {
  12
  13 struct TestOSEntry {
  14   const char* name;
  15   GPUTestConfig::OS os;
  16 };
  17
  18 static const TestOSEntry kOsFamilyWin = { "WIN", GPUTestConfig::kOsWin };
  19 static const TestOSEntry kOsFamilyMac = { "MAC", GPUTestConfig::kOsMac };
  20
  21 static const struct TestOsWithFamily {
  22   TestOSEntry version;
  23   TestOSEntry family;
  24 } kOSVersionsWithFamily[] = {
  25   { { "XP", GPUTestConfig::kOsWinXP }, kOsFamilyWin },
  26   { { "VISTA", GPUTestConfig::kOsWinVista }, kOsFamilyWin },
  27   { { "WIN7", GPUTestConfig::kOsWin7 }, kOsFamilyWin },
  28   { { "WIN8", GPUTestConfig::kOsWin8 }, kOsFamilyWin },
  29   { { "WIN10", GPUTestConfig::kOsWin10 }, kOsFamilyWin },
  30   { { "LEOPARD", GPUTestConfig::kOsMacLeopard }, kOsFamilyMac },
  31   { { "SNOWLEOPARD", GPUTestConfig::kOsMacSnowLeopard }, kOsFamilyMac },
  32   { { "LION", GPUTestConfig::kOsMacLion }, kOsFamilyMac },
  33   { { "MOUNTAINLION", GPUTestConfig::kOsMacMountainLion }, kOsFamilyMac },
  34   { { "MAVERICKS", GPUTestConfig::kOsMacMavericks }, kOsFamilyMac },
  35   { { "YOSEMITE", GPUTestConfig::kOsMacYosemite }, kOsFamilyMac },
  36   { { "LINUX", GPUTestConfig::kOsLinux },
  37     { "LINUX", GPUTestConfig::kOsLinux } },
  38   { { "CHROMEOS", GPUTestConfig::kOsChromeOS },
  39     { "CHROMEOS", GPUTestConfig::kOsChromeOS } },
  40   { { "ANDROID", GPUTestConfig::kOsAndroid },
  41     { "ANDROID", GPUTestConfig::kOsAndroid } }
  42 };
  43
  44 TestOSEntry GetUnrelatedOS(const TestOSEntry& os) {
  45   return (os.os & kOsFamilyWin.os) ? kOsFamilyMac : kOsFamilyWin;
  46 }
  47
  48 // Prints test parameter details.
  49 std::ostream& operator << (std::ostream& out, const TestOsWithFamily& os) {
  50   out << "{ os_name: \"" << os.version.name
  51       << "\", os_flag: " << os.version.os
  52       << ", os_family: \"" << os.family.name
  53       << "\", os_family_flag: " << os.family.os
  54       << " }";
  55   return out;
  56 }
  57
  58 class GPUTestExpectationsParserTest : public testing::Test {
  59  public:
  60   GPUTestExpectationsParserTest() { }
  61
  62   ~GPUTestExpectationsParserTest() override {}
  63
  64   const GPUTestBotConfig& bot_config() const {
  65     return bot_config_;
  66   }
  67
  68  protected:
  69   void SetUp() override {
  70     bot_config_.set_os(GPUTestConfig::kOsWin7);
  71     bot_config_.set_build_type(GPUTestConfig::kBuildTypeRelease);
  72     bot_config_.AddGPUVendor(0x10de);
  73     bot_config_.set_gpu_device_id(0x0640);
  74     bot_config_.set_api(GPUTestConfig::kAPID3D11);
  75     ASSERT_TRUE(bot_config_.IsValid());
  76   }
  77
  78   void TearDown() override {}
  79
  80   void set_os(int32 os) {
  81     bot_config_.set_os(os);
  82     ASSERT_TRUE(bot_config_.IsValid());
  83   }
  84
  85  private:
  86   GPUTestBotConfig bot_config_;
  87 };
  88
  89 class GPUTestExpectationsParserParamTest
  90     : public GPUTestExpectationsParserTest,
  91       public testing::WithParamInterface<TestOsWithFamily> {
  92  public:
  93   GPUTestExpectationsParserParamTest() { }
  94
  95   ~GPUTestExpectationsParserParamTest() override {}
  96
  97  protected:
  98   const GPUTestBotConfig& GetBotConfig() {
  99     set_os(GetParam().version.os);
 100     return bot_config();
 101   }
 102
 103  private:
 104   // Restrict access to bot_config() function.
 105   // GetBotConfig() should be used instead.
 106   using GPUTestExpectationsParserTest::bot_config;
 107 };
 108
 109 TEST_F(GPUTestExpectationsParserTest, CommentOnly) {
 110   const std::string text =
 111       "  \n"
 112       "// This is just some comment\n"
 113       "";
 114   GPUTestExpectationsParser parser;
 115   EXPECT_TRUE(parser.LoadTestExpectations(text));
 116   EXPECT_EQ(0u, parser.GetErrorMessages().size());
 117   EXPECT_EQ(GPUTestExpectationsParser::kGpuTestPass,
 118             parser.GetTestExpectation("some_test", bot_config()));
 119 }
 120
 121 TEST_P(GPUTestExpectationsParserParamTest, ValidFullEntry) {
 122   const std::string text =
 123       base::StringPrintf("BUG12345 %s RELEASE NVIDIA 0x0640 : MyTest = FAIL",
 124                          GetParam().version.name);
 125
 126   GPUTestExpectationsParser parser;
 127   EXPECT_TRUE(parser.LoadTestExpectations(text));
 128   EXPECT_EQ(0u, parser.GetErrorMessages().size());
 129   EXPECT_EQ(GPUTestExpectationsParser::kGpuTestFail,
 130             parser.GetTestExpectation("MyTest", GetBotConfig()));
 131 }
 132
 133 TEST_P(GPUTestExpectationsParserParamTest, ValidPartialEntry) {
 134   const std::string text =
 135       base::StringPrintf("BUG12345 %s NVIDIA : MyTest = TIMEOUT",
 136                          GetParam().family.name);
 137
 138   GPUTestExpectationsParser parser;
 139   EXPECT_TRUE(parser.LoadTestExpectations(text));
 140   EXPECT_EQ(0u, parser.GetErrorMessages().size());
 141   EXPECT_EQ(GPUTestExpectationsParser::kGpuTestTimeout,
 142             parser.GetTestExpectation("MyTest", GetBotConfig()));
 143 }
 144
 145 TEST_P(GPUTestExpectationsParserParamTest, ValidUnrelatedOsEntry) {
 146   const std::string text = base::StringPrintf(
 147       "BUG12345 %s : MyTest = TIMEOUT",
 148       GetUnrelatedOS(GetParam().version).name);
 149
 150   GPUTestExpectationsParser parser;
 151   EXPECT_TRUE(parser.LoadTestExpectations(text));
 152   EXPECT_EQ(0u, parser.GetErrorMessages().size());
 153   EXPECT_EQ(GPUTestExpectationsParser::kGpuTestPass,
 154             parser.GetTestExpectation("MyTest", GetBotConfig()));
 155 }
 156
 157 TEST_F(GPUTestExpectationsParserTest, ValidUnrelatedTestEntry) {
 158   const std::string text =
 159       "BUG12345 WIN7 RELEASE NVIDIA 0x0640 : AnotherTest = FAIL";
 160
 161   GPUTestExpectationsParser parser;
 162   EXPECT_TRUE(parser.LoadTestExpectations(text));
 163   EXPECT_EQ(0u, parser.GetErrorMessages().size());
 164   EXPECT_EQ(GPUTestExpectationsParser::kGpuTestPass,
 165             parser.GetTestExpectation("MyTest", bot_config()));
 166 }
 167
 168 TEST_F(GPUTestExpectationsParserTest, AllModifiers) {
 169   const std::string text =
 170       "BUG12345 XP VISTA WIN7 WIN8 WIN10 LEOPARD SNOWLEOPARD LION MOUNTAINLION "
 171       "MAVERICKS LINUX CHROMEOS ANDROID "
 172       "NVIDIA INTEL AMD VMWARE RELEASE DEBUG : MyTest = "
 173       "PASS FAIL FLAKY TIMEOUT SKIP";
 174
 175   GPUTestExpectationsParser parser;
 176   EXPECT_TRUE(parser.LoadTestExpectations(text));
 177   EXPECT_EQ(0u, parser.GetErrorMessages().size());
 178   EXPECT_EQ(GPUTestExpectationsParser::kGpuTestPass |
 179             GPUTestExpectationsParser::kGpuTestFail |
 180             GPUTestExpectationsParser::kGpuTestFlaky |
 181             GPUTestExpectationsParser::kGpuTestTimeout |
 182             GPUTestExpectationsParser::kGpuTestSkip,
 183             parser.GetTestExpectation("MyTest", bot_config()));
 184 }
 185
 186 TEST_P(GPUTestExpectationsParserParamTest, DuplicateModifiers) {
 187   const std::string text = base::StringPrintf(
 188       "BUG12345 %s %s RELEASE NVIDIA 0x0640 : MyTest = FAIL",
 189       GetParam().version.name,
 190       GetParam().version.name);
 191
 192   GPUTestExpectationsParser parser;
 193   EXPECT_FALSE(parser.LoadTestExpectations(text));
 194   EXPECT_NE(0u, parser.GetErrorMessages().size());
 195 }
 196
 197 TEST_F(GPUTestExpectationsParserTest, AllModifiersLowerCase) {
 198   const std::string text =
 199       "BUG12345 xp vista win7 win8 win10 leopard snowleopard lion linux "
 200       "chromeos android nvidia intel amd vmware release debug : MyTest = "
 201       "pass fail flaky timeout skip";
 202
 203   GPUTestExpectationsParser parser;
 204   EXPECT_TRUE(parser.LoadTestExpectations(text));
 205   EXPECT_EQ(0u, parser.GetErrorMessages().size());
 206   EXPECT_EQ(GPUTestExpectationsParser::kGpuTestPass |
 207             GPUTestExpectationsParser::kGpuTestFail |
 208             GPUTestExpectationsParser::kGpuTestFlaky |
 209             GPUTestExpectationsParser::kGpuTestTimeout |
 210             GPUTestExpectationsParser::kGpuTestSkip,
 211             parser.GetTestExpectation("MyTest", bot_config()));
 212 }
 213
 214 TEST_F(GPUTestExpectationsParserTest, MissingColon) {
 215   const std::string text =
 216       "BUG12345 XP MyTest = FAIL";
 217
 218   GPUTestExpectationsParser parser;
 219   EXPECT_FALSE(parser.LoadTestExpectations(text));
 220   EXPECT_NE(0u, parser.GetErrorMessages().size());
 221 }
 222
 223 TEST_F(GPUTestExpectationsParserTest, MissingEqual) {
 224   const std::string text =
 225       "BUG12345 XP : MyTest FAIL";
 226
 227   GPUTestExpectationsParser parser;
 228   EXPECT_FALSE(parser.LoadTestExpectations(text));
 229   EXPECT_NE(0u, parser.GetErrorMessages().size());
 230 }
 231
 232 TEST_F(GPUTestExpectationsParserTest, IllegalModifier) {
 233   const std::string text =
 234       "BUG12345 XP XXX : MyTest = FAIL";
 235
 236   GPUTestExpectationsParser parser;
 237   EXPECT_FALSE(parser.LoadTestExpectations(text));
 238   EXPECT_NE(0u, parser.GetErrorMessages().size());
 239 }
 240
 241 TEST_P(GPUTestExpectationsParserParamTest, OsConflicts) {
 242   const std::string text = base::StringPrintf("BUG12345 %s %s : MyTest = FAIL",
 243                                               GetParam().version.name,
 244                                               GetParam().family.name);
 245
 246   GPUTestExpectationsParser parser;
 247   EXPECT_FALSE(parser.LoadTestExpectations(text));
 248   EXPECT_NE(0u, parser.GetErrorMessages().size());
 249 }
 250
 251 TEST_F(GPUTestExpectationsParserTest, InvalidModifierCombination) {
 252   const std::string text =
 253       "BUG12345 XP NVIDIA INTEL 0x0640 : MyTest = FAIL";
 254
 255   GPUTestExpectationsParser parser;
 256   EXPECT_FALSE(parser.LoadTestExpectations(text));
 257   EXPECT_NE(0u, parser.GetErrorMessages().size());
 258 }
 259
 260 TEST_F(GPUTestExpectationsParserTest, BadGpuDeviceID) {
 261   const std::string text =
 262       "BUG12345 XP NVIDIA 0xU07X : MyTest = FAIL";
 263
 264   GPUTestExpectationsParser parser;
 265   EXPECT_FALSE(parser.LoadTestExpectations(text));
 266   EXPECT_NE(0u, parser.GetErrorMessages().size());
 267 }
 268
 269 TEST_F(GPUTestExpectationsParserTest, MoreThanOneGpuDeviceID) {
 270   const std::string text =
 271       "BUG12345 XP NVIDIA 0x0640 0x0641 : MyTest = FAIL";
 272
 273   GPUTestExpectationsParser parser;
 274   EXPECT_FALSE(parser.LoadTestExpectations(text));
 275   EXPECT_NE(0u, parser.GetErrorMessages().size());
 276 }
 277
 278 TEST_P(GPUTestExpectationsParserParamTest, MultipleEntriesConflicts) {
 279   const std::string text = base::StringPrintf(
 280       "BUG12345 %s RELEASE NVIDIA 0x0640 : MyTest = FAIL\n"
 281       "BUG12345 %s : MyTest = FAIL",
 282       GetParam().version.name,
 283       GetParam().family.name);
 284
 285   GPUTestExpectationsParser parser;
 286   EXPECT_FALSE(parser.LoadTestExpectations(text));
 287   EXPECT_NE(0u, parser.GetErrorMessages().size());
 288 }
 289
 290 TEST_F(GPUTestExpectationsParserTest, MultipleTests) {
 291   const std::string text =
 292       "BUG12345 WIN7 RELEASE NVIDIA 0x0640 : MyTest = FAIL\n"
 293       "BUG12345 WIN : AnotherTest = FAIL";
 294
 295   GPUTestExpectationsParser parser;
 296   EXPECT_TRUE(parser.LoadTestExpectations(text));
 297   EXPECT_EQ(0u, parser.GetErrorMessages().size());
 298 }
 299
 300 TEST_F(GPUTestExpectationsParserTest, ValidMultipleEntries) {
 301   const std::string text =
 302       "BUG12345 WIN7 RELEASE NVIDIA 0x0640 : MyTest = FAIL\n"
 303       "BUG12345 LINUX : MyTest = TIMEOUT";
 304
 305   GPUTestExpectationsParser parser;
 306   EXPECT_TRUE(parser.LoadTestExpectations(text));
 307   EXPECT_EQ(0u, parser.GetErrorMessages().size());
 308   EXPECT_EQ(GPUTestExpectationsParser::kGpuTestFail,
 309             parser.GetTestExpectation("MyTest", bot_config()));
 310 }
 311
 312 TEST_F(GPUTestExpectationsParserTest, StarMatching) {
 313   const std::string text =
 314       "BUG12345 WIN7 RELEASE NVIDIA 0x0640 : MyTest* = FAIL";
 315
 316   GPUTestExpectationsParser parser;
 317   EXPECT_TRUE(parser.LoadTestExpectations(text));
 318   EXPECT_EQ(0u, parser.GetErrorMessages().size());
 319   EXPECT_EQ(GPUTestExpectationsParser::kGpuTestFail,
 320             parser.GetTestExpectation("MyTest0", bot_config()));
 321   EXPECT_EQ(GPUTestExpectationsParser::kGpuTestPass,
 322             parser.GetTestExpectation("OtherTest", bot_config()));
 323 }
 324
 325 TEST_F(GPUTestExpectationsParserTest, ValidAPI) {
 326   const std::string text =
 327       "BUG12345 WIN7 NVIDIA D3D9 D3D11 OPENGL GLES : MyTest = FAIL";
 328
 329   GPUTestExpectationsParser parser;
 330   EXPECT_TRUE(parser.LoadTestExpectations(text));
 331   EXPECT_EQ(0u, parser.GetErrorMessages().size());
 332   EXPECT_EQ(GPUTestExpectationsParser::kGpuTestFail,
 333             parser.GetTestExpectation("MyTest", bot_config()));
 334 }
 335
 336 TEST_F(GPUTestExpectationsParserTest, MultipleAPIsConflict) {
 337   const std::string text = "BUG12345 WIN7 NVIDIA D3D9 D3D9 : MyTest = FAIL";
 338
 339   GPUTestExpectationsParser parser;
 340   EXPECT_FALSE(parser.LoadTestExpectations(text));
 341   EXPECT_NE(0u, parser.GetErrorMessages().size());
 342 }
 343
 344 INSTANTIATE_TEST_CASE_P(GPUTestExpectationsParser,
 345                         GPUTestExpectationsParserParamTest,
 346                         ::testing::ValuesIn(kOSVersionsWithFamily));
 347
 348 }  // namespace gpu
 349