chrome/test/gpu/gpu_mapsgl_endurance_browsertest.cc

   1 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
   2 // Use of this source code is governed by a BSD-style license that can be
   3 // found in the LICENSE file.
   4
   5 #include <cmath>
   6
   7 #include "base/command_line.h"
   8 #include "base/file_path.h"
   9 #include "base/file_util.h"
  10 #include "base/json/json_reader.h"
  11 #include "base/path_service.h"
  12 #include "chrome/browser/ui/browser.h"
  13 #include "chrome/browser/ui/browser_tabstrip.h"
  14 #include "chrome/browser/ui/browser_window.h"
  15 #include "chrome/browser/ui/window_snapshot/window_snapshot.h"
  16 #include "chrome/common/chrome_paths.h"
  17 #include "chrome/test/base/in_process_browser_test.h"
  18 #include "chrome/test/base/tracing.h"
  19 #include "chrome/test/base/ui_test_utils.h"
  20 #include "content/public/browser/web_contents.h"
  21 #include "content/public/test/browser_test_utils.h"
  22 #include "googleurl/src/gurl.h"
  23 #include "net/base/net_util.h"
  24 #include "testing/gmock/include/gmock/gmock.h"
  25 #include "testing/gtest/include/gtest/gtest.h"
  26 #include "third_party/skia/include/core/SkBitmap.h"
  27 #include "third_party/skia/include/core/SkColor.h"
  28 #include "third_party/skia/include/core/SkPoint.h"
  29 #include "ui/compositor/compositor_setup.h"
  30 #include "ui/gfx/codec/png_codec.h"
  31 #include "ui/gfx/size.h"
  32
  33 using ::testing::AllOf;
  34 using ::testing::Le;
  35 using ::testing::Ge;
  36
  37 // Test fixture for the MapsGL endurance tests.
  38 //
  39 // This runs the MapsGL harness one or more times, and checks the
  40 // value of certain pixels at the end of the run in order to make sure
  41 // that the rendering actually occurred as we expected it to. Which
  42 // pixels are checked, their expected values and tolerances are all
  43 // encoded in a JSON file accompanying the test.
  44 //
  45 // Pass the command line argument --save-test-failures to save the PNG
  46 // of any failing test runs. Currently there is only one test and it
  47 // will write its output to "single-run-basic-output.png" in the
  48 // current working directory.
  49 //
  50 // TODO(kbr): Add more documentation on adding to and modifying these
  51 // tests.
  52 class MapsGLEnduranceTest : public InProcessBrowserTest {
  53  public:
  54   MapsGLEnduranceTest() {
  55   }
  56
  57   virtual void SetUpInProcessBrowserTestFixture() {
  58     InProcessBrowserTest::SetUpInProcessBrowserTestFixture();
  59
  60     ASSERT_TRUE(PathService::Get(chrome::DIR_TEST_DATA, &test_data_dir_));
  61     test_data_dir_ = test_data_dir_.AppendASCII("gpu");
  62
  63     ui::DisableTestCompositor();
  64   }
  65
  66   void RunSingleTest(const gfx::Size& tab_container_size,
  67                      const std::string& url,
  68                      const std::string& json_test_expectations_filename,
  69                      const std::string& failure_filename_prefix) {
  70     std::vector<SinglePixelExpectation> expectations;
  71     FilePath test_expectations_path =
  72         test_data_dir().AppendASCII(json_test_expectations_filename);
  73     if (!ReadTestExpectations(test_expectations_path, &expectations)) {
  74       LOG(ERROR) << "Failed to read test expectations from file "
  75                  << test_expectations_path.value();
  76       return;
  77     }
  78
  79 #if defined(OS_WIN)
  80     ASSERT_TRUE(tracing::BeginTracing("-test_*"));
  81 #endif
  82
  83     ASSERT_TRUE(ui_test_utils::BringBrowserWindowToFront(browser()));
  84     gfx::Rect new_bounds = GetNewTabContainerBounds(tab_container_size);
  85     browser()->window()->SetBounds(new_bounds);
  86
  87     content::DOMMessageQueue message_queue;
  88     ui_test_utils::NavigateToURL(browser(), GURL(url));
  89
  90     // Wait for notification that the test completed.
  91     std::string message;
  92     ASSERT_TRUE(message_queue.WaitForMessage(&message));
  93     message_queue.ClearQueue();
  94     // TODO(kbr): figure out why this is escaped
  95     EXPECT_EQ("\"FINISHED\"", message);
  96
  97     // Take a snapshot of the web page and compare it to the test
  98     // expectations.
  99     SkBitmap bitmap;
 100     ASSERT_TRUE(TabSnapShotToImage(&bitmap));
 101
 102     bool all_pixels_match =
 103         CompareToExpectedResults(bitmap, expectations);
 104
 105     if (!all_pixels_match &&
 106         CommandLine::ForCurrentProcess()->HasSwitch("save-test-failures")) {
 107       std::vector<unsigned char> output;
 108       if (!gfx::PNGCodec::EncodeBGRASkBitmap(bitmap, true, &output)) {
 109         LOG(ERROR) << "Re-encode PNG failed";
 110       } else {
 111         FilePath output_path;
 112         output_path = output_path.AppendASCII(
 113             failure_filename_prefix + "-output.png");
 114         if (file_util::WriteFile(
 115                 output_path,
 116                 reinterpret_cast<char*>(&*output.begin()), output.size()) < 0) {
 117           LOG(ERROR) << "Write PNG to disk failed";
 118         }
 119       }
 120     }
 121
 122 #if defined(OS_WIN)
 123     // For debugging the flaky test, this prints out a trace of what happened on
 124     // failure.
 125     std::string trace_events;
 126     ASSERT_TRUE(tracing::EndTracing(&trace_events));
 127     if (!all_pixels_match)
 128       fprintf(stderr, "\n\nTRACE JSON:\n\n%s\n\n", trace_events.c_str());
 129 #endif
 130   }
 131
 132   const FilePath& test_data_dir() const {
 133     return test_data_dir_;
 134   }
 135
 136  private:
 137   struct SinglePixelExpectation {
 138     SkIPoint location;
 139     SkColor color;
 140     int tolerance;
 141   };
 142
 143   FilePath test_data_dir_;
 144
 145   // Test expectations are expressed in the following JSON format:
 146   // [
 147   //   { "location": [ 25, 50 ],     // x, y (upper left origin)
 148   //     "color": [ 127, 127, 127 ], // red, green, blue
 149   //     "tolerance": 3
 150   //   }, ...
 151   // ]
 152   bool ReadTestExpectations(const FilePath& json_test_expectations_path,
 153                             std::vector<SinglePixelExpectation>* expectations) {
 154     std::string json_contents;
 155     if (!file_util::ReadFileToString(
 156             json_test_expectations_path, &json_contents)) {
 157       DLOG(ERROR) << "ReadFileToString failed for "
 158                   << json_test_expectations_path.value();
 159       return false;
 160     }
 161     scoped_ptr<Value> root;
 162     int error_code;
 163     std::string error_msg;
 164     root.reset(base::JSONReader::ReadAndReturnError(
 165         json_contents, base::JSON_ALLOW_TRAILING_COMMAS,
 166         &error_code, &error_msg));
 167     if (root.get() == NULL) {
 168       DLOG(ERROR) << "Root was NULL: error code "
 169                   << error_code << ", error message " << error_msg;
 170       return false;
 171     }
 172     ListValue* root_list;
 173     if (!root->GetAsList(&root_list)) {
 174       DLOG(ERROR) << "Root was not a list (type == " << root->GetType() << ")";
 175       return false;
 176     }
 177     for (size_t ii = 0; ii < root_list->GetSize(); ++ii) {
 178       DictionaryValue* entry;
 179       if (!root_list->GetDictionary(ii, &entry)) {
 180         DLOG(ERROR) << "Root entry " << ii << " was not a dictionary";
 181         return false;
 182       }
 183       ListValue* location;
 184       if (!entry->GetList("location", &location)) {
 185         DLOG(ERROR) << "Root entry " << ii << "'s location was not a list";
 186         return false;
 187       }
 188       if (location->GetSize() != 2) {
 189         DLOG(ERROR) << "Root entry " << ii << "'s location list not length 2";
 190         return false;
 191       }
 192       int x, y;
 193       if (!location->GetInteger(0, &x) ||
 194           !location->GetInteger(1, &y)) {
 195         DLOG(ERROR) << "Root entry " << ii << "'s location list not integers";
 196         return false;
 197       }
 198       ListValue* color;
 199       if (!entry->GetList("color", &color)) {
 200         DLOG(ERROR) << "Root entry " << ii << "'s color was not a list";
 201         return false;
 202       }
 203       if (color->GetSize() != 3) {
 204         DLOG(ERROR) << "Root entry " << ii << "'s color list not length 3";
 205         return false;
 206       }
 207       int red, green, blue;
 208       if (!color->GetInteger(0, &red) ||
 209           !color->GetInteger(1, &green) ||
 210           !color->GetInteger(2, &blue)) {
 211         DLOG(ERROR) << "Root entry " << ii << "'s color list not integers";
 212         return false;
 213       }
 214       int tolerance;
 215       if (!entry->GetInteger("tolerance", &tolerance)) {
 216         DLOG(ERROR) << "Root entry " << ii << "'s tolerance not an integer";
 217         return false;
 218       }
 219       SinglePixelExpectation expectation;
 220       expectation.location = SkIPoint::Make(x, y);
 221       expectation.color = SkColorSetRGB(red, green, blue);
 222       expectation.tolerance = tolerance;
 223       expectations->push_back(expectation);
 224     }
 225     return true;
 226   }
 227
 228   // Take snapshot of the current tab, encode it as PNG, and save to a SkBitmap.
 229   bool TabSnapShotToImage(SkBitmap* bitmap) {
 230     CHECK(bitmap);
 231     std::vector<unsigned char> png;
 232
 233     gfx::Rect root_bounds = browser()->window()->GetBounds();
 234     gfx::Rect tab_contents_bounds;
 235     chrome::GetActiveWebContents(browser())->GetContainerBounds(
 236         &tab_contents_bounds);
 237
 238     gfx::Rect snapshot_bounds(tab_contents_bounds.x() - root_bounds.x(),
 239                               tab_contents_bounds.y() - root_bounds.y(),
 240                               tab_contents_bounds.width(),
 241                               tab_contents_bounds.height());
 242
 243     gfx::NativeWindow native_window = browser()->window()->GetNativeWindow();
 244     if (!chrome::GrabWindowSnapshotForUser(native_window, &png,
 245                                            snapshot_bounds)) {
 246       LOG(ERROR) << "browser::GrabWindowSnapShot() failed";
 247       return false;
 248     }
 249
 250     if (!gfx::PNGCodec::Decode(reinterpret_cast<unsigned char*>(&*png.begin()),
 251                                png.size(), bitmap)) {
 252       LOG(ERROR) << "Decode PNG to a SkBitmap failed";
 253       return false;
 254     }
 255     return true;
 256   }
 257
 258   // Returns a gfx::Rect representing the bounds that the browser window should
 259   // have if the tab contents have the desired size.
 260   gfx::Rect GetNewTabContainerBounds(const gfx::Size& desired_size) {
 261     gfx::Rect container_rect;
 262     chrome::GetActiveWebContents(
 263         browser())->GetContainerBounds(&container_rect);
 264     // Size cannot be negative, so use a point.
 265     gfx::Point correction(
 266         desired_size.width() - container_rect.size().width(),
 267         desired_size.height() - container_rect.size().height());
 268
 269     gfx::Rect window_rect = browser()->window()->GetRestoredBounds();
 270     gfx::Size new_size = window_rect.size();
 271     new_size.Enlarge(correction.x(), correction.y());
 272     window_rect.set_size(new_size);
 273     return window_rect;
 274   }
 275
 276   bool CompareColorChannel(uint8_t value,
 277                            uint8_t expected,
 278                            int tolerance) {
 279     int32_t signed_value = value;
 280     int32_t signed_expected = expected;
 281     EXPECT_THAT(signed_value, AllOf(
 282         Ge(signed_expected - tolerance),
 283         Le(signed_expected + tolerance)));
 284     return (signed_value >= signed_expected - tolerance &&
 285             signed_value <= signed_expected + tolerance);
 286   }
 287
 288   bool CompareToExpectedResults(
 289       const SkBitmap& bitmap,
 290       const std::vector<SinglePixelExpectation>& expectations) {
 291     SkAutoLockPixels lock_bitmap(bitmap);
 292     bool result = true;
 293     for (size_t ii = 0; ii < expectations.size(); ++ii) {
 294       const SinglePixelExpectation& expectation = expectations[ii];
 295       SkColor color = bitmap.getColor(expectation.location.x(),
 296                                       expectation.location.y());
 297       result &= CompareColorChannel(SkColorGetR(color),
 298                                     SkColorGetR(expectation.color),
 299                                     expectation.tolerance);
 300       result &= CompareColorChannel(SkColorGetG(color),
 301                                     SkColorGetG(expectation.color),
 302                                     expectation.tolerance);
 303       result &= CompareColorChannel(SkColorGetB(color),
 304                                     SkColorGetB(expectation.color),
 305                                     expectation.tolerance);
 306     }
 307     return result;
 308   }
 309
 310   FRIEND_TEST_ALL_PREFIXES(MapsGLEnduranceTest, TestParseExpectations);
 311
 312   DISALLOW_COPY_AND_ASSIGN(MapsGLEnduranceTest);
 313 };
 314
 315 IN_PROC_BROWSER_TEST_F(MapsGLEnduranceTest, TestParseExpectations) {
 316   std::vector<SinglePixelExpectation> expectations;
 317   ASSERT_TRUE(ReadTestExpectations(
 318       test_data_dir().AppendASCII("mapsgl_unittest_expectations.json"),
 319       &expectations));
 320   ASSERT_EQ(2u, expectations.size());
 321   // These values are hardcoded in the test data.
 322   EXPECT_EQ(25, expectations[0].location.x());
 323   EXPECT_EQ(50, expectations[0].location.y());
 324   EXPECT_EQ(64u, SkColorGetR(expectations[0].color));
 325   EXPECT_EQ(128u, SkColorGetG(expectations[0].color));
 326   EXPECT_EQ(192u, SkColorGetB(expectations[0].color));
 327   EXPECT_EQ(3, expectations[0].tolerance);
 328
 329   EXPECT_EQ(1920, expectations[1].location.x());
 330   EXPECT_EQ(1200, expectations[1].location.y());
 331   EXPECT_EQ(0u, SkColorGetR(expectations[1].color));
 332   EXPECT_EQ(255u, SkColorGetG(expectations[1].color));
 333   EXPECT_EQ(127u, SkColorGetB(expectations[1].color));
 334   EXPECT_EQ(12, expectations[1].tolerance);
 335 }
 336
 337 // This test is being marked MANUAL so that it does not run
 338 // automatically yet, but can be run on demand with the --run-manual
 339 // command line argument. More work is needed to get the test harness
 340 // running on the bots, and to fix flakiness in the test.
 341 IN_PROC_BROWSER_TEST_F(MapsGLEnduranceTest, MANUAL_SingleRunBasic) {
 342   // This expects the MapsGL python server to be running.
 343   RunSingleTest(gfx::Size(1024, 768),
 344                 "http://localhost:8000/basic.html",
 345                 "mapsgl_single_run_basic_expectations.json",
 346                 "single-run-basic");
 347 }