content/browser/gpu/gpu_pixel_browsertest.cc

   1 // Copyright (c) 2013 The Chromium Authors. All rights reserved.
   2 // Use of this source code is governed by a BSD-style license that can be
   3 // found in the LICENSE file.
   4
   5 #include "base/command_line.h"
   6 #include "base/file_util.h"
   7 #include "base/files/file_enumerator.h"
   8 #include "base/files/file_path.h"
   9 #include "base/path_service.h"
  10 #include "base/strings/string_number_conversions.h"
  11 #include "base/strings/string_util.h"
  12 #include "base/strings/stringprintf.h"
  13 #include "content/public/browser/render_view_host.h"
  14 #include "content/public/browser/render_widget_host_view.h"
  15 #include "content/public/browser/web_contents.h"
  16 #include "content/public/common/content_paths.h"
  17 #include "content/public/common/content_switches.h"
  18 #include "content/public/test/browser_test_utils.h"
  19 #include "content/shell/shell.h"
  20 #include "content/test/content_browser_test.h"
  21 #include "content/test/content_browser_test_utils.h"
  22 #include "net/base/net_util.h"
  23 #include "testing/gtest/include/gtest/gtest.h"
  24 #include "third_party/skia/include/core/SkBitmap.h"
  25 #include "third_party/skia/include/core/SkColor.h"
  26 #include "ui/compositor/compositor_setup.h"
  27 #include "ui/gfx/codec/png_codec.h"
  28 #include "ui/gfx/size.h"
  29 #include "ui/gl/gl_switches.h"
  30 #include "ui/snapshot/snapshot.h"
  31
  32 namespace {
  33
  34 enum ReferenceImageOption {
  35   kReferenceImageLocal,
  36   kReferenceImageCheckedIn,
  37   kReferenceImageNone  // Only check a few key pixels.
  38 };
  39
  40 struct ReferencePixel {
  41   int x, y;
  42   unsigned char r, g, b;
  43 };
  44
  45 // Command line flag for overriding the default location for putting generated
  46 // test images that do not match references.
  47 const char kGeneratedDir[] = "generated-dir";
  48 // Command line flag for overriding the default location for reference images.
  49 const char kReferenceDir[] = "reference-dir";
  50 // Command line flag for Chromium build revision.
  51 const char kBuildRevision[] = "build-revision";
  52
  53 // Reads and decodes a PNG image to a bitmap. Returns true on success. The PNG
  54 // should have been encoded using |gfx::PNGCodec::Encode|.
  55 bool ReadPNGFile(const base::FilePath& file_path, SkBitmap* bitmap) {
  56   DCHECK(bitmap);
  57   base::FilePath abs_path(base::MakeAbsoluteFilePath(file_path));
  58   if (abs_path.empty())
  59     return false;
  60
  61   std::string png_data;
  62   return file_util::ReadFileToString(abs_path, &png_data) &&
  63          gfx::PNGCodec::Decode(reinterpret_cast<unsigned char*>(&png_data[0]),
  64                                png_data.length(),
  65                                bitmap);
  66 }
  67
  68 // Encodes a bitmap into a PNG and write to disk. Returns true on success. The
  69 // parent directory does not have to exist.
  70 bool WritePNGFile(const SkBitmap& bitmap, const base::FilePath& file_path) {
  71   std::vector<unsigned char> png_data;
  72   if (gfx::PNGCodec::EncodeBGRASkBitmap(bitmap, true, &png_data) &&
  73       file_util::CreateDirectory(file_path.DirName())) {
  74     int bytes_written = file_util::WriteFile(
  75         file_path, reinterpret_cast<char*>(&png_data[0]), png_data.size());
  76     if (bytes_written == static_cast<int>(png_data.size()))
  77       return true;
  78   }
  79   return false;
  80 }
  81
  82 // Write an empty file, whose name indicates the chrome revision when the ref
  83 // image was generated.
  84 bool WriteREVFile(const base::FilePath& file_path) {
  85   if (file_util::CreateDirectory(file_path.DirName())) {
  86     char one_byte = 0;
  87     int bytes_written = file_util::WriteFile(file_path, &one_byte, 1);
  88     if (bytes_written == 1)
  89       return true;
  90   }
  91   return false;
  92 }
  93
  94 }  // namespace anonymous
  95
  96 namespace content {
  97
  98 // Test fixture for GPU image comparison tests.
  99 // TODO(kkania): Document how to add to/modify these tests.
 100 class GpuPixelBrowserTest : public ContentBrowserTest {
 101  public:
 102   GpuPixelBrowserTest()
 103       : ref_img_revision_(0),
 104         ref_img_revision_no_older_than_(0),
 105         ref_img_option_(kReferenceImageNone) {
 106   }
 107
 108   virtual void SetUpCommandLine(CommandLine* command_line) OVERRIDE {
 109     command_line->AppendSwitchASCII(switches::kTestGLLib,
 110                                     "libllvmpipe.so");
 111   }
 112
 113   virtual void SetUpInProcessBrowserTestFixture() OVERRIDE {
 114     ContentBrowserTest::SetUpInProcessBrowserTestFixture();
 115
 116     CommandLine* command_line = CommandLine::ForCurrentProcess();
 117     if (command_line->HasSwitch(switches::kUseGpuInTests))
 118       ref_img_option_ = kReferenceImageLocal;
 119
 120     if (command_line->HasSwitch(kBuildRevision))
 121       build_revision_ = command_line->GetSwitchValueASCII(kBuildRevision);
 122
 123     ASSERT_TRUE(PathService::Get(DIR_TEST_DATA, &test_data_dir_));
 124     test_data_dir_ = test_data_dir_.AppendASCII("gpu");
 125
 126     if (command_line->HasSwitch(kGeneratedDir))
 127       generated_img_dir_ = command_line->GetSwitchValuePath(kGeneratedDir);
 128     else
 129       generated_img_dir_ = test_data_dir_.AppendASCII("generated");
 130
 131     switch (ref_img_option_) {
 132       case kReferenceImageLocal:
 133         if (command_line->HasSwitch(kReferenceDir))
 134           ref_img_dir_ = command_line->GetSwitchValuePath(kReferenceDir);
 135         else
 136           ref_img_dir_ = test_data_dir_.AppendASCII("gpu_reference");
 137         break;
 138       case kReferenceImageCheckedIn:
 139         ref_img_dir_ = test_data_dir_.AppendASCII("llvmpipe_reference");
 140         break;
 141       default:
 142         break;
 143     }
 144
 145     test_name_ = testing::UnitTest::GetInstance()->current_test_info()->name();
 146     const char* test_status_prefixes[] = {
 147         "DISABLED_", "FLAKY_", "FAILS_", "MANUAL_"};
 148     for (size_t i = 0; i < arraysize(test_status_prefixes); ++i) {
 149       ReplaceFirstSubstringAfterOffset(
 150           &test_name_, 0, test_status_prefixes[i], std::string());
 151     }
 152
 153     ui::DisableTestCompositor();
 154   }
 155
 156   // If the existing ref image was saved from an revision older than the
 157   // ref_img_update_revision, refresh the ref image.
 158   void RunPixelTest(const gfx::Size& tab_container_size,
 159                     const base::FilePath& url,
 160                     int64 ref_img_update_revision,
 161                     const ReferencePixel* ref_pixels,
 162                     size_t ref_pixel_count) {
 163     if (ref_img_option_ == kReferenceImageLocal) {
 164       ref_img_revision_no_older_than_ = ref_img_update_revision;
 165       ObtainLocalRefImageRevision();
 166     }
 167
 168     DOMMessageQueue message_queue;
 169     NavigateToURL(shell(), net::FilePathToFileURL(url));
 170
 171     std::string message;
 172     // Wait for notification that page is loaded.
 173     ASSERT_TRUE(message_queue.WaitForMessage(&message));
 174     EXPECT_STREQ("\"SUCCESS\"", message.c_str()) << message;
 175
 176     SkBitmap bitmap;
 177     ASSERT_TRUE(TabSnapShotToImage(&bitmap, tab_container_size));
 178     bool same_pixels = true;
 179     if (ref_img_option_ == kReferenceImageNone && ref_pixels && ref_pixel_count)
 180       same_pixels = ComparePixels(bitmap, ref_pixels, ref_pixel_count);
 181     else
 182       same_pixels = CompareImages(bitmap);
 183     EXPECT_TRUE(same_pixels);
 184   }
 185
 186   const base::FilePath& test_data_dir() const {
 187     return test_data_dir_;
 188   }
 189
 190  private:
 191   base::FilePath test_data_dir_;
 192   base::FilePath generated_img_dir_;
 193   base::FilePath ref_img_dir_;
 194   int64 ref_img_revision_;
 195   std::string build_revision_;
 196   // The name of the test, with any special prefixes dropped.
 197   std::string test_name_;
 198
 199   // Any local ref image generated from older revision is ignored.
 200   int64 ref_img_revision_no_older_than_;
 201
 202   // Whether use locally generated ref images, or checked in ref images, or
 203   // simply check a few key pixels.
 204   ReferenceImageOption ref_img_option_;
 205
 206   // Compares the generated bitmap with the appropriate reference image on disk.
 207   // Returns true iff the images were the same.
 208   //
 209   // If no valid reference image exists, save the generated bitmap to the disk.
 210   // The image format is:
 211   //     <test_name>_<revision>.png
 212   // E.g.,
 213   //     WebGLTeapot_19762.png
 214   // The number is the chromium revision that generated the image.
 215   //
 216   // On failure or on ref image generation, the image and diff image will be
 217   // written to disk. The formats are:
 218   //     FAIL_<ref_image_name>, DIFF_<ref_image_name>
 219   // E.g.,
 220   //     FAIL_WebGLTeapot_19762.png, DIFF_WebGLTeapot_19762.png
 221   bool CompareImages(const SkBitmap& gen_bmp) {
 222     SkBitmap ref_bmp_on_disk;
 223
 224     base::FilePath img_path = ref_img_dir_.AppendASCII(test_name_ + ".png");
 225     bool found_ref_img = ReadPNGFile(img_path, &ref_bmp_on_disk);
 226
 227     if (!found_ref_img && ref_img_option_ == kReferenceImageCheckedIn) {
 228       LOG(ERROR) << "Couldn't find reference image: "
 229                  << img_path.value();
 230       // No image to compare to, exit early.
 231       return false;
 232     }
 233
 234     const SkBitmap* ref_bmp;
 235     bool save_gen = false;
 236     bool save_diff = true;
 237     bool rt = true;
 238
 239     if ((ref_img_revision_ <= 0 && ref_img_option_ == kReferenceImageLocal) ||
 240         !found_ref_img) {
 241       base::FilePath rev_path = ref_img_dir_.AppendASCII(
 242           test_name_ + "_" + build_revision_ + ".rev");
 243       if (!WritePNGFile(gen_bmp, img_path)) {
 244         LOG(ERROR) << "Can't save generated image to: "
 245                    << img_path.value()
 246                    << " as future reference.";
 247         rt = false;
 248       } else {
 249         LOG(INFO) << "Saved reference image to: "
 250                   << img_path.value();
 251       }
 252       if (rt) {
 253         if (!WriteREVFile(rev_path)) {
 254           LOG(ERROR) << "Can't save revision file to: "
 255                      << rev_path.value();
 256           rt = false;
 257           file_util::Delete(img_path, false);
 258         } else {
 259           LOG(INFO) << "Saved revision file to: "
 260                     << rev_path.value();
 261         }
 262       }
 263       if (ref_img_revision_ > 0) {
 264         LOG(ERROR) << "Can't read the local ref image: "
 265                    << img_path.value()
 266                    << ", reset it.";
 267         rt = false;
 268       }
 269       // If we re-generate the ref image, we save the gen and diff images so
 270       // the ref image can be uploaded to the server and be viewed later.
 271       save_gen = true;
 272       save_diff = true;
 273       ref_bmp = &gen_bmp;
 274     } else {
 275       ref_bmp = &ref_bmp_on_disk;
 276     }
 277
 278     SkBitmap diff_bmp;
 279     if (ref_bmp->width() != gen_bmp.width() ||
 280         ref_bmp->height() != gen_bmp.height()) {
 281       LOG(ERROR)
 282           << "Dimensions do not match (Expected) vs (Actual):"
 283           << "(" << ref_bmp->width() << "x" << ref_bmp->height()
 284               << ") vs. "
 285           << "(" << gen_bmp.width() << "x" << gen_bmp.height() << ")";
 286       if (ref_img_option_ == kReferenceImageLocal)
 287         save_gen = true;
 288       rt = false;
 289     } else {
 290       // Compare pixels and create a simple diff image.
 291       int diff_pixels_count = 0;
 292       diff_bmp.setConfig(SkBitmap::kARGB_8888_Config,
 293                          gen_bmp.width(), gen_bmp.height());
 294       diff_bmp.allocPixels();
 295       diff_bmp.eraseColor(SK_ColorWHITE);
 296       SkAutoLockPixels lock_bmp(gen_bmp);
 297       SkAutoLockPixels lock_ref_bmp(*ref_bmp);
 298       SkAutoLockPixels lock_diff_bmp(diff_bmp);
 299       // The reference images were saved with no alpha channel. Use the mask to
 300       // set alpha to 0.
 301       uint32_t kAlphaMask = 0x00FFFFFF;
 302       for (int x = 0; x < gen_bmp.width(); ++x) {
 303         for (int y = 0; y < gen_bmp.height(); ++y) {
 304           if ((*gen_bmp.getAddr32(x, y) & kAlphaMask) !=
 305               (*ref_bmp->getAddr32(x, y) & kAlphaMask)) {
 306             ++diff_pixels_count;
 307             *diff_bmp.getAddr32(x, y) = 192 << 16;  // red
 308           }
 309         }
 310       }
 311       if (diff_pixels_count > 0) {
 312         LOG(ERROR) << diff_pixels_count
 313                    << " pixels do not match.";
 314         if (ref_img_option_ == kReferenceImageLocal) {
 315           save_gen = true;
 316           save_diff = true;
 317         }
 318         rt = false;
 319       }
 320     }
 321
 322     std::string ref_img_filename = img_path.BaseName().MaybeAsASCII();
 323     if (save_gen) {
 324       base::FilePath img_fail_path = generated_img_dir_.AppendASCII(
 325           "FAIL_" + ref_img_filename);
 326       if (!WritePNGFile(gen_bmp, img_fail_path)) {
 327         LOG(ERROR) << "Can't save generated image to: "
 328                    << img_fail_path.value();
 329       } else {
 330         LOG(INFO) << "Saved generated image to: "
 331                   << img_fail_path.value();
 332       }
 333     }
 334     if (save_diff) {
 335       base::FilePath img_diff_path = generated_img_dir_.AppendASCII(
 336           "DIFF_" + ref_img_filename);
 337       if (!WritePNGFile(diff_bmp, img_diff_path)) {
 338         LOG(ERROR) << "Can't save generated diff image to: "
 339                    << img_diff_path.value();
 340       } else {
 341         LOG(INFO) << "Saved difference image to: "
 342                   << img_diff_path.value();
 343       }
 344     }
 345     return rt;
 346   }
 347
 348   bool ComparePixels(const SkBitmap& gen_bmp,
 349                      const ReferencePixel* ref_pixels,
 350                      size_t ref_pixel_count) {
 351     SkAutoLockPixels lock_bmp(gen_bmp);
 352
 353     for (size_t i = 0; i < ref_pixel_count; ++i) {
 354       int x = ref_pixels[i].x;
 355       int y = ref_pixels[i].y;
 356       unsigned char r = ref_pixels[i].r;
 357       unsigned char g = ref_pixels[i].g;
 358       unsigned char b = ref_pixels[i].b;
 359
 360       DCHECK(x >= 0 && x < gen_bmp.width() && y >= 0 && y < gen_bmp.height());
 361
 362       unsigned char* rgba = reinterpret_cast<unsigned char*>(
 363           gen_bmp.getAddr32(x, y));
 364       DCHECK(rgba);
 365       if (rgba[0] != b || rgba[1] != g || rgba[2] != r) {
 366         std::string error_message = base::StringPrintf(
 367             "pixel(%d,%d) expects [%u,%u,%u], but gets [%u,%u,%u] instead",
 368             x, y, r, g, b, rgba[0], rgba[1], rgba[2]);
 369         LOG(ERROR) << error_message.c_str();
 370         return false;
 371       }
 372     }
 373     return true;
 374   }
 375
 376   // Take snapshot of the tab, encode it as PNG, and save to a SkBitmap.
 377   bool TabSnapShotToImage(SkBitmap* bitmap, const gfx::Size& size) {
 378     CHECK(bitmap);
 379     std::vector<unsigned char> png;
 380
 381     gfx::Rect snapshot_bounds(size);
 382     RenderViewHost* view_host = shell()->web_contents()->GetRenderViewHost();
 383     if (!ui::GrabViewSnapshot(view_host->GetView()->GetNativeView(),
 384                               &png, snapshot_bounds)) {
 385       LOG(ERROR) << "ui::GrabViewSnapShot() failed";
 386       return false;
 387     }
 388
 389     if (!gfx::PNGCodec::Decode(reinterpret_cast<unsigned char*>(&*png.begin()),
 390                                png.size(), bitmap)) {
 391       LOG(ERROR) << "Decode PNG to a SkBitmap failed";
 392       return false;
 393     }
 394     return true;
 395   }
 396
 397   // If no valid local revision file is located, the ref_img_revision_ is 0.
 398   void ObtainLocalRefImageRevision() {
 399     base::FilePath filter;
 400     filter = filter.AppendASCII(test_name_ + "_*.rev");
 401     base::FileEnumerator locator(ref_img_dir_,
 402                                  false,  // non recursive
 403                                  base::FileEnumerator::FILES,
 404                                  filter.value());
 405     int64 max_revision = 0;
 406     std::vector<base::FilePath> outdated_revs;
 407     for (base::FilePath full_path = locator.Next();
 408          !full_path.empty();
 409          full_path = locator.Next()) {
 410       std::string filename =
 411           full_path.BaseName().RemoveExtension().MaybeAsASCII();
 412       std::string revision_string =
 413           filename.substr(test_name_.length() + 1);
 414       int64 revision = 0;
 415       bool converted = base::StringToInt64(revision_string, &revision);
 416       if (!converted)
 417         continue;
 418       if (revision < ref_img_revision_no_older_than_ ||
 419           revision < max_revision) {
 420         outdated_revs.push_back(full_path);
 421         continue;
 422       }
 423       max_revision = revision;
 424     }
 425     ref_img_revision_ = max_revision;
 426     for (size_t i = 0; i < outdated_revs.size(); ++i)
 427       file_util::Delete(outdated_revs[i], false);
 428   }
 429
 430   DISALLOW_COPY_AND_ASSIGN(GpuPixelBrowserTest);
 431 };
 432
 433 IN_PROC_BROWSER_TEST_F(GpuPixelBrowserTest, MANUAL_WebGLGreenTriangle) {
 434   // If test baseline needs to be updated after a given revision, update the
 435   // following number. If no revision requirement, then 0.
 436   const int64 ref_img_revision_update = 123489;
 437
 438   const ReferencePixel ref_pixels[] = {
 439     // x, y, r, g, b
 440     {50, 100, 0, 0, 0},
 441     {100, 100, 0, 255, 0},
 442     {150, 100, 0, 0, 0},
 443     {50, 150, 0, 255, 0},
 444     {100, 150, 0, 255, 0},
 445     {150, 150, 0, 255, 0}
 446   };
 447   const size_t ref_pixel_count = sizeof(ref_pixels) / sizeof(ReferencePixel);
 448
 449   gfx::Size container_size(400, 300);
 450   base::FilePath url =
 451       test_data_dir().AppendASCII("pixel_webgl.html");
 452   RunPixelTest(container_size, url, ref_img_revision_update,
 453                ref_pixels, ref_pixel_count);
 454 }
 455
 456 IN_PROC_BROWSER_TEST_F(GpuPixelBrowserTest, MANUAL_CSS3DBlueBox) {
 457   // If test baseline needs to be updated after a given revision, update the
 458   // following number. If no revision requirement, then 0.
 459   const int64 ref_img_revision_update = 123489;
 460
 461   const ReferencePixel ref_pixels[] = {
 462     // x, y, r, g, b
 463     {70, 50, 0, 0, 255},
 464     {150, 50, 0, 0, 0},
 465     {70, 90, 0, 0, 255},
 466     {150, 90, 0, 0, 255},
 467     {70, 125, 0, 0, 255},
 468     {150, 125, 0, 0, 0}
 469   };
 470   const size_t ref_pixel_count = sizeof(ref_pixels) / sizeof(ReferencePixel);
 471
 472   gfx::Size container_size(400, 300);
 473   base::FilePath url =
 474       test_data_dir().AppendASCII("pixel_css3d.html");
 475   RunPixelTest(container_size, url, ref_img_revision_update,
 476                ref_pixels, ref_pixel_count);
 477 }
 478
 479 IN_PROC_BROWSER_TEST_F(GpuPixelBrowserTest, MANUAL_Canvas2DRedBoxHD) {
 480   // If test baseline needs to be updated after a given revision, update the
 481   // following number. If no revision requirement, then 0.
 482   const int64 ref_img_revision_update = 123489;
 483
 484   const ReferencePixel ref_pixels[] = {
 485     // x, y, r, g, b
 486     {40, 100, 0, 0, 0},
 487     {60, 100, 127, 0, 0},
 488     {140, 100, 127, 0, 0},
 489     {160, 100, 0, 0, 0}
 490   };
 491   const size_t ref_pixel_count = sizeof(ref_pixels) / sizeof(ReferencePixel);
 492
 493   gfx::Size container_size(400, 300);
 494   base::FilePath url =
 495       test_data_dir().AppendASCII("pixel_canvas2d.html");
 496   RunPixelTest(container_size, url, ref_img_revision_update,
 497                ref_pixels, ref_pixel_count);
 498 }
 499
 500 class GpuPixelTestCanvas2DSD : public GpuPixelBrowserTest {
 501  public:
 502   virtual void SetUpCommandLine(CommandLine* command_line) OVERRIDE {
 503     GpuPixelBrowserTest::SetUpCommandLine(command_line);
 504     command_line->AppendSwitch(switches::kDisableAccelerated2dCanvas);
 505   }
 506 };
 507
 508 IN_PROC_BROWSER_TEST_F(GpuPixelTestCanvas2DSD, MANUAL_Canvas2DRedBoxSD) {
 509   // If test baseline needs to be updated after a given revision, update the
 510   // following number. If no revision requirement, then 0.
 511   const int64 ref_img_revision_update = 123489;
 512
 513   const ReferencePixel ref_pixels[] = {
 514     // x, y, r, g, b
 515     {40, 100, 0, 0, 0},
 516     {60, 100, 127, 0, 0},
 517     {140, 100, 127, 0, 0},
 518     {160, 100, 0, 0, 0}
 519   };
 520   const size_t ref_pixel_count = sizeof(ref_pixels) / sizeof(ReferencePixel);
 521
 522   gfx::Size container_size(400, 300);
 523   base::FilePath url =
 524       test_data_dir().AppendASCII("pixel_canvas2d.html");
 525   RunPixelTest(container_size, url, ref_img_revision_update,
 526                ref_pixels, ref_pixel_count);
 527 }
 528
 529 class GpuPixelTestBrowserPlugin : public GpuPixelBrowserTest {
 530  public:
 531   virtual void SetUpCommandLine(CommandLine* command_line) OVERRIDE {
 532     GpuPixelBrowserTest::SetUpCommandLine(command_line);
 533     command_line->AppendSwitch(switches::kEnableBrowserPluginForAllViewTypes);
 534   }
 535 };
 536
 537 IN_PROC_BROWSER_TEST_F(GpuPixelTestBrowserPlugin, MANUAL_BrowserPluginBlueBox) {
 538   // If test baseline needs to be updated after a given revision, update the
 539   // following number. If no revision requirement, then 0.
 540   const int64 ref_img_revision_update = 123489;
 541
 542   const ReferencePixel ref_pixels[] = {
 543     // x, y, r, g, b
 544     {70, 50, 0, 0, 255},
 545     {150, 50, 0, 0, 0},
 546     {70, 90, 0, 0, 255},
 547     {150, 90, 0, 0, 255},
 548     {70, 125, 0, 0, 255},
 549     {150, 125, 0, 0, 0}
 550   };
 551   const size_t ref_pixel_count = sizeof(ref_pixels) / sizeof(ReferencePixel);
 552
 553   gfx::Size container_size(400, 300);
 554   base::FilePath url =
 555       test_data_dir().AppendASCII("pixel_browser_plugin.html");
 556   RunPixelTest(container_size, url, ref_img_revision_update,
 557                ref_pixels, ref_pixel_count);
 558 }
 559
 560 }  // namespace content
 561