chrome/browser/chromeos/login/screenshot_testing/screenshot_tester.cc

   1 // Copyright 2014 The Chromium Authors. All rights reserved.
   2 // Use of this source code is governed by a BSD-style license that can be
   3 // found in the LICENSE file.
   4
   5 #include "chrome/browser/chromeos/login/screenshot_testing/screenshot_tester.h"
   6
   7 #include "ash/shell.h"
   8 #include "base/command_line.h"
   9 #include "base/files/file_util.h"
  10 #include "chrome/browser/chromeos/login/screenshot_testing/SkDiffPixelsMetric.h"
  11 #include "chrome/browser/chromeos/login/screenshot_testing/SkImageDiffer.h"
  12 #include "chrome/browser/chromeos/login/screenshot_testing/SkPMetric.h"
  13 #include "chromeos/chromeos_switches.h"
  14 #include "content/public/browser/browser_thread.h"
  15 #include "testing/gtest/include/gtest/gtest.h"
  16 #include "third_party/skia/include/core/SkCanvas.h"
  17 #include "ui/compositor/compositor_switches.h"
  18 #include "ui/gfx/codec/png_codec.h"
  19 #include "ui/gfx/image/image.h"
  20 #include "ui/snapshot/snapshot.h"
  21
  22 namespace {
  23
  24 // Sets test mode for screenshot testing, using regular comparison.
  25 const char kTestMode[] = "test";
  26
  27 // Sets update mode for screenshot testing.
  28 const char kUpdateMode[] = "update";
  29
  30 // Sets test mode for screenshot testing, using PerceptualDiff as comparison.
  31 const char kPdiffTestMode[] = "pdiff-test";
  32
  33 }  // namespace
  34
  35 namespace chromeos {
  36
  37 ScreenshotTester::ScreenshotTester()
  38     : test_mode_(false), pdiff_enabled_(false), weak_factory_(this) {
  39 }
  40
  41 ScreenshotTester::~ScreenshotTester() {
  42 }
  43
  44 ScreenshotTester::Result::Result() {
  45 }
  46
  47 ScreenshotTester::Result::~Result() {
  48 }
  49
  50 bool ScreenshotTester::TryInitialize() {
  51   base::CommandLine& command_line = *base::CommandLine::ForCurrentProcess();
  52   if (!command_line.HasSwitch(switches::kEnableScreenshotTestingWithMode))
  53     return false;
  54
  55   std::string mode = command_line.GetSwitchValueASCII(
  56       switches::kEnableScreenshotTestingWithMode);
  57   CHECK(mode == kUpdateMode || mode == kTestMode || mode == kPdiffTestMode)
  58       << "Invalid mode for screenshot testing: " << mode;
  59
  60   CHECK(command_line.HasSwitch(chromeos::switches::kGoldenScreenshotsDir))
  61       << "No directory with golden screenshots specified, use "
  62          "--golden-screenshots-dir";
  63
  64   golden_screenshots_dir_ =
  65       command_line.GetSwitchValuePath(switches::kGoldenScreenshotsDir);
  66
  67   if (mode == kTestMode || mode == kPdiffTestMode) {
  68     test_mode_ = true;
  69     generate_artifacts_ = command_line.HasSwitch(switches::kArtifactsDir);
  70     if (generate_artifacts_) {
  71       artifacts_dir_ = command_line.GetSwitchValuePath(switches::kArtifactsDir);
  72     }
  73   }
  74   if (mode == kPdiffTestMode) {
  75     pdiff_enabled_ = true;
  76   }
  77   return true;
  78 }
  79
  80 std::string ScreenshotTester::GetImageFileName(
  81     const std::string& file_name_prefix,
  82     ImageCategories category) {
  83   std::string file_name = file_name_prefix + "_";
  84   switch (category) {
  85     case kGoldenScreenshot: {
  86       file_name += "golden_screenshot";
  87       break;
  88     }
  89     case kFailedScreenshot: {
  90       file_name += "failed_screenshot";
  91       break;
  92     }
  93     case kDifferenceImage: {
  94       file_name += "difference";
  95       break;
  96     }
  97   }
  98   return file_name + ".png";
  99 }
 100
 101 base::FilePath ScreenshotTester::GetImageFilePath(
 102     const std::string& file_name_prefix,
 103     ImageCategories category) {
 104   std::string file_name = GetImageFileName(file_name_prefix, category);
 105   base::FilePath file_path;
 106   if (category == kGoldenScreenshot) {
 107     file_path = golden_screenshots_dir_.AppendASCII(file_name);
 108   } else {
 109     file_path = artifacts_dir_.AppendASCII(file_name);
 110   }
 111   return file_path;
 112 }
 113
 114 void ScreenshotTester::Run(const std::string& test_name) {
 115   PNGFile current_screenshot = TakeScreenshot();
 116   base::FilePath golden_screenshot_path =
 117       GetImageFilePath(test_name, kGoldenScreenshot);
 118   PNGFile golden_screenshot = LoadGoldenScreenshot(golden_screenshot_path);
 119   if (test_mode_) {
 120     CHECK(golden_screenshot.get())
 121         << "A golden screenshot is required for screenshot testing";
 122     VLOG(0) << "Loaded golden screenshot";
 123     Result result = CompareScreenshots(golden_screenshot, current_screenshot);
 124     VLOG(0) << "Compared";
 125     LogComparisonResults(result);
 126     if (!result.screenshots_match && generate_artifacts_) {
 127       // Saving diff imag
 128       if (!pdiff_enabled_) {
 129         base::FilePath difference_image_path =
 130             GetImageFilePath(test_name, kDifferenceImage);
 131         CHECK(SaveImage(difference_image_path, result.diff_image));
 132       }
 133
 134       // Saving failed screenshot
 135       base::FilePath failed_screenshot_path =
 136           GetImageFilePath(test_name, kFailedScreenshot);
 137       CHECK(SaveImage(failed_screenshot_path, current_screenshot));
 138     }
 139     ASSERT_TRUE(result.screenshots_match);
 140   } else {
 141     bool golden_screenshot_needs_update;
 142     if (golden_screenshot.get()) {
 143       // There is a golden screenshot, so we need to check it first.
 144       Result result = CompareScreenshots(golden_screenshot, current_screenshot);
 145       golden_screenshot_needs_update = (!result.screenshots_match);
 146     } else {
 147       // There is no golden screenshot for this test at all.
 148       golden_screenshot_needs_update = true;
 149     }
 150     if (golden_screenshot_needs_update) {
 151       bool golden_screenshot_saved =
 152           SaveImage(golden_screenshot_path, current_screenshot);
 153       CHECK(golden_screenshot_saved);
 154     } else {
 155       VLOG(0) << "Golden screenshot does not differ from the current one, no "
 156                  "need to update";
 157     }
 158   }
 159 }
 160
 161 void ScreenshotTester::IgnoreArea(const SkIRect& area) {
 162   ignored_areas_.push_back(area);
 163 }
 164
 165 void ScreenshotTester::EraseIgnoredAreas(SkBitmap& bitmap) {
 166   for (std::vector<SkIRect>::iterator it = ignored_areas_.begin();
 167        it != ignored_areas_.end();
 168        ++it) {
 169     bitmap.eraseArea((*it), SK_ColorWHITE);
 170   }
 171 }
 172
 173 bool ScreenshotTester::SaveImage(const base::FilePath& image_path,
 174                                  PNGFile png_data) {
 175   DCHECK_CURRENTLY_ON(content::BrowserThread::UI);
 176   if (!png_data.get()) {
 177     LOG(ERROR) << "There is no png data";
 178     return false;
 179   }
 180   if (!base::CreateDirectory(image_path.DirName())) {
 181     LOG(ERROR) << "Can't create directory" << image_path.DirName().value();
 182     return false;
 183   }
 184   if (static_cast<size_t>(
 185           base::WriteFile(image_path,
 186                           reinterpret_cast<char*>(&(png_data->data()[0])),
 187                           png_data->size())) != png_data->size()) {
 188     LOG(ERROR) << "Can't save screenshot " << image_path.BaseName().value()
 189                << ".";
 190     return false;
 191   }
 192   VLOG(0) << "Screenshot " << image_path.BaseName().value() << " saved to "
 193           << image_path.DirName().value() << ".";
 194   return true;
 195 }
 196
 197 void ScreenshotTester::ReturnScreenshot(const PNGFile& screenshot,
 198                                         PNGFile png_data) {
 199   DCHECK_CURRENTLY_ON(content::BrowserThread::UI);
 200   screenshot->data() = png_data->data();
 201   content::BrowserThread::PostTask(
 202       content::BrowserThread::UI, FROM_HERE, run_loop_quitter_);
 203 }
 204
 205 ScreenshotTester::PNGFile ScreenshotTester::TakeScreenshot() {
 206   DCHECK_CURRENTLY_ON(content::BrowserThread::UI);
 207   aura::Window* primary_window = ash::Shell::GetPrimaryRootWindow();
 208   gfx::Rect rect = primary_window->bounds();
 209   PNGFile screenshot = new base::RefCountedBytes;
 210   ui::GrabWindowSnapshotAsync(primary_window,
 211                               rect,
 212                               content::BrowserThread::GetBlockingPool(),
 213                               base::Bind(&ScreenshotTester::ReturnScreenshot,
 214                                          weak_factory_.GetWeakPtr(),
 215                                          screenshot));
 216   base::RunLoop run_loop;
 217   run_loop_quitter_ = run_loop.QuitClosure();
 218   run_loop.Run();
 219   return screenshot;
 220 }
 221
 222 ScreenshotTester::PNGFile ScreenshotTester::LoadGoldenScreenshot(
 223     base::FilePath image_path) {
 224   DCHECK_CURRENTLY_ON(content::BrowserThread::UI);
 225
 226   if (!base::PathExists(image_path)) {
 227     LOG(WARNING) << "Can't find a golden screenshot for this test";
 228     return 0;
 229   }
 230
 231   int64 golden_screenshot_size;
 232   base::GetFileSize(image_path, &golden_screenshot_size);
 233
 234   if (golden_screenshot_size == -1) {
 235     CHECK(false) << "Can't get golden screenshot size";
 236   }
 237   PNGFile png_data = new base::RefCountedBytes;
 238   png_data->data().resize(golden_screenshot_size);
 239   base::ReadFile(image_path,
 240                  reinterpret_cast<char*>(&(png_data->data()[0])),
 241                  golden_screenshot_size);
 242
 243   return png_data;
 244 }
 245
 246 SkBitmap ScreenshotTester::ProcessImageForComparison(const PNGFile& image) {
 247   CHECK(image.get());
 248   SkBitmap current_bitmap;
 249   gfx::PNGCodec::Decode(reinterpret_cast<unsigned char*>(&(image->data()[0])),
 250                         image->data().size(),
 251                         &current_bitmap);
 252   EraseIgnoredAreas(current_bitmap);
 253   return current_bitmap;
 254 }
 255
 256 ScreenshotTester::Result ScreenshotTester::CompareScreenshots(
 257     const PNGFile& model,
 258     const PNGFile& sample) {
 259   DCHECK_CURRENTLY_ON(content::BrowserThread::UI);
 260
 261   SkBitmap model_bitmap;
 262   SkBitmap sample_bitmap;
 263
 264   model_bitmap = ProcessImageForComparison(model);
 265   sample_bitmap = ProcessImageForComparison(sample);
 266
 267   if (pdiff_enabled_) {
 268     return CompareScreenshotsPerceptually(model_bitmap, sample_bitmap);
 269   } else {
 270     return CompareScreenshotsRegularly(model_bitmap, sample_bitmap);
 271   }
 272 }
 273
 274 void ScreenshotTester::LogSimilarity(double similarity,
 275                                      bool screenshots_match) {
 276   VLOG(0) << "Screenshots similarity: " << std::setprecision(5)
 277           << similarity * 100 << "\%";
 278   if (!pdiff_enabled_ && screenshots_match) {
 279     if (similarity == 1) {  // 100%
 280       VLOG(0) << "Screenshots match perfectly";
 281     } else {
 282       VLOG(0) << "Screenshots differ slightly, but it is still a match";
 283     }
 284   }
 285 }
 286
 287 void ScreenshotTester::LogComparisonResults(
 288     const ScreenshotTester::Result& result) {
 289   std::string comparison_type = pdiff_enabled_ ? "PerceptualDiff" : "regular";
 290   if (result.screenshots_match) {
 291     VLOG(0) << "Screenshot testing passed using " << comparison_type
 292             << " comparison";
 293   } else {
 294     LOG(ERROR) << "Screenshot testing failed using " << comparison_type
 295                << " comparison";
 296     if (!pdiff_enabled_) {
 297       VLOG(0) << "(HINT): Result may be false negative. Try using "
 298                  "PerceptualDiff comparison (use pdiff-test mode instead of "
 299                  "test)";
 300     }
 301   }
 302   LogSimilarity(result.similarity, result.screenshots_match);
 303 }
 304
 305 ScreenshotTester::Result ScreenshotTester::CompareScreenshotsRegularly(
 306     SkBitmap model_bitmap,
 307     SkBitmap sample_bitmap) {
 308   SkDifferentPixelsMetric differ;
 309
 310   SkImageDiffer::BitmapsToCreate diff_parameters;
 311   diff_parameters.rgbDiff = true;
 312   SkImageDiffer::Result result;
 313
 314   differ.diff(&model_bitmap, &sample_bitmap, diff_parameters, &result);
 315
 316   Result testing_result;
 317
 318   testing_result.screenshots_match =
 319       (result.result >= kPrecision &&
 320        result.maxRedDiff <= kMaxAllowedColorDifference &&
 321        result.maxGreenDiff <= kMaxAllowedColorDifference &&
 322        result.maxBlueDiff <= kMaxAllowedColorDifference);
 323
 324   testing_result.similarity = result.result;
 325
 326   testing_result.diff_image = new base::RefCountedBytes;
 327   testing_result.diff_image->data().resize(result.rgbDiffBitmap.getSize());
 328   CHECK(gfx::PNGCodec::EncodeBGRASkBitmap(
 329       result.rgbDiffBitmap, false, &testing_result.diff_image->data()))
 330       << "Could not encode difference to PNG";
 331
 332   return testing_result;
 333 }
 334
 335 ScreenshotTester::Result ScreenshotTester::CompareScreenshotsPerceptually(
 336     SkBitmap model_bitmap,
 337     SkBitmap sample_bitmap) {
 338   SkPMetric differ;
 339   SkImageDiffer::BitmapsToCreate diff_parameters;
 340   SkImageDiffer::Result result;
 341
 342   differ.diff(&model_bitmap, &sample_bitmap, diff_parameters, &result);
 343
 344   ScreenshotTester::Result testing_result;
 345   testing_result.similarity = result.result;
 346   testing_result.screenshots_match =
 347       (result.result == SkImageDiffer::RESULT_CORRECT);
 348
 349   return testing_result;
 350 }
 351
 352 }  // namespace chromeos