content/test/gpu/gpu_tests/pixel.py

   1 # Copyright (c) 2012 The Chromium Authors. All rights reserved.
   2 # Use of this source code is governed by a BSD-style license that can be
   3 # found in the LICENSE file.
   4 from datetime import datetime
   5 import glob
   6 import json
   7 import optparse
   8 import os
   9 import re
  10
  11 import cloud_storage_test_base
  12 import page_sets
  13 import pixel_expectations
  14
  15 from catapult_base import cloud_storage
  16 from telemetry.page import page_test
  17 from telemetry.util import image_util
  18
  19
  20 test_data_dir = os.path.abspath(os.path.join(
  21     os.path.dirname(__file__), '..', '..', 'data', 'gpu'))
  22
  23 default_reference_image_dir = os.path.join(test_data_dir, 'gpu_reference')
  24
  25 test_harness_script = r"""
  26   var domAutomationController = {};
  27
  28   domAutomationController._succeeded = false;
  29   domAutomationController._finished = false;
  30
  31   domAutomationController.setAutomationId = function(id) {}
  32
  33   domAutomationController.send = function(msg) {
  34     domAutomationController._finished = true;
  35
  36     if(msg.toLowerCase() == "success") {
  37       domAutomationController._succeeded = true;
  38     } else {
  39       domAutomationController._succeeded = false;
  40     }
  41   }
  42
  43   window.domAutomationController = domAutomationController;
  44 """
  45
  46 class PixelTestFailure(Exception):
  47   pass
  48
  49 def _DidTestSucceed(tab):
  50   return tab.EvaluateJavaScript('domAutomationController._succeeded')
  51
  52 class PixelValidator(cloud_storage_test_base.ValidatorBase):
  53   def CustomizeBrowserOptions(self, options):
  54     options.AppendExtraBrowserArgs('--enable-gpu-benchmarking')
  55
  56   def ValidateAndMeasurePageInner(self, page, tab, results):
  57     if not _DidTestSucceed(tab):
  58       raise page_test.Failure('Page indicated a failure')
  59
  60     if not tab.screenshot_supported:
  61       raise page_test.Failure('Browser does not support screenshot capture')
  62
  63     screenshot = tab.Screenshot(5)
  64
  65     if screenshot is None:
  66       raise page_test.Failure('Could not capture screenshot')
  67
  68     dpr = tab.EvaluateJavaScript('window.devicePixelRatio')
  69
  70     if hasattr(page, 'test_rect'):
  71       screenshot = image_util.Crop(
  72           screenshot, page.test_rect[0] * dpr, page.test_rect[1] * dpr,
  73           page.test_rect[2] * dpr, page.test_rect[3] * dpr)
  74
  75     if hasattr(page, 'expected_colors'):
  76       # Use expected pixels instead of ref images for validation.
  77       expected_colors_file = os.path.abspath(os.path.join(
  78           os.path.dirname(__file__), page.expected_colors))
  79       expected_colors = self._ReadPixelExpectations(expected_colors_file)
  80       self._ValidateScreenshotSamples(
  81           page.display_name, screenshot, expected_colors, dpr)
  82       return
  83
  84     image_name = self._UrlToImageName(page.display_name)
  85
  86     if self.options.upload_refimg_to_cloud_storage:
  87       if self._ConditionallyUploadToCloudStorage(image_name, page, tab,
  88                                                  screenshot):
  89         # This is the new reference image; there's nothing to compare against.
  90         ref_png = screenshot
  91       else:
  92         # There was a preexisting reference image, so we might as well
  93         # compare against it.
  94         ref_png = self._DownloadFromCloudStorage(image_name, page, tab)
  95     elif self.options.download_refimg_from_cloud_storage:
  96       # This bot doesn't have the ability to properly generate a
  97       # reference image, so download it from cloud storage.
  98       try:
  99         ref_png = self._DownloadFromCloudStorage(image_name, page, tab)
 100       except cloud_storage.NotFoundError as e:
 101         # There is no reference image yet in cloud storage. This
 102         # happens when the revision of the test is incremented or when
 103         # a new test is added, because the trybots are not allowed to
 104         # produce reference images, only the bots on the main
 105         # waterfalls. Report this as a failure so the developer has to
 106         # take action by explicitly suppressing the failure and
 107         # removing the suppression once the reference images have been
 108         # generated. Otherwise silent failures could happen for long
 109         # periods of time.
 110         raise page_test.Failure('Could not find image %s in cloud storage' %
 111                                 image_name)
 112     else:
 113       # Legacy path using on-disk results.
 114       ref_png = self._GetReferenceImage(self.options.reference_dir,
 115           image_name, page.revision, screenshot)
 116
 117     # Test new snapshot against existing reference image
 118     if not image_util.AreEqual(ref_png, screenshot, tolerance=2):
 119       if self.options.test_machine_name:
 120         self._UploadErrorImagesToCloudStorage(image_name, screenshot, ref_png)
 121       else:
 122         self._WriteErrorImages(self.options.generated_dir, image_name,
 123                                screenshot, ref_png)
 124       raise page_test.Failure('Reference image did not match captured screen')
 125
 126   def _DeleteOldReferenceImages(self, ref_image_path, cur_revision):
 127     if not cur_revision:
 128       return
 129
 130     old_revisions = glob.glob(ref_image_path + "_*.png")
 131     for rev_path in old_revisions:
 132       m = re.match(r'^.*_(\d+)\.png$', rev_path)
 133       if m and int(m.group(1)) < cur_revision:
 134         print 'Found deprecated reference image. Deleting rev ' + m.group(1)
 135         os.remove(rev_path)
 136
 137   def _GetReferenceImage(self, img_dir, img_name, cur_revision, screenshot):
 138     if not cur_revision:
 139       cur_revision = 0
 140
 141     image_path = os.path.join(img_dir, img_name)
 142
 143     self._DeleteOldReferenceImages(image_path, cur_revision)
 144
 145     image_path = image_path + '_' + str(cur_revision) + '.png'
 146
 147     try:
 148       ref_png = image_util.FromPngFile(image_path)
 149     # This can raise a couple of exceptions including IOError and ValueError.
 150     except Exception:
 151       ref_png = None
 152
 153     if ref_png is not None:
 154       return ref_png
 155
 156     print 'Reference image not found. Writing tab contents as reference.'
 157
 158     self._WriteImage(image_path, screenshot)
 159     return screenshot
 160
 161   def _ReadPixelExpectations(self, json_file_path):
 162     with open(json_file_path, 'r') as f:
 163       json_contents = json.load(f)
 164     return json_contents
 165
 166 class Pixel(cloud_storage_test_base.TestBase):
 167   test = PixelValidator
 168
 169   @classmethod
 170   def Name(cls):
 171     return 'pixel'
 172
 173   @classmethod
 174   def AddBenchmarkCommandLineArgs(cls, group):
 175     super(Pixel, cls).AddBenchmarkCommandLineArgs(group)
 176     group.add_option('--reference-dir',
 177         help='Overrides the default on-disk location for reference images '
 178         '(only used for local testing without a cloud storage account)',
 179         default=default_reference_image_dir)
 180
 181   def CreateStorySet(self, options):
 182     story_set = page_sets.PixelTestsStorySet(self.GetExpectations())
 183     for page in story_set:
 184       page.script_to_evaluate_on_commit = test_harness_script
 185     return story_set
 186
 187   def _CreateExpectations(self):
 188     return pixel_expectations.PixelExpectations()