content/test/gpu/gpu_tests/pixel.py

   1 # Copyright (c) 2012 The Chromium Authors. All rights reserved.
   2 # Use of this source code is governed by a BSD-style license that can be
   3 # found in the LICENSE file.
   4 from datetime import datetime
   5 import glob
   6 import optparse
   7 import os
   8 import re
   9
  10 import cloud_storage_test_base
  11 import page_sets
  12 import pixel_expectations
  13
  14 from catapult_base import cloud_storage
  15 from telemetry.page import page_test
  16 from telemetry.util import image_util
  17
  18
  19 test_data_dir = os.path.abspath(os.path.join(
  20     os.path.dirname(__file__), '..', '..', 'data', 'gpu'))
  21
  22 default_reference_image_dir = os.path.join(test_data_dir, 'gpu_reference')
  23
  24 test_harness_script = r"""
  25   var domAutomationController = {};
  26
  27   domAutomationController._succeeded = false;
  28   domAutomationController._finished = false;
  29
  30   domAutomationController.setAutomationId = function(id) {}
  31
  32   domAutomationController.send = function(msg) {
  33     domAutomationController._finished = true;
  34
  35     if(msg.toLowerCase() == "success") {
  36       domAutomationController._succeeded = true;
  37     } else {
  38       domAutomationController._succeeded = false;
  39     }
  40   }
  41
  42   window.domAutomationController = domAutomationController;
  43 """
  44
  45 class PixelTestFailure(Exception):
  46   pass
  47
  48 def _DidTestSucceed(tab):
  49   return tab.EvaluateJavaScript('domAutomationController._succeeded')
  50
  51 class _PixelValidator(cloud_storage_test_base.ValidatorBase):
  52   def CustomizeBrowserOptions(self, options):
  53     options.AppendExtraBrowserArgs('--enable-gpu-benchmarking')
  54
  55   def ValidateAndMeasurePageInner(self, page, tab, results):
  56     if not _DidTestSucceed(tab):
  57       raise page_test.Failure('Page indicated a failure')
  58
  59     if not tab.screenshot_supported:
  60       raise page_test.Failure('Browser does not support screenshot capture')
  61
  62     screenshot = tab.Screenshot(5)
  63
  64     if screenshot is None:
  65       raise page_test.Failure('Could not capture screenshot')
  66
  67     if hasattr(page, 'test_rect'):
  68       screenshot = image_util.Crop(
  69           screenshot, page.test_rect[0], page.test_rect[1],
  70           page.test_rect[2], page.test_rect[3])
  71
  72     image_name = self._UrlToImageName(page.display_name)
  73
  74     if self.options.upload_refimg_to_cloud_storage:
  75       if self._ConditionallyUploadToCloudStorage(image_name, page, tab,
  76                                                  screenshot):
  77         # This is the new reference image; there's nothing to compare against.
  78         ref_png = screenshot
  79       else:
  80         # There was a preexisting reference image, so we might as well
  81         # compare against it.
  82         ref_png = self._DownloadFromCloudStorage(image_name, page, tab)
  83     elif self.options.download_refimg_from_cloud_storage:
  84       # This bot doesn't have the ability to properly generate a
  85       # reference image, so download it from cloud storage.
  86       try:
  87         ref_png = self._DownloadFromCloudStorage(image_name, page, tab)
  88       except cloud_storage.NotFoundError as e:
  89         # There is no reference image yet in cloud storage. This
  90         # happens when the revision of the test is incremented or when
  91         # a new test is added, because the trybots are not allowed to
  92         # produce reference images, only the bots on the main
  93         # waterfalls. Report this as a failure so the developer has to
  94         # take action by explicitly suppressing the failure and
  95         # removing the suppression once the reference images have been
  96         # generated. Otherwise silent failures could happen for long
  97         # periods of time.
  98         raise page_test.Failure('Could not find image %s in cloud storage' %
  99                                 image_name)
 100     else:
 101       # Legacy path using on-disk results.
 102       ref_png = self._GetReferenceImage(self.options.reference_dir,
 103           image_name, page.revision, screenshot)
 104
 105     # Test new snapshot against existing reference image
 106     if not image_util.AreEqual(ref_png, screenshot, tolerance=2):
 107       if self.options.test_machine_name:
 108         self._UploadErrorImagesToCloudStorage(image_name, screenshot, ref_png)
 109       else:
 110         self._WriteErrorImages(self.options.generated_dir, image_name,
 111                                screenshot, ref_png)
 112       raise page_test.Failure('Reference image did not match captured screen')
 113
 114   def _DeleteOldReferenceImages(self, ref_image_path, cur_revision):
 115     if not cur_revision:
 116       return
 117
 118     old_revisions = glob.glob(ref_image_path + "_*.png")
 119     for rev_path in old_revisions:
 120       m = re.match(r'^.*_(\d+)\.png$', rev_path)
 121       if m and int(m.group(1)) < cur_revision:
 122         print 'Found deprecated reference image. Deleting rev ' + m.group(1)
 123         os.remove(rev_path)
 124
 125   def _GetReferenceImage(self, img_dir, img_name, cur_revision, screenshot):
 126     if not cur_revision:
 127       cur_revision = 0
 128
 129     image_path = os.path.join(img_dir, img_name)
 130
 131     self._DeleteOldReferenceImages(image_path, cur_revision)
 132
 133     image_path = image_path + '_' + str(cur_revision) + '.png'
 134
 135     try:
 136       ref_png = image_util.FromPngFile(image_path)
 137     except:
 138       ref_png = None
 139
 140     if ref_png is not None:
 141       return ref_png
 142
 143     print 'Reference image not found. Writing tab contents as reference.'
 144
 145     self._WriteImage(image_path, screenshot)
 146     return screenshot
 147
 148 class Pixel(cloud_storage_test_base.TestBase):
 149   test = _PixelValidator
 150
 151   @classmethod
 152   def Name(cls):
 153     return 'pixel'
 154
 155   @classmethod
 156   def AddBenchmarkCommandLineArgs(cls, group):
 157     super(Pixel, cls).AddBenchmarkCommandLineArgs(group)
 158     group.add_option('--reference-dir',
 159         help='Overrides the default on-disk location for reference images '
 160         '(only used for local testing without a cloud storage account)',
 161         default=default_reference_image_dir)
 162
 163   def CreateStorySet(self, options):
 164     story_set = page_sets.PixelTestsStorySet(self.GetExpectations())
 165     for page in story_set:
 166       page.script_to_evaluate_on_commit = test_harness_script
 167     return story_set
 168
 169   def _CreateExpectations(self):
 170     return pixel_expectations.PixelExpectations()