content/test/gpu/gpu_tests/pixel.py

   1 # Copyright (c) 2012 The Chromium Authors. All rights reserved.
   2 # Use of this source code is governed by a BSD-style license that can be
   3 # found in the LICENSE file.
   4 from datetime import datetime
   5 import glob
   6 import optparse
   7 import os
   8 import re
   9
  10 from telemetry import test
  11 from telemetry.core.backends import png_bitmap
  12 from telemetry.page import page_test
  13
  14 test_data_dir = os.path.abspath(os.path.join(
  15     os.path.dirname(__file__), '..', '..', 'data', 'gpu'))
  16
  17 default_generated_data_dir = os.path.join(test_data_dir, 'generated')
  18 default_reference_image_dir = os.path.join(test_data_dir, 'gpu_reference')
  19
  20 test_harness_script = r"""
  21   var domAutomationController = {};
  22
  23   domAutomationController._succeeded = false;
  24   domAutomationController._finished = false;
  25
  26   domAutomationController.setAutomationId = function(id) {}
  27
  28   domAutomationController.send = function(msg) {
  29     domAutomationController._finished = true;
  30
  31     if(msg.toLowerCase() == "success") {
  32       domAutomationController._succeeded = true;
  33     } else {
  34       domAutomationController._succeeded = false;
  35     }
  36   }
  37
  38   window.domAutomationController = domAutomationController;
  39 """
  40
  41 class PixelTestFailure(Exception):
  42   pass
  43
  44 def _DidTestSucceed(tab):
  45   return tab.EvaluateJavaScript('domAutomationController._succeeded')
  46
  47 class PixelValidator(page_test.PageTest):
  48   def __init__(self):
  49     super(PixelValidator, self).__init__('ValidatePage')
  50
  51   def CustomizeBrowserOptions(self, options):
  52     options.AppendExtraBrowserArgs('--enable-gpu-benchmarking')
  53
  54   def ValidatePage(self, page, tab, results):
  55     if not _DidTestSucceed(tab):
  56       raise page_test.Failure('Page indicated a failure')
  57
  58     if not tab.screenshot_supported:
  59       raise page_test.Failure('Browser does not support screenshot capture')
  60
  61     screenshot = tab.Screenshot(5)
  62
  63     if not screenshot:
  64       raise page_test.Failure('Could not capture screenshot')
  65
  66     if hasattr(page, 'test_rect'):
  67       screenshot = screenshot.Crop(
  68           page.test_rect[0], page.test_rect[1],
  69           page.test_rect[2], page.test_rect[3])
  70
  71     image_name = PixelValidator.UrlToImageName(page.display_name)
  72
  73     ref_png = PixelValidator.GetReferenceImage(self.options.reference_dir,
  74         image_name, page.revision, screenshot)
  75
  76     # Test new snapshot against existing reference image
  77     if not ref_png.IsEqual(screenshot, tolerance=2):
  78       PixelValidator.WriteErrorImages(self.options.generated_dir, image_name,
  79           self.options.build_revision, screenshot, ref_png)
  80       raise page_test.Failure('Reference image did not match captured screen')
  81
  82   @staticmethod
  83   def UrlToImageName(url):
  84     image_name = re.sub(r'^(http|https|file)://(/*)', '', url)
  85     image_name = re.sub(r'\.\./', '', image_name)
  86     image_name = re.sub(r'(\.|/|-)', '_', image_name)
  87     return image_name
  88
  89   @staticmethod
  90   def DeleteOldReferenceImages(ref_image_path, cur_revision):
  91     if not cur_revision:
  92       return
  93
  94     old_revisions = glob.glob(ref_image_path + "_*.png")
  95     for rev_path in old_revisions:
  96       m = re.match(r'^.*_(\d+)\.png$', rev_path)
  97       if m and int(m.group(1)) < cur_revision:
  98         print 'Found deprecated reference image. Deleting rev ' + m.group(1)
  99         os.remove(rev_path)
 100
 101   @staticmethod
 102   def GetReferenceImage(img_dir, img_name, cur_revision, screenshot):
 103     if not cur_revision:
 104       cur_revision = 0
 105
 106     image_path = os.path.join(img_dir, img_name)
 107
 108     PixelValidator.DeleteOldReferenceImages(image_path, cur_revision)
 109
 110     image_path = image_path + '_' + str(cur_revision) + '.png'
 111
 112     try:
 113       ref_png = png_bitmap.PngBitmap.FromFile(image_path)
 114     except IOError:
 115       ref_png = None
 116
 117     if ref_png:
 118       return ref_png
 119
 120     print 'Reference image not found. Writing tab contents as reference.'
 121
 122     PixelValidator.WriteImage(image_path, screenshot)
 123     return screenshot
 124
 125   @staticmethod
 126   def WriteErrorImages(img_dir, img_name, build_revision, screenshot, ref_png):
 127     full_image_name = img_name + '_' + str(build_revision)
 128     full_image_name = full_image_name + '.png'
 129
 130     # Save the reference image
 131     # This ensures that we get the right revision number
 132     PixelValidator.WriteImage(
 133         os.path.join(img_dir, full_image_name), ref_png)
 134
 135     PixelValidator.WriteImage(
 136         os.path.join(img_dir, 'FAIL_' + full_image_name), screenshot)
 137
 138     diff_png = screenshot.Diff(ref_png)
 139     PixelValidator.WriteImage(
 140         os.path.join(img_dir, 'DIFF_' + full_image_name), diff_png)
 141
 142   @staticmethod
 143   def WriteImage(image_path, png_image):
 144     output_dir = os.path.dirname(image_path)
 145     if not os.path.exists(output_dir):
 146       os.makedirs(output_dir)
 147
 148     png_image.WriteFile(image_path)
 149
 150 class Pixel(test.Test):
 151   enabled = False
 152   test = PixelValidator
 153   page_set = 'page_sets/pixel_tests.json'
 154
 155   @staticmethod
 156   def AddTestCommandLineOptions(parser):
 157     group = optparse.OptionGroup(parser, 'Pixel test options')
 158     group.add_option('--generated-dir',
 159         help='Overrides the default location for generated test images that do '
 160         'not match reference images',
 161         default=default_generated_data_dir)
 162     group.add_option('--reference-dir',
 163         help='Overrides the default location for reference images',
 164         default=default_reference_image_dir)
 165     group.add_option('--build-revision',
 166         help='Chrome revision being tested.',
 167         default="unknownrev")
 168     parser.add_option_group(group)
 169
 170   def CreatePageSet(self, options):
 171     page_set = super(Pixel, self).CreatePageSet(options)
 172     for page in page_set.pages:
 173       page.script_to_evaluate_on_commit = test_harness_script
 174     return page_set