content/test/gpu/gpu_tests/cloud_storage_test_base.py

   1 # Copyright 2013 The Chromium Authors. All rights reserved.
   2 # Use of this source code is governed by a BSD-style license that can be
   3 # found in the LICENSE file.
   4
   5 """Base classes for a test and validator which upload results
   6 (reference images, error images) to cloud storage."""
   7
   8 import os
   9 import re
  10 import tempfile
  11
  12 from telemetry import benchmark
  13 from telemetry.core import bitmap
  14 from telemetry.page import page_test
  15 from telemetry.util import cloud_storage
  16
  17
  18 test_data_dir = os.path.abspath(os.path.join(
  19     os.path.dirname(__file__), '..', '..', 'data', 'gpu'))
  20
  21 default_generated_data_dir = os.path.join(test_data_dir, 'generated')
  22
  23 error_image_cloud_storage_bucket = 'chromium-browser-gpu-tests'
  24
  25 def _CompareScreenshotSamples(screenshot, expectations, device_pixel_ratio):
  26   for expectation in expectations:
  27     location = expectation["location"]
  28     x = int(location[0] * device_pixel_ratio)
  29     y = int(location[1] * device_pixel_ratio)
  30
  31     if x < 0 or y < 0 or x > screenshot.width or y > screenshot.height:
  32       raise page_test.Failure(
  33           'Expected pixel location [%d, %d] is out of range on [%d, %d] image' %
  34           (x, y, screenshot.width, screenshot.height))
  35
  36     actual_color = screenshot.GetPixelColor(x, y)
  37     expected_color = bitmap.RgbaColor(
  38         expectation["color"][0],
  39         expectation["color"][1],
  40         expectation["color"][2])
  41     if not actual_color.IsEqual(expected_color, expectation["tolerance"]):
  42       raise page_test.Failure('Expected pixel at ' + str(location) +
  43           ' to be ' +
  44           str(expectation["color"]) + " but got [" +
  45           str(actual_color.r) + ", " +
  46           str(actual_color.g) + ", " +
  47           str(actual_color.b) + "]")
  48
  49 class ValidatorBase(page_test.PageTest):
  50   def __init__(self):
  51     super(ValidatorBase, self).__init__()
  52     # Parameters for cloud storage reference images.
  53     self.vendor_id = None
  54     self.device_id = None
  55     self.vendor_string = None
  56     self.device_string = None
  57     self.msaa = False
  58
  59   ###
  60   ### Routines working with the local disk (only used for local
  61   ### testing without a cloud storage account -- the bots do not use
  62   ### this code path).
  63   ###
  64
  65   def _UrlToImageName(self, url):
  66     image_name = re.sub(r'^(http|https|file)://(/*)', '', url)
  67     image_name = re.sub(r'\.\./', '', image_name)
  68     image_name = re.sub(r'(\.|/|-)', '_', image_name)
  69     return image_name
  70
  71   def _WriteImage(self, image_path, png_image):
  72     output_dir = os.path.dirname(image_path)
  73     if not os.path.exists(output_dir):
  74       os.makedirs(output_dir)
  75     png_image.WritePngFile(image_path)
  76
  77   def _WriteErrorImages(self, img_dir, img_name, screenshot, ref_png):
  78     full_image_name = img_name + '_' + str(self.options.build_revision)
  79     full_image_name = full_image_name + '.png'
  80
  81     # Always write the failing image.
  82     self._WriteImage(
  83         os.path.join(img_dir, 'FAIL_' + full_image_name), screenshot)
  84
  85     if ref_png:
  86       # Save the reference image.
  87       # This ensures that we get the right revision number.
  88       self._WriteImage(
  89           os.path.join(img_dir, full_image_name), ref_png)
  90
  91       # Save the difference image.
  92       diff_png = screenshot.Diff(ref_png)
  93       self._WriteImage(
  94           os.path.join(img_dir, 'DIFF_' + full_image_name), diff_png)
  95
  96   ###
  97   ### Cloud storage code path -- the bots use this.
  98   ###
  99
 100   def _ComputeGpuInfo(self, tab):
 101     if ((self.vendor_id and self.device_id) or
 102         (self.vendor_string and self.device_string)):
 103       return
 104     browser = tab.browser
 105     if not browser.supports_system_info:
 106       raise Exception('System info must be supported by the browser')
 107     system_info = browser.GetSystemInfo()
 108     if not system_info.gpu:
 109       raise Exception('GPU information was absent')
 110     device = system_info.gpu.devices[0]
 111     if device.vendor_id and device.device_id:
 112       self.vendor_id = device.vendor_id
 113       self.device_id = device.device_id
 114     elif device.vendor_string and device.device_string:
 115       self.vendor_string = device.vendor_string
 116       self.device_string = device.device_string
 117     else:
 118       raise Exception('GPU device information was incomplete')
 119     self.msaa = not (
 120         'disable_multisampling' in system_info.gpu.driver_bug_workarounds)
 121
 122   def _FormatGpuInfo(self, tab):
 123     self._ComputeGpuInfo(tab)
 124     msaa_string = '_msaa' if self.msaa else '_non_msaa'
 125     if self.vendor_id:
 126       return '%s_%04x_%04x%s' % (
 127         self.options.os_type, self.vendor_id, self.device_id, msaa_string)
 128     else:
 129       return '%s_%s_%s%s' % (
 130         self.options.os_type, self.vendor_string, self.device_string,
 131         msaa_string)
 132
 133   def _FormatReferenceImageName(self, img_name, page, tab):
 134     return '%s_v%s_%s.png' % (
 135       img_name,
 136       page.revision,
 137       self._FormatGpuInfo(tab))
 138
 139   def _UploadBitmapToCloudStorage(self, bucket, name, bitmap, public=False):
 140     # This sequence of steps works on all platforms to write a temporary
 141     # PNG to disk, following the pattern in bitmap_unittest.py. The key to
 142     # avoiding PermissionErrors seems to be to not actually try to write to
 143     # the temporary file object, but to re-open its name for all operations.
 144     temp_file = tempfile.NamedTemporaryFile().name
 145     bitmap.WritePngFile(temp_file)
 146     cloud_storage.Insert(bucket, name, temp_file, publicly_readable=public)
 147
 148   def _ConditionallyUploadToCloudStorage(self, img_name, page, tab, screenshot):
 149     """Uploads the screenshot to cloud storage as the reference image
 150     for this test, unless it already exists. Returns True if the
 151     upload was actually performed."""
 152     if not self.options.refimg_cloud_storage_bucket:
 153       raise Exception('--refimg-cloud-storage-bucket argument is required')
 154     cloud_name = self._FormatReferenceImageName(img_name, page, tab)
 155     if not cloud_storage.Exists(self.options.refimg_cloud_storage_bucket,
 156                                 cloud_name):
 157       self._UploadBitmapToCloudStorage(self.options.refimg_cloud_storage_bucket,
 158                                        cloud_name,
 159                                        screenshot)
 160       return True
 161     return False
 162
 163   def _DownloadFromCloudStorage(self, img_name, page, tab):
 164     """Downloads the reference image for the given test from cloud
 165     storage, returning it as a Telemetry Bitmap object."""
 166     # TODO(kbr): there's a race condition between the deletion of the
 167     # temporary file and gsutil's overwriting it.
 168     if not self.options.refimg_cloud_storage_bucket:
 169       raise Exception('--refimg-cloud-storage-bucket argument is required')
 170     temp_file = tempfile.NamedTemporaryFile().name
 171     cloud_storage.Get(self.options.refimg_cloud_storage_bucket,
 172                       self._FormatReferenceImageName(img_name, page, tab),
 173                       temp_file)
 174     return bitmap.Bitmap.FromPngFile(temp_file)
 175
 176   def _UploadErrorImagesToCloudStorage(self, image_name, screenshot, ref_img):
 177     """For a failing run, uploads the failing image, reference image (if
 178     supplied), and diff image (if reference image was supplied) to cloud
 179     storage. This subsumes the functionality of the
 180     archive_gpu_pixel_test_results.py script."""
 181     machine_name = re.sub('\W+', '_', self.options.test_machine_name)
 182     upload_dir = '%s_%s_telemetry' % (self.options.build_revision, machine_name)
 183     base_bucket = '%s/runs/%s' % (error_image_cloud_storage_bucket, upload_dir)
 184     image_name_with_revision = '%s_%s.png' % (
 185       image_name, self.options.build_revision)
 186     self._UploadBitmapToCloudStorage(
 187       base_bucket + '/gen', image_name_with_revision, screenshot,
 188       public=True)
 189     if ref_img:
 190       self._UploadBitmapToCloudStorage(
 191         base_bucket + '/ref', image_name_with_revision, ref_img, public=True)
 192       diff_img = screenshot.Diff(ref_img)
 193       self._UploadBitmapToCloudStorage(
 194         base_bucket + '/diff', image_name_with_revision, diff_img,
 195         public=True)
 196     print ('See http://%s.commondatastorage.googleapis.com/'
 197            'view_test_results.html?%s for this run\'s test results') % (
 198       error_image_cloud_storage_bucket, upload_dir)
 199
 200   def _ValidateScreenshotSamples(self, url,
 201                                  screenshot, expectations, device_pixel_ratio):
 202     """Samples the given screenshot and verifies pixel color values.
 203        The sample locations and expected color values are given in expectations.
 204        In case any of the samples do not match the expected color, it raises
 205        a Failure and dumps the screenshot locally or cloud storage depending on
 206        what machine the test is being run."""
 207     try:
 208       _CompareScreenshotSamples(screenshot, expectations, device_pixel_ratio)
 209     except page_test.Failure:
 210       image_name = self._UrlToImageName(url)
 211       if self.options.test_machine_name:
 212         self._UploadErrorImagesToCloudStorage(image_name, screenshot, None)
 213       else:
 214         self._WriteErrorImages(self.options.generated_dir, image_name,
 215                                screenshot, None)
 216       raise
 217
 218
 219 class TestBase(benchmark.Benchmark):
 220   @classmethod
 221   def AddBenchmarkCommandLineArgs(cls, group):
 222     group.add_option('--build-revision',
 223         help='Chrome revision being tested.',
 224         default="unknownrev")
 225     group.add_option('--upload-refimg-to-cloud-storage',
 226         dest='upload_refimg_to_cloud_storage',
 227         action='store_true', default=False,
 228         help='Upload resulting images to cloud storage as reference images')
 229     group.add_option('--download-refimg-from-cloud-storage',
 230         dest='download_refimg_from_cloud_storage',
 231         action='store_true', default=False,
 232         help='Download reference images from cloud storage')
 233     group.add_option('--refimg-cloud-storage-bucket',
 234         help='Name of the cloud storage bucket to use for reference images; '
 235         'required with --upload-refimg-to-cloud-storage and '
 236         '--download-refimg-from-cloud-storage. Example: '
 237         '"chromium-gpu-archive/reference-images"')
 238     group.add_option('--os-type',
 239         help='Type of operating system on which the pixel test is being run, '
 240         'used only to distinguish different operating systems with the same '
 241         'graphics card. Any value is acceptable, but canonical values are '
 242         '"win", "mac", and "linux", and probably, eventually, "chromeos" '
 243         'and "android").',
 244         default='')
 245     group.add_option('--test-machine-name',
 246         help='Name of the test machine. Specifying this argument causes this '
 247         'script to upload failure images and diffs to cloud storage directly, '
 248         'instead of relying on the archive_gpu_pixel_test_results.py script.',
 249         default='')
 250     group.add_option('--generated-dir',
 251         help='Overrides the default on-disk location for generated test images '
 252         '(only used for local testing without a cloud storage account)',
 253         default=default_generated_data_dir)