content/test/gpu/gpu_tests/cloud_storage_test_base.py

   1 # Copyright 2013 The Chromium Authors. All rights reserved.
   2 # Use of this source code is governed by a BSD-style license that can be
   3 # found in the LICENSE file.
   4
   5 """Base classes for a test and validator which upload results
   6 (reference images, error images) to cloud storage."""
   7
   8 import os
   9 import re
  10 import tempfile
  11
  12 from catapult_base import cloud_storage
  13 from telemetry.page import page_test
  14 from telemetry.util import image_util
  15 from telemetry.util import rgba_color
  16
  17 import gpu_test_base
  18
  19 test_data_dir = os.path.abspath(os.path.join(
  20     os.path.dirname(__file__), '..', '..', 'data', 'gpu'))
  21
  22 default_generated_data_dir = os.path.join(test_data_dir, 'generated')
  23
  24 error_image_cloud_storage_bucket = 'chromium-browser-gpu-tests'
  25
  26 def _CompareScreenshotSamples(screenshot, expectations, device_pixel_ratio):
  27   for expectation in expectations:
  28     location = expectation["location"]
  29     size = expectation["size"]
  30     x0 = int(location[0] * device_pixel_ratio)
  31     x1 = int((location[0] + size[0]) * device_pixel_ratio)
  32     y0 = int(location[1] * device_pixel_ratio)
  33     y1 = int((location[1] + size[1]) * device_pixel_ratio)
  34     for x in range(x0, x1):
  35       for y in range(y0, y1):
  36         if (x < 0 or y < 0 or x >= image_util.Width(screenshot) or
  37             y >= image_util.Height(screenshot)):
  38           raise page_test.Failure(
  39               ('Expected pixel location [%d, %d] is out of range on ' +
  40                '[%d, %d] image') %
  41               (x, y, image_util.Width(screenshot),
  42                image_util.Height(screenshot)))
  43
  44         actual_color = image_util.GetPixelColor(screenshot, x, y)
  45         expected_color = rgba_color.RgbaColor(
  46             expectation["color"][0],
  47             expectation["color"][1],
  48             expectation["color"][2])
  49         if not actual_color.IsEqual(expected_color, expectation["tolerance"]):
  50           raise page_test.Failure('Expected pixel at ' + str(location) +
  51               ' to be ' +
  52               str(expectation["color"]) + " but got [" +
  53               str(actual_color.r) + ", " +
  54               str(actual_color.g) + ", " +
  55               str(actual_color.b) + "]")
  56
  57 class ValidatorBase(gpu_test_base.ValidatorBase):
  58   def __init__(self):
  59     super(ValidatorBase, self).__init__()
  60     # Parameters for cloud storage reference images.
  61     self.vendor_id = None
  62     self.device_id = None
  63     self.vendor_string = None
  64     self.device_string = None
  65     self.msaa = False
  66
  67   ###
  68   ### Routines working with the local disk (only used for local
  69   ### testing without a cloud storage account -- the bots do not use
  70   ### this code path).
  71   ###
  72
  73   def _UrlToImageName(self, url):
  74     image_name = re.sub(r'^(http|https|file)://(/*)', '', url)
  75     image_name = re.sub(r'\.\./', '', image_name)
  76     image_name = re.sub(r'(\.|/|-)', '_', image_name)
  77     return image_name
  78
  79   def _WriteImage(self, image_path, png_image):
  80     output_dir = os.path.dirname(image_path)
  81     if not os.path.exists(output_dir):
  82       os.makedirs(output_dir)
  83     image_util.WritePngFile(png_image, image_path)
  84
  85   def _WriteErrorImages(self, img_dir, img_name, screenshot, ref_png):
  86     full_image_name = img_name + '_' + str(self.options.build_revision)
  87     full_image_name = full_image_name + '.png'
  88
  89     # Always write the failing image.
  90     self._WriteImage(
  91         os.path.join(img_dir, 'FAIL_' + full_image_name), screenshot)
  92
  93     if ref_png is not None:
  94       # Save the reference image.
  95       # This ensures that we get the right revision number.
  96       self._WriteImage(
  97           os.path.join(img_dir, full_image_name), ref_png)
  98
  99       # Save the difference image.
 100       diff_png = image_util.Diff(screenshot, ref_png)
 101       self._WriteImage(
 102           os.path.join(img_dir, 'DIFF_' + full_image_name), diff_png)
 103
 104   ###
 105   ### Cloud storage code path -- the bots use this.
 106   ###
 107
 108   def _ComputeGpuInfo(self, tab):
 109     if ((self.vendor_id and self.device_id) or
 110         (self.vendor_string and self.device_string)):
 111       return
 112     browser = tab.browser
 113     if not browser.supports_system_info:
 114       raise Exception('System info must be supported by the browser')
 115     system_info = browser.GetSystemInfo()
 116     if not system_info.gpu:
 117       raise Exception('GPU information was absent')
 118     device = system_info.gpu.devices[0]
 119     if device.vendor_id and device.device_id:
 120       self.vendor_id = device.vendor_id
 121       self.device_id = device.device_id
 122     elif device.vendor_string and device.device_string:
 123       self.vendor_string = device.vendor_string
 124       self.device_string = device.device_string
 125     else:
 126       raise Exception('GPU device information was incomplete')
 127     # TODO(senorblanco): This should probably be checking
 128     # for the presence of the extensions in system_info.gpu_aux_attributes
 129     # in order to check for MSAA, rather than sniffing the blacklist.
 130     self.msaa = not (
 131         ('disable_chromium_framebuffer_multisample' in
 132           system_info.gpu.driver_bug_workarounds) or
 133         ('disable_multisample_render_to_texture' in
 134           system_info.gpu.driver_bug_workarounds))
 135
 136   def _FormatGpuInfo(self, tab):
 137     self._ComputeGpuInfo(tab)
 138     msaa_string = '_msaa' if self.msaa else '_non_msaa'
 139     if self.vendor_id:
 140       return '%s_%04x_%04x%s' % (
 141         self.options.os_type, self.vendor_id, self.device_id, msaa_string)
 142     else:
 143       return '%s_%s_%s%s' % (
 144         self.options.os_type, self.vendor_string, self.device_string,
 145         msaa_string)
 146
 147   def _FormatReferenceImageName(self, img_name, page, tab):
 148     return '%s_v%s_%s.png' % (
 149       img_name,
 150       page.revision,
 151       self._FormatGpuInfo(tab))
 152
 153   def _UploadBitmapToCloudStorage(self, bucket, name, bitmap, public=False):
 154     # This sequence of steps works on all platforms to write a temporary
 155     # PNG to disk, following the pattern in bitmap_unittest.py. The key to
 156     # avoiding PermissionErrors seems to be to not actually try to write to
 157     # the temporary file object, but to re-open its name for all operations.
 158     temp_file = tempfile.NamedTemporaryFile(suffix='.png').name
 159     image_util.WritePngFile(bitmap, temp_file)
 160     cloud_storage.Insert(bucket, name, temp_file, publicly_readable=public)
 161
 162   def _ConditionallyUploadToCloudStorage(self, img_name, page, tab, screenshot):
 163     """Uploads the screenshot to cloud storage as the reference image
 164     for this test, unless it already exists. Returns True if the
 165     upload was actually performed."""
 166     if not self.options.refimg_cloud_storage_bucket:
 167       raise Exception('--refimg-cloud-storage-bucket argument is required')
 168     cloud_name = self._FormatReferenceImageName(img_name, page, tab)
 169     if not cloud_storage.Exists(self.options.refimg_cloud_storage_bucket,
 170                                 cloud_name):
 171       self._UploadBitmapToCloudStorage(self.options.refimg_cloud_storage_bucket,
 172                                        cloud_name,
 173                                        screenshot)
 174       return True
 175     return False
 176
 177   def _DownloadFromCloudStorage(self, img_name, page, tab):
 178     """Downloads the reference image for the given test from cloud
 179     storage, returning it as a Telemetry Bitmap object."""
 180     # TODO(kbr): there's a race condition between the deletion of the
 181     # temporary file and gsutil's overwriting it.
 182     if not self.options.refimg_cloud_storage_bucket:
 183       raise Exception('--refimg-cloud-storage-bucket argument is required')
 184     temp_file = tempfile.NamedTemporaryFile(suffix='.png').name
 185     cloud_storage.Get(self.options.refimg_cloud_storage_bucket,
 186                       self._FormatReferenceImageName(img_name, page, tab),
 187                       temp_file)
 188     return image_util.FromPngFile(temp_file)
 189
 190   def _UploadErrorImagesToCloudStorage(self, image_name, screenshot, ref_img):
 191     """For a failing run, uploads the failing image, reference image (if
 192     supplied), and diff image (if reference image was supplied) to cloud
 193     storage. This subsumes the functionality of the
 194     archive_gpu_pixel_test_results.py script."""
 195     machine_name = re.sub('\W+', '_', self.options.test_machine_name)
 196     upload_dir = '%s_%s_telemetry' % (self.options.build_revision, machine_name)
 197     base_bucket = '%s/runs/%s' % (error_image_cloud_storage_bucket, upload_dir)
 198     image_name_with_revision = '%s_%s.png' % (
 199       image_name, self.options.build_revision)
 200     self._UploadBitmapToCloudStorage(
 201       base_bucket + '/gen', image_name_with_revision, screenshot,
 202       public=True)
 203     if ref_img is not None:
 204       self._UploadBitmapToCloudStorage(
 205         base_bucket + '/ref', image_name_with_revision, ref_img, public=True)
 206       diff_img = image_util.Diff(screenshot, ref_img)
 207       self._UploadBitmapToCloudStorage(
 208         base_bucket + '/diff', image_name_with_revision, diff_img,
 209         public=True)
 210     print ('See http://%s.commondatastorage.googleapis.com/'
 211            'view_test_results.html?%s for this run\'s test results') % (
 212       error_image_cloud_storage_bucket, upload_dir)
 213
 214   def _ValidateScreenshotSamples(self, url,
 215                                  screenshot, expectations, device_pixel_ratio):
 216     """Samples the given screenshot and verifies pixel color values.
 217        The sample locations and expected color values are given in expectations.
 218        In case any of the samples do not match the expected color, it raises
 219        a Failure and dumps the screenshot locally or cloud storage depending on
 220        what machine the test is being run."""
 221     try:
 222       _CompareScreenshotSamples(screenshot, expectations, device_pixel_ratio)
 223     except page_test.Failure:
 224       image_name = self._UrlToImageName(url)
 225       if self.options.test_machine_name:
 226         self._UploadErrorImagesToCloudStorage(image_name, screenshot, None)
 227       else:
 228         self._WriteErrorImages(self.options.generated_dir, image_name,
 229                                screenshot, None)
 230       raise
 231
 232
 233 class TestBase(gpu_test_base.TestBase):
 234   @classmethod
 235   def AddBenchmarkCommandLineArgs(cls, group):
 236     group.add_option('--build-revision',
 237         help='Chrome revision being tested.',
 238         default="unknownrev")
 239     group.add_option('--upload-refimg-to-cloud-storage',
 240         dest='upload_refimg_to_cloud_storage',
 241         action='store_true', default=False,
 242         help='Upload resulting images to cloud storage as reference images')
 243     group.add_option('--download-refimg-from-cloud-storage',
 244         dest='download_refimg_from_cloud_storage',
 245         action='store_true', default=False,
 246         help='Download reference images from cloud storage')
 247     group.add_option('--refimg-cloud-storage-bucket',
 248         help='Name of the cloud storage bucket to use for reference images; '
 249         'required with --upload-refimg-to-cloud-storage and '
 250         '--download-refimg-from-cloud-storage. Example: '
 251         '"chromium-gpu-archive/reference-images"')
 252     group.add_option('--os-type',
 253         help='Type of operating system on which the pixel test is being run, '
 254         'used only to distinguish different operating systems with the same '
 255         'graphics card. Any value is acceptable, but canonical values are '
 256         '"win", "mac", and "linux", and probably, eventually, "chromeos" '
 257         'and "android").',
 258         default='')
 259     group.add_option('--test-machine-name',
 260         help='Name of the test machine. Specifying this argument causes this '
 261         'script to upload failure images and diffs to cloud storage directly, '
 262         'instead of relying on the archive_gpu_pixel_test_results.py script.',
 263         default='')
 264     group.add_option('--generated-dir',
 265         help='Overrides the default on-disk location for generated test images '
 266         '(only used for local testing without a cloud storage account)',
 267         default=default_generated_data_dir)