content/test/gpu/gpu_tests/cloud_storage_test_base.py

   1 # Copyright 2013 The Chromium Authors. All rights reserved.
   2 # Use of this source code is governed by a BSD-style license that can be
   3 # found in the LICENSE file.
   4
   5 """Base classes for a test and validator which upload results
   6 (reference images, error images) to cloud storage."""
   7
   8 import os
   9 import re
  10 import tempfile
  11
  12 from catapult_base import cloud_storage
  13 from telemetry.page import page_test
  14 from telemetry.util import image_util
  15 from telemetry.util import rgba_color
  16
  17 import gpu_test_base
  18
  19 test_data_dir = os.path.abspath(os.path.join(
  20     os.path.dirname(__file__), '..', '..', 'data', 'gpu'))
  21
  22 default_generated_data_dir = os.path.join(test_data_dir, 'generated')
  23
  24 error_image_cloud_storage_bucket = 'chromium-browser-gpu-tests'
  25
  26 def _CompareScreenshotSamples(screenshot, expectations, device_pixel_ratio):
  27   for expectation in expectations:
  28     location = expectation["location"]
  29     x = int(location[0] * device_pixel_ratio)
  30     y = int(location[1] * device_pixel_ratio)
  31
  32     if (x < 0 or y < 0 or x > image_util.Width(screenshot) or
  33         y > image_util.Height(screenshot)):
  34       raise page_test.Failure(
  35           'Expected pixel location [%d, %d] is out of range on [%d, %d] image' %
  36           (x, y, image_util.Width(screenshot), image_util.Height(screenshot)))
  37
  38     actual_color = image_util.GetPixelColor(screenshot, x, y)
  39     expected_color = rgba_color.RgbaColor(
  40         expectation["color"][0],
  41         expectation["color"][1],
  42         expectation["color"][2])
  43     if not actual_color.IsEqual(expected_color, expectation["tolerance"]):
  44       raise page_test.Failure('Expected pixel at ' + str(location) +
  45           ' to be ' +
  46           str(expectation["color"]) + " but got [" +
  47           str(actual_color.r) + ", " +
  48           str(actual_color.g) + ", " +
  49           str(actual_color.b) + "]")
  50
  51 class ValidatorBase(gpu_test_base.ValidatorBase):
  52   def __init__(self):
  53     super(ValidatorBase, self).__init__()
  54     # Parameters for cloud storage reference images.
  55     self.vendor_id = None
  56     self.device_id = None
  57     self.vendor_string = None
  58     self.device_string = None
  59     self.msaa = False
  60
  61   ###
  62   ### Routines working with the local disk (only used for local
  63   ### testing without a cloud storage account -- the bots do not use
  64   ### this code path).
  65   ###
  66
  67   def _UrlToImageName(self, url):
  68     image_name = re.sub(r'^(http|https|file)://(/*)', '', url)
  69     image_name = re.sub(r'\.\./', '', image_name)
  70     image_name = re.sub(r'(\.|/|-)', '_', image_name)
  71     return image_name
  72
  73   def _WriteImage(self, image_path, png_image):
  74     output_dir = os.path.dirname(image_path)
  75     if not os.path.exists(output_dir):
  76       os.makedirs(output_dir)
  77     image_util.WritePngFile(png_image, image_path)
  78
  79   def _WriteErrorImages(self, img_dir, img_name, screenshot, ref_png):
  80     full_image_name = img_name + '_' + str(self.options.build_revision)
  81     full_image_name = full_image_name + '.png'
  82
  83     # Always write the failing image.
  84     self._WriteImage(
  85         os.path.join(img_dir, 'FAIL_' + full_image_name), screenshot)
  86
  87     if ref_png is not None:
  88       # Save the reference image.
  89       # This ensures that we get the right revision number.
  90       self._WriteImage(
  91           os.path.join(img_dir, full_image_name), ref_png)
  92
  93       # Save the difference image.
  94       diff_png = image_util.Diff(screenshot, ref_png)
  95       self._WriteImage(
  96           os.path.join(img_dir, 'DIFF_' + full_image_name), diff_png)
  97
  98   ###
  99   ### Cloud storage code path -- the bots use this.
 100   ###
 101
 102   def _ComputeGpuInfo(self, tab):
 103     if ((self.vendor_id and self.device_id) or
 104         (self.vendor_string and self.device_string)):
 105       return
 106     browser = tab.browser
 107     if not browser.supports_system_info:
 108       raise Exception('System info must be supported by the browser')
 109     system_info = browser.GetSystemInfo()
 110     if not system_info.gpu:
 111       raise Exception('GPU information was absent')
 112     device = system_info.gpu.devices[0]
 113     if device.vendor_id and device.device_id:
 114       self.vendor_id = device.vendor_id
 115       self.device_id = device.device_id
 116     elif device.vendor_string and device.device_string:
 117       self.vendor_string = device.vendor_string
 118       self.device_string = device.device_string
 119     else:
 120       raise Exception('GPU device information was incomplete')
 121     # TODO(senorblanco): This should probably be checking
 122     # for the presence of the extensions in system_info.gpu_aux_attributes
 123     # in order to check for MSAA, rather than sniffing the blacklist.
 124     self.msaa = not (
 125         ('disable_chromium_framebuffer_multisample' in
 126           system_info.gpu.driver_bug_workarounds) or
 127         ('disable_multisample_render_to_texture' in
 128           system_info.gpu.driver_bug_workarounds))
 129
 130   def _FormatGpuInfo(self, tab):
 131     self._ComputeGpuInfo(tab)
 132     msaa_string = '_msaa' if self.msaa else '_non_msaa'
 133     if self.vendor_id:
 134       return '%s_%04x_%04x%s' % (
 135         self.options.os_type, self.vendor_id, self.device_id, msaa_string)
 136     else:
 137       return '%s_%s_%s%s' % (
 138         self.options.os_type, self.vendor_string, self.device_string,
 139         msaa_string)
 140
 141   def _FormatReferenceImageName(self, img_name, page, tab):
 142     return '%s_v%s_%s.png' % (
 143       img_name,
 144       page.revision,
 145       self._FormatGpuInfo(tab))
 146
 147   def _UploadBitmapToCloudStorage(self, bucket, name, bitmap, public=False):
 148     # This sequence of steps works on all platforms to write a temporary
 149     # PNG to disk, following the pattern in bitmap_unittest.py. The key to
 150     # avoiding PermissionErrors seems to be to not actually try to write to
 151     # the temporary file object, but to re-open its name for all operations.
 152     temp_file = tempfile.NamedTemporaryFile(suffix='.png').name
 153     image_util.WritePngFile(bitmap, temp_file)
 154     cloud_storage.Insert(bucket, name, temp_file, publicly_readable=public)
 155
 156   def _ConditionallyUploadToCloudStorage(self, img_name, page, tab, screenshot):
 157     """Uploads the screenshot to cloud storage as the reference image
 158     for this test, unless it already exists. Returns True if the
 159     upload was actually performed."""
 160     if not self.options.refimg_cloud_storage_bucket:
 161       raise Exception('--refimg-cloud-storage-bucket argument is required')
 162     cloud_name = self._FormatReferenceImageName(img_name, page, tab)
 163     if not cloud_storage.Exists(self.options.refimg_cloud_storage_bucket,
 164                                 cloud_name):
 165       self._UploadBitmapToCloudStorage(self.options.refimg_cloud_storage_bucket,
 166                                        cloud_name,
 167                                        screenshot)
 168       return True
 169     return False
 170
 171   def _DownloadFromCloudStorage(self, img_name, page, tab):
 172     """Downloads the reference image for the given test from cloud
 173     storage, returning it as a Telemetry Bitmap object."""
 174     # TODO(kbr): there's a race condition between the deletion of the
 175     # temporary file and gsutil's overwriting it.
 176     if not self.options.refimg_cloud_storage_bucket:
 177       raise Exception('--refimg-cloud-storage-bucket argument is required')
 178     temp_file = tempfile.NamedTemporaryFile(suffix='.png').name
 179     cloud_storage.Get(self.options.refimg_cloud_storage_bucket,
 180                       self._FormatReferenceImageName(img_name, page, tab),
 181                       temp_file)
 182     return image_util.FromPngFile(temp_file)
 183
 184   def _UploadErrorImagesToCloudStorage(self, image_name, screenshot, ref_img):
 185     """For a failing run, uploads the failing image, reference image (if
 186     supplied), and diff image (if reference image was supplied) to cloud
 187     storage. This subsumes the functionality of the
 188     archive_gpu_pixel_test_results.py script."""
 189     machine_name = re.sub('\W+', '_', self.options.test_machine_name)
 190     upload_dir = '%s_%s_telemetry' % (self.options.build_revision, machine_name)
 191     base_bucket = '%s/runs/%s' % (error_image_cloud_storage_bucket, upload_dir)
 192     image_name_with_revision = '%s_%s.png' % (
 193       image_name, self.options.build_revision)
 194     self._UploadBitmapToCloudStorage(
 195       base_bucket + '/gen', image_name_with_revision, screenshot,
 196       public=True)
 197     if ref_img is not None:
 198       self._UploadBitmapToCloudStorage(
 199         base_bucket + '/ref', image_name_with_revision, ref_img, public=True)
 200       diff_img = image_util.Diff(screenshot, ref_img)
 201       self._UploadBitmapToCloudStorage(
 202         base_bucket + '/diff', image_name_with_revision, diff_img,
 203         public=True)
 204     print ('See http://%s.commondatastorage.googleapis.com/'
 205            'view_test_results.html?%s for this run\'s test results') % (
 206       error_image_cloud_storage_bucket, upload_dir)
 207
 208   def _ValidateScreenshotSamples(self, url,
 209                                  screenshot, expectations, device_pixel_ratio):
 210     """Samples the given screenshot and verifies pixel color values.
 211        The sample locations and expected color values are given in expectations.
 212        In case any of the samples do not match the expected color, it raises
 213        a Failure and dumps the screenshot locally or cloud storage depending on
 214        what machine the test is being run."""
 215     try:
 216       _CompareScreenshotSamples(screenshot, expectations, device_pixel_ratio)
 217     except page_test.Failure:
 218       image_name = self._UrlToImageName(url)
 219       if self.options.test_machine_name:
 220         self._UploadErrorImagesToCloudStorage(image_name, screenshot, None)
 221       else:
 222         self._WriteErrorImages(self.options.generated_dir, image_name,
 223                                screenshot, None)
 224       raise
 225
 226
 227 class TestBase(gpu_test_base.TestBase):
 228   @classmethod
 229   def AddBenchmarkCommandLineArgs(cls, group):
 230     group.add_option('--build-revision',
 231         help='Chrome revision being tested.',
 232         default="unknownrev")
 233     group.add_option('--upload-refimg-to-cloud-storage',
 234         dest='upload_refimg_to_cloud_storage',
 235         action='store_true', default=False,
 236         help='Upload resulting images to cloud storage as reference images')
 237     group.add_option('--download-refimg-from-cloud-storage',
 238         dest='download_refimg_from_cloud_storage',
 239         action='store_true', default=False,
 240         help='Download reference images from cloud storage')
 241     group.add_option('--refimg-cloud-storage-bucket',
 242         help='Name of the cloud storage bucket to use for reference images; '
 243         'required with --upload-refimg-to-cloud-storage and '
 244         '--download-refimg-from-cloud-storage. Example: '
 245         '"chromium-gpu-archive/reference-images"')
 246     group.add_option('--os-type',
 247         help='Type of operating system on which the pixel test is being run, '
 248         'used only to distinguish different operating systems with the same '
 249         'graphics card. Any value is acceptable, but canonical values are '
 250         '"win", "mac", and "linux", and probably, eventually, "chromeos" '
 251         'and "android").',
 252         default='')
 253     group.add_option('--test-machine-name',
 254         help='Name of the test machine. Specifying this argument causes this '
 255         'script to upload failure images and diffs to cloud storage directly, '
 256         'instead of relying on the archive_gpu_pixel_test_results.py script.',
 257         default='')
 258     group.add_option('--generated-dir',
 259         help='Overrides the default on-disk location for generated test images '
 260         '(only used for local testing without a cloud storage account)',
 261         default=default_generated_data_dir)