1 # Copyright (c) 2012 The Chromium Authors. All rights reserved.
2 # Use of this source code is governed by a BSD-style license that can be
3 # found in the LICENSE file.
4 from datetime
import datetime
10 import cloud_storage_test_base
12 import pixel_expectations
14 from telemetry
import benchmark
15 from telemetry
.core
import bitmap
16 from telemetry
.page
import page_test
17 from telemetry
.util
import cloud_storage
20 test_data_dir
= os
.path
.abspath(os
.path
.join(
21 os
.path
.dirname(__file__
), '..', '..', 'data', 'gpu'))
23 default_reference_image_dir
= os
.path
.join(test_data_dir
, 'gpu_reference')
25 test_harness_script
= r
"""
26 var domAutomationController = {};
28 domAutomationController._succeeded = false;
29 domAutomationController._finished = false;
31 domAutomationController.setAutomationId = function(id) {}
33 domAutomationController.send = function(msg) {
34 domAutomationController._finished = true;
36 if(msg.toLowerCase() == "success") {
37 domAutomationController._succeeded = true;
39 domAutomationController._succeeded = false;
43 window.domAutomationController = domAutomationController;
46 class PixelTestFailure(Exception):
49 def _DidTestSucceed(tab
):
50 return tab
.EvaluateJavaScript('domAutomationController._succeeded')
52 class _PixelValidator(cloud_storage_test_base
.ValidatorBase
):
53 def CustomizeBrowserOptions(self
, options
):
54 options
.AppendExtraBrowserArgs('--enable-gpu-benchmarking')
56 def ValidateAndMeasurePage(self
, page
, tab
, results
):
57 if not _DidTestSucceed(tab
):
58 raise page_test
.Failure('Page indicated a failure')
60 if not tab
.screenshot_supported
:
61 raise page_test
.Failure('Browser does not support screenshot capture')
63 screenshot
= tab
.Screenshot(5)
66 raise page_test
.Failure('Could not capture screenshot')
68 if hasattr(page
, 'test_rect'):
69 screenshot
= screenshot
.Crop(
70 page
.test_rect
[0], page
.test_rect
[1],
71 page
.test_rect
[2], page
.test_rect
[3])
73 image_name
= self
._UrlToImageName
(page
.display_name
)
75 if self
.options
.upload_refimg_to_cloud_storage
:
76 if self
._ConditionallyUploadToCloudStorage
(image_name
, page
, tab
,
78 # This is the new reference image; there's nothing to compare against.
81 # There was a preexisting reference image, so we might as well
83 ref_png
= self
._DownloadFromCloudStorage
(image_name
, page
, tab
)
84 elif self
.options
.download_refimg_from_cloud_storage
:
85 # This bot doesn't have the ability to properly generate a
86 # reference image, so download it from cloud storage.
88 ref_png
= self
._DownloadFromCloudStorage
(image_name
, page
, tab
)
89 except cloud_storage
.NotFoundError
as e
:
90 # There is no reference image yet in cloud storage. This
91 # happens when the revision of the test is incremented or when
92 # a new test is added, because the trybots are not allowed to
93 # produce reference images, only the bots on the main
94 # waterfalls. Report this as a failure so the developer has to
95 # take action by explicitly suppressing the failure and
96 # removing the suppression once the reference images have been
97 # generated. Otherwise silent failures could happen for long
99 raise page_test
.Failure('Could not find image %s in cloud storage' %
102 # Legacy path using on-disk results.
103 ref_png
= self
._GetReferenceImage
(self
.options
.reference_dir
,
104 image_name
, page
.revision
, screenshot
)
106 # Test new snapshot against existing reference image
107 if not ref_png
.IsEqual(screenshot
, tolerance
=2):
108 if self
.options
.test_machine_name
:
109 self
._UploadErrorImagesToCloudStorage
(image_name
, screenshot
, ref_png
)
111 self
._WriteErrorImages
(self
.options
.generated_dir
, image_name
,
113 raise page_test
.Failure('Reference image did not match captured screen')
115 def _DeleteOldReferenceImages(self
, ref_image_path
, cur_revision
):
119 old_revisions
= glob
.glob(ref_image_path
+ "_*.png")
120 for rev_path
in old_revisions
:
121 m
= re
.match(r
'^.*_(\d+)\.png$', rev_path
)
122 if m
and int(m
.group(1)) < cur_revision
:
123 print 'Found deprecated reference image. Deleting rev ' + m
.group(1)
126 def _GetReferenceImage(self
, img_dir
, img_name
, cur_revision
, screenshot
):
130 image_path
= os
.path
.join(img_dir
, img_name
)
132 self
._DeleteOldReferenceImages
(image_path
, cur_revision
)
134 image_path
= image_path
+ '_' + str(cur_revision
) + '.png'
137 ref_png
= bitmap
.Bitmap
.FromPngFile(image_path
)
144 print 'Reference image not found. Writing tab contents as reference.'
146 self
._WriteImage
(image_path
, screenshot
)
149 class Pixel(cloud_storage_test_base
.TestBase
):
150 test
= _PixelValidator
151 page_set
= page_sets
.PixelTestsPageSet
154 def AddTestCommandLineArgs(cls
, group
):
155 super(Pixel
, cls
).AddTestCommandLineArgs(group
)
156 group
.add_option('--reference-dir',
157 help='Overrides the default on-disk location for reference images '
158 '(only used for local testing without a cloud storage account)',
159 default
=default_reference_image_dir
)
161 def CreatePageSet(self
, options
):
162 page_set
= super(Pixel
, self
).CreatePageSet(options
)
163 for page
in page_set
.pages
:
164 page
.script_to_evaluate_on_commit
= test_harness_script
167 def CreateExpectations(self
, page_set
):
168 return pixel_expectations
.PixelExpectations()