1 # Copyright 2015 The Chromium Authors. All rights reserved.
2 # Use of this source code is governed by a BSD-style license that can be
3 # found in the LICENSE file.
7 from telemetry
import benchmark
as benchmark_module
8 from telemetry
.core
import exceptions
9 from telemetry
.page
import page
as page_module
10 from telemetry
.page
import page_test
11 from telemetry
.page
import shared_page_state
12 from telemetry
.value
import skip
14 import exception_formatter
15 import test_expectations
17 """Base classes for all GPU tests in this directory. Implements
18 support for per-page test expectations."""
20 # TODO(kbr): add unit tests for these classes, specifically:
21 # - one matching testHandlingOfCrashedTabWithExpectedFailure from
22 # telemetry/page/page_run_end_to_end_unittest.py
23 # (https://crbug.com/444240), which was removed during the test
24 # expectations refactoring
25 # - verify behavior of expected failures
26 # - verify behavior of skipping tests
27 # - after GpuTestExpectations is folded into TestExpectations,
28 # verify handling of flaky tests
29 # See http://crbug.com/495870.
31 def _PageOperationWrapper(page
, tab
, expectations
,
32 show_expected_failure
, inner_func
,
34 """Wraps an operation to be done against the page. If an error has
35 occurred in an earlier step, skips this entirely."""
40 expectation
= expectations
.GetExpectationForPage(tab
.browser
, page
)
41 if expectation
== 'skip':
43 results
.AddValue(skip
.SkipValue(
45 "Skipped by test expectations"))
51 if expectation
== 'pass':
53 elif expectation
== 'fail':
54 msg
= 'Expected exception while running %s' % page
.display_name
55 exception_formatter
.PrintFormattedException(msg
=msg
)
58 'Unknown expectation %s while handling exception for %s' %
59 (expectation
, page
.display_name
))
62 if show_expected_failure
and expectation
== 'fail':
64 '%s was expected to fail, but passed.\n', page
.display_name
)
67 class TestBase(benchmark_module
.Benchmark
):
68 def __init__(self
, max_failures
=None):
69 super(TestBase
, self
).__init
__(max_failures
=max_failures
)
70 self
._cached
_expectations
= None
72 def GetExpectations(self
):
73 """Returns the expectations that apply to this test."""
74 if not self
._cached
_expectations
:
75 self
._cached
_expectations
= self
._CreateExpectations
()
76 return self
._cached
_expectations
78 def _CreateExpectations(self
):
79 # By default, creates an empty TestExpectations object. Override
80 # this in subclasses to set up test-specific expectations. Don't
81 # call this directly. Call GetExpectations where necessary.
82 return test_expectations
.TestExpectations()
85 class ValidatorBase(page_test
.PageTest
):
87 needs_browser_restart_after_each_page
=False,
88 discard_first_result
=False,
89 clear_cache_before_each_run
=False):
90 super(ValidatorBase
, self
).__init
__(
91 needs_browser_restart_after_each_page
=\
92 needs_browser_restart_after_each_page
,
93 discard_first_result
=discard_first_result
,
94 clear_cache_before_each_run
=clear_cache_before_each_run
)
96 def ValidateAndMeasurePage(self
, page
, tab
, results
):
97 """Validates and measures the page, taking into account test
98 expectations. Do not override this method. Override
99 ValidateAndMeasurePageInner, below."""
101 _PageOperationWrapper(
102 page
, tab
, page
.GetExpectations(), True,
103 lambda: self
.ValidateAndMeasurePageInner(page
, tab
, results
),
106 # Clear the error state of the page at this point so that if
107 # --page-repeat or --pageset-repeat are used, the subsequent
108 # iterations don't turn into no-ops.
111 def ValidateAndMeasurePageInner(self
, page
, tab
, results
):
115 class PageBase(page_module
.Page
):
116 # The convention is that pages subclassing this class must be
117 # configured with the test expectations.
118 def __init__(self
, url
, page_set
=None, base_dir
=None, name
='',
119 shared_page_state_class
=shared_page_state
.SharedPageState
,
120 make_javascript_deterministic
=True,
122 super(PageBase
, self
).__init
__(
123 url
=url
, page_set
=page_set
, base_dir
=base_dir
, name
=name
,
124 shared_page_state_class
=shared_page_state_class
,
125 make_javascript_deterministic
=make_javascript_deterministic
)
126 self
._expectations
= expectations
127 self
._had
_error
= False
129 def GetExpectations(self
):
130 return self
._expectations
133 return self
._had
_error
135 def SetHadError(self
):
136 self
._had
_error
= True
138 def ClearHadError(self
):
139 self
._had
_error
= False
141 def RunNavigateSteps(self
, action_runner
):
142 """Runs navigation steps, taking into account test expectations.
143 Do not override this method. Override RunNavigateStepsInner, below."""
145 self
.RunDefaultNavigateSteps(action_runner
)
146 self
.RunNavigateStepsInner(action_runner
)
147 _PageOperationWrapper(self
, action_runner
.tab
, self
.GetExpectations(),
150 def RunDefaultNavigateSteps(self
, action_runner
):
151 """Runs the default set of navigation steps inherited from page.Page."""
152 super(PageBase
, self
).RunNavigateSteps(action_runner
)
154 def RunNavigateStepsInner(self
, action_runner
):
157 def RunPageInteractions(self
, action_runner
):
158 """Runs page interactions, taking into account test expectations. Do not
159 override this method. Override RunPageInteractionsInner, below."""
161 super(PageBase
, self
).RunPageInteractions(action_runner
)
162 self
.RunPageInteractionsInner(action_runner
)
163 _PageOperationWrapper(self
, action_runner
.tab
, self
.GetExpectations(),
166 def RunPageInteractionsInner(self
, action_runner
):