1 # Copyright 2014 The Chromium Authors. All rights reserved.
2 # Use of this source code is governed by a BSD-style license that can be
3 # found in the LICENSE file.
5 """This file contains printing-related functionality of the bisect."""
10 from bisect_results
import BisectResults
15 # The perf dashboard looks for a string like "Estimated Confidence: 95%"
16 # to decide whether or not to cc the author(s). If you change this, please
17 # update the perf dashboard as well.
19 ===== BISECT JOB RESULTS =====
22 Test Command: %(command)s
23 Test Metric: %(metrics)s
24 Relative Change: %(change)s
25 Estimated Confidence: %(confidence).02f%%
26 Retested CL with revert: %(retest)s"""
28 # When the bisect was aborted without a bisect failure the following template
30 ABORT_REASON_TEMPLATE
= """
31 ===== BISECTION ABORTED =====
32 The bisect was aborted because %(abort_reason)s
33 Please contact the the team (see below) if you believe this is in error.
37 Test Command: %(command)s
38 Test Metric: %(metric)s
39 Good revision: %(good_revision)s
40 Bad revision: %(bad_revision)s """
42 # The perf dashboard specifically looks for the string
43 # "Author : " to parse out who to cc on a bug. If you change the
44 # formatting here, please update the perf dashboard as well.
45 RESULTS_REVISION_INFO
= """
46 ===== SUSPECTED CL(s) =====
48 Author : %(author)s%(commit_info)s
52 RESULTS_THANKYOU
= """
53 | O O | Visit http://www.chromium.org/developers/core-principles for Chrome's
54 | X | policy on perf regressions. Contact chrome-perf-dashboard-team with any
55 | / \\ | questions or suggestions about bisecting. THANK YOU."""
57 REPRO_STEPS_LOCAL
= """
58 ==== INSTRUCTIONS TO REPRODUCE ====
60 - Use the test command given under 'BISECT JOB RESULTS' above.
61 - Consider using a profiler. Pass --profiler=list to list available profilers.
64 REPRO_STEPS_TRYJOB
= """
65 To reproduce on a performance try bot:
66 1. Edit run-perf-test.cfg
67 2. git try -b bot-name --svn_repo='svn://svn.chromium.org/chrome-try/try-perf'
70 a) Follow the in-file instructions in run-perf-test.cfg.
71 b) run-perf-test.cfg is under tools/ or under third_party/WebKit/Tools.
72 c) Do your edits preferably under a new git branch.
73 d) --browser=release and --browser=android-chromium-testshell are supported
74 depending on the platform (desktop|android).
75 e) Strip any src/ directories from the head of relative path names.
76 f) Make sure to use the appropriate bot on step 3.
78 For more details please visit
79 https://sites.google.com/a/chromium.org/dev/developers/performance-try-bots"""
81 REPRO_STEPS_TRYJOB_TELEMETRY
= """
82 To reproduce on a performance try bot:
84 (Where bot-name comes from tools/perf/run_benchmark --browser=list)
86 For more details please visit
87 https://sites.google.com/a/chromium.org/dev/developers/performance-try-bots
91 class BisectPrinter(object):
93 def __init__(self
, opts
, depot_registry
):
95 self
.depot_registry
= depot_registry
97 def FormatAndPrintResults(self
, bisect_results
):
98 """Prints the results from a bisection run in a readable format.
100 Also prints annotations creating buildbot step "Results".
103 bisect_results: BisectResult object containing results to be printed.
105 if bisect_results
.abort_reason
:
106 self
._PrintAbortResults
(bisect_results
.abort_reason
)
109 if self
.opts
.output_buildbot_annotations
:
110 bisect_utils
.OutputAnnotationStepStart('Build Status Per Revision')
113 print 'Full results of bisection:'
114 for revision_state
in bisect_results
.state
.GetRevisionStates():
115 build_status
= revision_state
.passed
117 if type(build_status
) is bool:
119 build_status
= 'Good'
123 print ' %20s %40s %s' % (revision_state
.depot
,
124 revision_state
.revision
,
128 if self
.opts
.output_buildbot_annotations
:
129 bisect_utils
.OutputAnnotationStepClosed()
130 # The perf dashboard scrapes the "results" step in order to comment on
131 # bugs. If you change this, please update the perf dashboard as well.
132 bisect_utils
.OutputAnnotationStepStart('Results')
134 self
._PrintBanner
(bisect_results
)
135 self
._PrintWarnings
(bisect_results
.warnings
)
137 if bisect_results
.culprit_revisions
and bisect_results
.confidence
:
138 for culprit
in bisect_results
.culprit_revisions
:
139 cl
, info
, depot
= culprit
140 self
._PrintRevisionInfo
(cl
, info
, depot
)
141 if bisect_results
.other_regressions
:
142 self
._PrintOtherRegressions
(bisect_results
.other_regressions
)
143 self
._PrintRetestResults
(bisect_results
)
144 self
._PrintTestedCommitsTable
(bisect_results
.state
.GetRevisionStates(),
145 bisect_results
.first_working_revision
,
146 bisect_results
.last_broken_revision
,
147 bisect_results
.confidence
,
149 self
._PrintStepTime
(bisect_results
.state
.GetRevisionStates())
150 self
._PrintReproSteps
()
151 self
._PrintThankYou
()
152 if self
.opts
.output_buildbot_annotations
:
153 bisect_utils
.OutputAnnotationStepClosed()
155 def PrintPartialResults(self
, bisect_state
):
156 revision_states
= bisect_state
.GetRevisionStates()
157 first_working_rev
, last_broken_rev
= BisectResults
.FindBreakingRevRange(
159 self
._PrintTestedCommitsTable
(revision_states
, first_working_rev
,
160 last_broken_rev
, 100, final_step
=False)
162 def _PrintAbortResults(self
, abort_reason
):
164 if self
.opts
.output_buildbot_annotations
:
165 bisect_utils
.OutputAnnotationStepStart('Results')
166 print ABORT_REASON_TEMPLATE
% {
167 'abort_reason': abort_reason
,
168 'bug_id': self
.opts
.bug_id
or 'NOT SPECIFIED',
169 'command': self
.opts
.command
,
170 'metric': '/'.join(self
.opts
.metric
),
171 'good_revision': self
.opts
.good_revision
,
172 'bad_revision': self
.opts
.bad_revision
,
174 self
._PrintThankYou
()
175 if self
.opts
.output_buildbot_annotations
:
176 bisect_utils
.OutputAnnotationStepClosed()
179 def _PrintThankYou():
180 print RESULTS_THANKYOU
183 def _PrintStepTime(revision_states
):
184 """Prints information about how long various steps took.
187 revision_states: Ordered list of revision states."""
188 step_perf_time_avg
= 0.0
189 step_build_time_avg
= 0.0
191 for revision_state
in revision_states
:
192 if revision_state
.value
:
193 step_perf_time_avg
+= revision_state
.perf_time
194 step_build_time_avg
+= revision_state
.build_time
197 step_perf_time_avg
= step_perf_time_avg
/ step_count
198 step_build_time_avg
= step_build_time_avg
/ step_count
200 print 'Average build time : %s' % datetime
.timedelta(
201 seconds
=int(step_build_time_avg
))
202 print 'Average test time : %s' % datetime
.timedelta(
203 seconds
=int(step_perf_time_avg
))
206 def _GetViewVCLinkFromDepotAndHash(git_revision
, depot
):
207 """Gets link to the repository browser."""
208 if depot
and 'viewvc' in bisect_utils
.DEPOT_DEPS_NAME
[depot
]:
209 return bisect_utils
.DEPOT_DEPS_NAME
[depot
]['viewvc'] + git_revision
212 def _PrintRevisionInfo(self
, cl
, info
, depot
=None):
213 commit_link
= self
._GetViewVCLinkFromDepotAndHash
(cl
, depot
)
215 commit_link
= '\nLink : %s' % commit_link
217 commit_link
= ('\Description:\n%s' % info
['body'])
218 print RESULTS_REVISION_INFO
% {
219 'subject': info
['subject'],
220 'author': info
['email'],
221 'commit_info': commit_link
,
223 'cl_date': info
['date']
227 def _PrintTableRow(column_widths
, row_data
):
228 """Prints out a row in a formatted table that has columns aligned.
231 column_widths: A list of column width numbers.
232 row_data: A list of items for each column in this row.
234 assert len(column_widths
) == len(row_data
)
236 for i
in xrange(len(column_widths
)):
237 current_row_data
= row_data
[i
].center(column_widths
[i
], ' ')
238 text
+= ('%%%ds' % column_widths
[i
]) % current_row_data
241 def _PrintTestedCommitsHeader(self
):
242 if self
.opts
.bisect_mode
== bisect_utils
.BISECT_MODE_MEAN
:
244 [20, 12, 70, 14, 12, 13],
245 ['Depot', 'Position', 'SHA', 'Mean', 'Std. Error', 'State'])
246 elif self
.opts
.bisect_mode
== bisect_utils
.BISECT_MODE_STD_DEV
:
248 [20, 12, 70, 14, 12, 13],
249 ['Depot', 'Position', 'SHA', 'Std. Error', 'Mean', 'State'])
250 elif self
.opts
.bisect_mode
== bisect_utils
.BISECT_MODE_RETURN_CODE
:
252 [20, 12, 70, 14, 13],
253 ['Depot', 'Position', 'SHA', 'Return Code', 'State'])
255 assert False, 'Invalid bisect_mode specified.'
257 def _PrintTestedCommitsEntry(self
, revision_state
, commit_position
, cl_link
,
259 if self
.opts
.bisect_mode
== bisect_utils
.BISECT_MODE_MEAN
:
260 std_error
= '+-%.02f' % revision_state
.value
['std_err']
261 mean
= '%.02f' % revision_state
.value
['mean']
263 [20, 12, 70, 12, 14, 13],
264 [revision_state
.depot
, commit_position
, cl_link
, mean
, std_error
,
266 elif self
.opts
.bisect_mode
== bisect_utils
.BISECT_MODE_STD_DEV
:
267 std_error
= '+-%.02f' % revision_state
.value
['std_err']
268 mean
= '%.02f' % revision_state
.value
['mean']
270 [20, 12, 70, 12, 14, 13],
271 [revision_state
.depot
, commit_position
, cl_link
, std_error
, mean
,
273 elif self
.opts
.bisect_mode
== bisect_utils
.BISECT_MODE_RETURN_CODE
:
274 mean
= '%d' % revision_state
.value
['mean']
276 [20, 12, 70, 14, 13],
277 [revision_state
.depot
, commit_position
, cl_link
, mean
,
280 def _PrintTestedCommitsTable(
281 self
, revision_states
, first_working_revision
, last_broken_revision
,
282 confidence
, final_step
=True):
285 print '===== TESTED COMMITS ====='
287 print '===== PARTIAL RESULTS ====='
288 self
._PrintTestedCommitsHeader
()
290 for revision_state
in revision_states
:
291 if revision_state
.value
:
292 if (revision_state
== last_broken_revision
or
293 revision_state
== first_working_revision
):
294 # If confidence is too low, don't add this empty line since it's
295 # used to put focus on a suspected CL.
296 if confidence
and final_step
:
299 if state
== 2 and not final_step
:
300 # Just want a separation between "bad" and "good" cl's.
304 if state
== 1 and final_step
:
305 state_str
= 'Suspected CL'
309 # If confidence is too low, don't bother outputting good/bad.
312 state_str
= state_str
.center(13, ' ')
313 commit_position
= source_control
.GetCommitPosition(
314 revision_state
.revision
,
315 self
.depot_registry
.GetDepotDir(revision_state
.depot
))
316 display_commit_pos
= ''
318 display_commit_pos
= str(commit_position
)
319 self
._PrintTestedCommitsEntry
(revision_state
,
321 revision_state
.revision
,
324 def _PrintRetestResults(self
, bisect_results
):
325 if (not bisect_results
.retest_results_tot
or
326 not bisect_results
.retest_results_reverted
):
329 print '===== RETEST RESULTS ====='
330 self
._PrintTestedCommitsEntry
(
331 bisect_results
.retest_results_tot
, '', '', '')
332 self
._PrintTestedCommitsEntry
(
333 bisect_results
.retest_results_reverted
, '', '', '')
335 def _PrintReproSteps(self
):
336 """Prints out a section of the results explaining how to run the test.
338 This message includes the command used to run the test.
340 command
= '$ ' + self
.opts
.command
341 if bisect_utils
.IsTelemetryCommand(self
.opts
.command
):
342 command
+= ('\nAlso consider passing --profiler=list to see available '
344 print REPRO_STEPS_LOCAL
345 if bisect_utils
.IsTelemetryCommand(self
.opts
.command
):
346 telemetry_command
= re
.sub(r
'--browser=[^\s]+',
347 '--browser=bot-name',
349 print REPRO_STEPS_TRYJOB_TELEMETRY
% {'command': telemetry_command
}
351 print REPRO_STEPS_TRYJOB
353 def _PrintOtherRegressions(self
, other_regressions
):
354 """Prints a section of the results about other potential regressions."""
356 print 'Other regressions may have occurred:'
357 self
._PrintTableRow
([8, 70, 10], ['Depot', 'Range', 'Confidence'])
358 for regression
in other_regressions
:
359 current_rev_state
, prev_rev_state
, confidence
= regression
362 [current_rev_state
.depot
, current_rev_state
.revision
,
363 '%d%%' % confidence
])
365 [8, 70], [prev_rev_state
.depot
, prev_rev_state
.revision
])
369 def _ConfidenceLevelStatus(bisect_results
):
370 if not bisect_results
.confidence
:
372 confidence_status
= 'Successful with %(level)s confidence%(warning)s.'
373 if bisect_results
.confidence
>= bisect_utils
.HIGH_CONFIDENCE
:
377 warning
= ' and warnings'
378 if not bisect_results
.warnings
:
380 return confidence_status
% {'level': level
, 'warning': warning
}
382 def _PrintBanner(self
, bisect_results
):
383 if self
.opts
.bisect_mode
== bisect_utils
.BISECT_MODE_RETURN_CODE
:
387 metrics
= '/'.join(self
.opts
.metric
)
388 change
= '%.02f%% (+/-%.02f%%)' % (
389 bisect_results
.regression_size
, bisect_results
.regression_std_err
)
391 if bisect_results
.culprit_revisions
and bisect_results
.confidence
:
392 status
= self
._ConfidenceLevelStatus
(bisect_results
)
394 status
= 'Failure, could not reproduce.'
395 change
= 'Bisect could not reproduce a change.'
397 retest_text
= 'Yes' if bisect_results
.retest_results_tot
else 'No'
399 print RESULTS_BANNER
% {
401 'command': self
.opts
.command
,
404 'confidence': bisect_results
.confidence
,
405 'retest': retest_text
,
409 def _PrintWarnings(warnings
):
410 """Prints a list of warning strings if there are any."""
415 for w
in set(warnings
):