Don't show supervised user as "already on this device" while they're being imported.
[chromium-blink-merge.git] / tools / auto_bisect / bisect_printer.py
blob34baa319030d32db09ed716150165c20e3639336
1 # Copyright 2014 The Chromium Authors. All rights reserved.
2 # Use of this source code is governed by a BSD-style license that can be
3 # found in the LICENSE file.
5 """This file contains printing-related functionality of the bisect."""
7 import datetime
8 import re
10 from bisect_results import BisectResults
11 import bisect_utils
12 import source_control
15 # The perf dashboard looks for a string like "Estimated Confidence: 95%"
16 # to decide whether or not to cc the author(s). If you change this, please
17 # update the perf dashboard as well.
18 RESULTS_BANNER = """
19 ===== BISECT JOB RESULTS =====
20 Status: %(status)s
22 Test Command: %(command)s
23 Test Metric: %(metrics)s
24 Relative Change: %(change)s
25 Estimated Confidence: %(confidence).02f%%
26 Retested CL with revert: %(retest)s"""
28 # When the bisect was aborted without a bisect failure the following template
29 # is used.
30 ABORT_REASON_TEMPLATE = """
31 ===== BISECTION ABORTED =====
32 The bisect was aborted because %(abort_reason)s
33 Please contact the the team (see below) if you believe this is in error.
35 Bug ID: %(bug_id)s
37 Test Command: %(command)s
38 Test Metric: %(metric)s
39 Good revision: %(good_revision)s
40 Bad revision: %(bad_revision)s """
42 # The perf dashboard specifically looks for the string
43 # "Author : " to parse out who to cc on a bug. If you change the
44 # formatting here, please update the perf dashboard as well.
45 RESULTS_REVISION_INFO = """
46 ===== SUSPECTED CL(s) =====
47 Subject : %(subject)s
48 Author : %(author)s%(commit_info)s
49 Commit : %(cl)s
50 Date : %(cl_date)s"""
52 RESULTS_THANKYOU = """
53 | O O | Visit http://www.chromium.org/developers/core-principles for Chrome's
54 | X | policy on perf regressions. Contact chrome-perf-dashboard-team with any
55 | / \\ | questions or suggestions about bisecting. THANK YOU."""
57 REPRO_STEPS_LOCAL = """
58 ==== INSTRUCTIONS TO REPRODUCE ====
59 To run locally:
60 - Use the test command given under 'BISECT JOB RESULTS' above.
61 - Consider using a profiler. Pass --profiler=list to list available profilers.
62 """
64 REPRO_STEPS_TRYJOB = """
65 To reproduce on a performance try bot:
66 1. Edit run-perf-test.cfg
67 2. git try -b bot-name --svn_repo='svn://svn.chromium.org/chrome-try/try-perf'
69 Notes:
70 a) Follow the in-file instructions in run-perf-test.cfg.
71 b) run-perf-test.cfg is under tools/ or under third_party/WebKit/Tools.
72 c) Do your edits preferably under a new git branch.
73 d) --browser=release and --browser=android-chromium-testshell are supported
74 depending on the platform (desktop|android).
75 e) Strip any src/ directories from the head of relative path names.
76 f) Make sure to use the appropriate bot on step 3.
78 For more details please visit
79 https://sites.google.com/a/chromium.org/dev/developers/performance-try-bots"""
81 REPRO_STEPS_TRYJOB_TELEMETRY = """
82 To reproduce on a performance try bot:
83 %(command)s
84 (Where bot-name comes from tools/perf/run_benchmark --browser=list)
86 For more details please visit
87 https://sites.google.com/a/chromium.org/dev/developers/performance-try-bots
88 """
91 class BisectPrinter(object):
93 def __init__(self, opts, depot_registry):
94 self.opts = opts
95 self.depot_registry = depot_registry
97 def FormatAndPrintResults(self, bisect_results):
98 """Prints the results from a bisection run in a readable format.
100 Also prints annotations creating buildbot step "Results".
102 Args:
103 bisect_results: BisectResult object containing results to be printed.
105 if bisect_results.abort_reason:
106 self._PrintAbortResults(bisect_results.abort_reason)
107 return
109 if self.opts.output_buildbot_annotations:
110 bisect_utils.OutputAnnotationStepStart('Build Status Per Revision')
112 print
113 print 'Full results of bisection:'
114 for revision_state in bisect_results.state.GetRevisionStates():
115 build_status = revision_state.passed
117 if type(build_status) is bool:
118 if build_status:
119 build_status = 'Good'
120 else:
121 build_status = 'Bad'
123 print ' %20s %40s %s' % (revision_state.depot,
124 revision_state.revision,
125 build_status)
126 print
128 if self.opts.output_buildbot_annotations:
129 bisect_utils.OutputAnnotationStepClosed()
130 # The perf dashboard scrapes the "results" step in order to comment on
131 # bugs. If you change this, please update the perf dashboard as well.
132 bisect_utils.OutputAnnotationStepStart('Results')
134 self._PrintBanner(bisect_results)
135 self._PrintWarnings(bisect_results.warnings)
137 if bisect_results.culprit_revisions and bisect_results.confidence:
138 for culprit in bisect_results.culprit_revisions:
139 cl, info, depot = culprit
140 self._PrintRevisionInfo(cl, info, depot)
141 if bisect_results.other_regressions:
142 self._PrintOtherRegressions(bisect_results.other_regressions)
143 self._PrintRetestResults(bisect_results)
144 self._PrintTestedCommitsTable(bisect_results.state.GetRevisionStates(),
145 bisect_results.first_working_revision,
146 bisect_results.last_broken_revision,
147 bisect_results.confidence,
148 final_step=True)
149 self._PrintStepTime(bisect_results.state.GetRevisionStates())
150 self._PrintReproSteps()
151 self._PrintThankYou()
152 if self.opts.output_buildbot_annotations:
153 bisect_utils.OutputAnnotationStepClosed()
155 def PrintPartialResults(self, bisect_state):
156 revision_states = bisect_state.GetRevisionStates()
157 first_working_rev, last_broken_rev = BisectResults.FindBreakingRevRange(
158 revision_states)
159 self._PrintTestedCommitsTable(revision_states, first_working_rev,
160 last_broken_rev, 100, final_step=False)
162 def _PrintAbortResults(self, abort_reason):
164 if self.opts.output_buildbot_annotations:
165 bisect_utils.OutputAnnotationStepStart('Results')
166 print ABORT_REASON_TEMPLATE % {
167 'abort_reason': abort_reason,
168 'bug_id': self.opts.bug_id or 'NOT SPECIFIED',
169 'command': self.opts.command,
170 'metric': '/'.join(self.opts.metric),
171 'good_revision': self.opts.good_revision,
172 'bad_revision': self.opts.bad_revision,
174 self._PrintThankYou()
175 if self.opts.output_buildbot_annotations:
176 bisect_utils.OutputAnnotationStepClosed()
178 @staticmethod
179 def _PrintThankYou():
180 print RESULTS_THANKYOU
182 @staticmethod
183 def _PrintStepTime(revision_states):
184 """Prints information about how long various steps took.
186 Args:
187 revision_states: Ordered list of revision states."""
188 step_perf_time_avg = 0.0
189 step_build_time_avg = 0.0
190 step_count = 0.0
191 for revision_state in revision_states:
192 if revision_state.value:
193 step_perf_time_avg += revision_state.perf_time
194 step_build_time_avg += revision_state.build_time
195 step_count += 1
196 if step_count:
197 step_perf_time_avg = step_perf_time_avg / step_count
198 step_build_time_avg = step_build_time_avg / step_count
199 print
200 print 'Average build time : %s' % datetime.timedelta(
201 seconds=int(step_build_time_avg))
202 print 'Average test time : %s' % datetime.timedelta(
203 seconds=int(step_perf_time_avg))
205 @staticmethod
206 def _GetViewVCLinkFromDepotAndHash(git_revision, depot):
207 """Gets link to the repository browser."""
208 if depot and 'viewvc' in bisect_utils.DEPOT_DEPS_NAME[depot]:
209 return bisect_utils.DEPOT_DEPS_NAME[depot]['viewvc'] + git_revision
210 return ''
212 def _PrintRevisionInfo(self, cl, info, depot=None):
213 commit_link = self._GetViewVCLinkFromDepotAndHash(cl, depot)
214 if commit_link:
215 commit_link = '\nLink : %s' % commit_link
216 else:
217 commit_link = ('\Description:\n%s' % info['body'])
218 print RESULTS_REVISION_INFO % {
219 'subject': info['subject'],
220 'author': info['email'],
221 'commit_info': commit_link,
222 'cl': cl,
223 'cl_date': info['date']
226 @staticmethod
227 def _PrintTableRow(column_widths, row_data):
228 """Prints out a row in a formatted table that has columns aligned.
230 Args:
231 column_widths: A list of column width numbers.
232 row_data: A list of items for each column in this row.
234 assert len(column_widths) == len(row_data)
235 text = ''
236 for i in xrange(len(column_widths)):
237 current_row_data = row_data[i].center(column_widths[i], ' ')
238 text += ('%%%ds' % column_widths[i]) % current_row_data
239 print text
241 def _PrintTestedCommitsHeader(self):
242 if self.opts.bisect_mode == bisect_utils.BISECT_MODE_MEAN:
243 self._PrintTableRow(
244 [20, 12, 70, 14, 12, 13],
245 ['Depot', 'Position', 'SHA', 'Mean', 'Std. Error', 'State'])
246 elif self.opts.bisect_mode == bisect_utils.BISECT_MODE_STD_DEV:
247 self._PrintTableRow(
248 [20, 12, 70, 14, 12, 13],
249 ['Depot', 'Position', 'SHA', 'Std. Error', 'Mean', 'State'])
250 elif self.opts.bisect_mode == bisect_utils.BISECT_MODE_RETURN_CODE:
251 self._PrintTableRow(
252 [20, 12, 70, 14, 13],
253 ['Depot', 'Position', 'SHA', 'Return Code', 'State'])
254 else:
255 assert False, 'Invalid bisect_mode specified.'
257 def _PrintTestedCommitsEntry(self, revision_state, commit_position, cl_link,
258 state_str):
259 if self.opts.bisect_mode == bisect_utils.BISECT_MODE_MEAN:
260 std_error = '+-%.02f' % revision_state.value['std_err']
261 mean = '%.02f' % revision_state.value['mean']
262 self._PrintTableRow(
263 [20, 12, 70, 12, 14, 13],
264 [revision_state.depot, commit_position, cl_link, mean, std_error,
265 state_str])
266 elif self.opts.bisect_mode == bisect_utils.BISECT_MODE_STD_DEV:
267 std_error = '+-%.02f' % revision_state.value['std_err']
268 mean = '%.02f' % revision_state.value['mean']
269 self._PrintTableRow(
270 [20, 12, 70, 12, 14, 13],
271 [revision_state.depot, commit_position, cl_link, std_error, mean,
272 state_str])
273 elif self.opts.bisect_mode == bisect_utils.BISECT_MODE_RETURN_CODE:
274 mean = '%d' % revision_state.value['mean']
275 self._PrintTableRow(
276 [20, 12, 70, 14, 13],
277 [revision_state.depot, commit_position, cl_link, mean,
278 state_str])
280 def _PrintTestedCommitsTable(
281 self, revision_states, first_working_revision, last_broken_revision,
282 confidence, final_step=True):
283 print
284 if final_step:
285 print '===== TESTED COMMITS ====='
286 else:
287 print '===== PARTIAL RESULTS ====='
288 self._PrintTestedCommitsHeader()
289 state = 0
290 for revision_state in revision_states:
291 if revision_state.value:
292 if (revision_state == last_broken_revision or
293 revision_state == first_working_revision):
294 # If confidence is too low, don't add this empty line since it's
295 # used to put focus on a suspected CL.
296 if confidence and final_step:
297 print
298 state += 1
299 if state == 2 and not final_step:
300 # Just want a separation between "bad" and "good" cl's.
301 print
303 state_str = 'Bad'
304 if state == 1 and final_step:
305 state_str = 'Suspected CL'
306 elif state == 2:
307 state_str = 'Good'
309 # If confidence is too low, don't bother outputting good/bad.
310 if not confidence:
311 state_str = ''
312 state_str = state_str.center(13, ' ')
313 commit_position = source_control.GetCommitPosition(
314 revision_state.revision,
315 self.depot_registry.GetDepotDir(revision_state.depot))
316 display_commit_pos = ''
317 if commit_position:
318 display_commit_pos = str(commit_position)
319 self._PrintTestedCommitsEntry(revision_state,
320 display_commit_pos,
321 revision_state.revision,
322 state_str)
324 def _PrintRetestResults(self, bisect_results):
325 if (not bisect_results.retest_results_tot or
326 not bisect_results.retest_results_reverted):
327 return
328 print
329 print '===== RETEST RESULTS ====='
330 self._PrintTestedCommitsEntry(
331 bisect_results.retest_results_tot, '', '', '')
332 self._PrintTestedCommitsEntry(
333 bisect_results.retest_results_reverted, '', '', '')
335 def _PrintReproSteps(self):
336 """Prints out a section of the results explaining how to run the test.
338 This message includes the command used to run the test.
340 command = '$ ' + self.opts.command
341 if bisect_utils.IsTelemetryCommand(self.opts.command):
342 command += ('\nAlso consider passing --profiler=list to see available '
343 'profilers.')
344 print REPRO_STEPS_LOCAL
345 if bisect_utils.IsTelemetryCommand(self.opts.command):
346 telemetry_command = re.sub(r'--browser=[^\s]+',
347 '--browser=bot-name',
348 command)
349 print REPRO_STEPS_TRYJOB_TELEMETRY % {'command': telemetry_command}
350 else:
351 print REPRO_STEPS_TRYJOB
353 def _PrintOtherRegressions(self, other_regressions):
354 """Prints a section of the results about other potential regressions."""
355 print
356 print 'Other regressions may have occurred:'
357 self._PrintTableRow([8, 70, 10], ['Depot', 'Range', 'Confidence'])
358 for regression in other_regressions:
359 current_rev_state, prev_rev_state, confidence = regression
360 self._PrintTableRow(
361 [8, 70, 10],
362 [current_rev_state.depot, current_rev_state.revision,
363 '%d%%' % confidence])
364 self._PrintTableRow(
365 [8, 70], [prev_rev_state.depot, prev_rev_state.revision])
366 print
368 @staticmethod
369 def _ConfidenceLevelStatus(bisect_results):
370 if not bisect_results.confidence:
371 return None
372 confidence_status = 'Successful with %(level)s confidence%(warning)s.'
373 if bisect_results.confidence >= bisect_utils.HIGH_CONFIDENCE:
374 level = 'high'
375 else:
376 level = 'low'
377 warning = ' and warnings'
378 if not bisect_results.warnings:
379 warning = ''
380 return confidence_status % {'level': level, 'warning': warning}
382 def _PrintBanner(self, bisect_results):
383 if self.opts.bisect_mode == bisect_utils.BISECT_MODE_RETURN_CODE:
384 metrics = 'N/A'
385 change = 'Yes'
386 else:
387 metrics = '/'.join(self.opts.metric)
388 change = '%.02f%% (+/-%.02f%%)' % (
389 bisect_results.regression_size, bisect_results.regression_std_err)
391 if bisect_results.culprit_revisions and bisect_results.confidence:
392 status = self._ConfidenceLevelStatus(bisect_results)
393 else:
394 status = 'Failure, could not reproduce.'
395 change = 'Bisect could not reproduce a change.'
397 retest_text = 'Yes' if bisect_results.retest_results_tot else 'No'
399 print RESULTS_BANNER % {
400 'status': status,
401 'command': self.opts.command,
402 'metrics': metrics,
403 'change': change,
404 'confidence': bisect_results.confidence,
405 'retest': retest_text,
408 @staticmethod
409 def _PrintWarnings(warnings):
410 """Prints a list of warning strings if there are any."""
411 if not warnings:
412 return
413 print
414 print 'WARNINGS:'
415 for w in set(warnings):
416 print ' ! %s' % w