tools/auto_bisect/bisect_printer.py

   1 # Copyright 2014 The Chromium Authors. All rights reserved.
   2 # Use of this source code is governed by a BSD-style license that can be
   3 # found in the LICENSE file.
   4
   5 """This file contains printing-related functionality of the bisect."""
   6
   7 import datetime
   8 import re
   9
  10 from bisect_results import BisectResults
  11 import bisect_utils
  12 import source_control
  13
  14
  15 # The perf dashboard looks for a string like "Estimated Confidence: 95%"
  16 # to decide whether or not to cc the author(s). If you change this, please
  17 # update the perf dashboard as well.
  18 RESULTS_BANNER = """
  19 ===== BISECT JOB RESULTS =====
  20 Status: %(status)s
  21
  22 Test Command: %(command)s
  23 Test Metric: %(metrics)s
  24 Relative Change: %(change)s
  25 Estimated Confidence: %(confidence).02f%%
  26 Retested CL with revert: %(retest)s"""
  27
  28 # When the bisect was aborted without a bisect failure the following template
  29 # is used.
  30 ABORT_REASON_TEMPLATE = """
  31 ===== BISECTION ABORTED =====
  32 The bisect was aborted because %(abort_reason)s
  33 Please contact the the team (see below) if you believe this is in error.
  34
  35 Bug ID: %(bug_id)s
  36
  37 Test Command: %(command)s
  38 Test Metric: %(metric)s
  39 Good revision: %(good_revision)s
  40 Bad revision: %(bad_revision)s """
  41
  42 # The perf dashboard specifically looks for the string
  43 # "Author  : " to parse out who to cc on a bug. If you change the
  44 # formatting here, please update the perf dashboard as well.
  45 RESULTS_REVISION_INFO = """
  46 ===== SUSPECTED CL(s) =====
  47 Subject : %(subject)s
  48 Author  : %(author)s%(commit_info)s
  49 Commit  : %(cl)s
  50 Date    : %(cl_date)s"""
  51
  52 RESULTS_THANKYOU = """
  53 | O O | Visit http://www.chromium.org/developers/core-principles for Chrome's
  54 |  X  | policy on perf regressions. Contact chrome-perf-dashboard-team with any
  55 | / \\ | questions or suggestions about bisecting. THANK YOU."""
  56
  57 REPRO_STEPS_LOCAL = """
  58 ==== INSTRUCTIONS TO REPRODUCE ====
  59 To run locally:
  60  - Use the test command given under 'BISECT JOB RESULTS' above.
  61  - Consider using a profiler. Pass --profiler=list to list available profilers.
  62 """
  63
  64 REPRO_STEPS_TRYJOB = """
  65 To reproduce on a performance try bot:
  66  1. Edit run-perf-test.cfg
  67  2. $ git try -b <bot> --svn_repo='svn://svn.chromium.org/chrome-try/try-perf'
  68
  69 Notes:
  70  a) Follow the in-file instructions in run-perf-test.cfg.
  71  b) run-perf-test.cfg is under tools/ or under third_party/WebKit/Tools.
  72  c) Do your edits preferably under a new git branch.
  73  d) --browser=release and --browser=android-chromium-testshell are supported
  74     depending on the platform (desktop|android).
  75  e) Strip any src/ directories from the head of relative path names.
  76  f) Make sure to use the appropriate bot on step 3.
  77
  78 For more details please visit
  79 https://sites.google.com/a/chromium.org/dev/developers/performance-try-bots"""
  80
  81 REPRO_STEPS_TRYJOB_TELEMETRY = """
  82 To reproduce on a performance try bot:
  83 %(command)s
  84 (Where <bot-name> comes from tools/perf/run_benchmark --browser=list)
  85
  86 For more details please visit
  87 https://sites.google.com/a/chromium.org/dev/developers/performance-try-bots
  88 """
  89
  90
  91 class BisectPrinter(object):
  92
  93   def __init__(self, opts, depot_registry):
  94     self.opts = opts
  95     self.depot_registry = depot_registry
  96
  97   def FormatAndPrintResults(self, bisect_results):
  98     """Prints the results from a bisection run in a readable format.
  99
 100     Also prints annotations creating buildbot step "Results".
 101
 102     Args:
 103       bisect_results: BisectResult object containing results to be printed.
 104     """
 105     if bisect_results.abort_reason:
 106       self._PrintAbortResults(bisect_results.abort_reason)
 107       return
 108
 109     if self.opts.output_buildbot_annotations:
 110       bisect_utils.OutputAnnotationStepStart('Build Status Per Revision')
 111
 112     print
 113     print 'Full results of bisection:'
 114     for revision_state in bisect_results.state.GetRevisionStates():
 115       build_status = revision_state.passed
 116
 117       if type(build_status) is bool:
 118         if build_status:
 119           build_status = 'Good'
 120         else:
 121           build_status = 'Bad'
 122
 123       print '  %20s  %40s  %s' % (revision_state.depot, revision_state.revision,
 124                                   build_status)
 125     print
 126
 127     if self.opts.output_buildbot_annotations:
 128       bisect_utils.OutputAnnotationStepClosed()
 129       # The perf dashboard scrapes the "results" step in order to comment on
 130       # bugs. If you change this, please update the perf dashboard as well.
 131       bisect_utils.OutputAnnotationStepStart('Results')
 132
 133     self._PrintBanner(bisect_results)
 134     self._PrintWarnings(bisect_results.warnings)
 135
 136     if bisect_results.culprit_revisions and bisect_results.confidence:
 137       for culprit in bisect_results.culprit_revisions:
 138         cl, info, depot = culprit
 139         self._PrintRevisionInfo(cl, info, depot)
 140       if bisect_results.other_regressions:
 141         self._PrintOtherRegressions(bisect_results.other_regressions)
 142     self._PrintRetestResults(bisect_results)
 143     self._PrintTestedCommitsTable(bisect_results.state.GetRevisionStates(),
 144                                   bisect_results.first_working_revision,
 145                                   bisect_results.last_broken_revision,
 146                                   bisect_results.confidence,
 147                                   final_step=True)
 148     self._PrintStepTime(bisect_results.state.GetRevisionStates())
 149     self._PrintReproSteps()
 150     self._PrintThankYou()
 151     if self.opts.output_buildbot_annotations:
 152       bisect_utils.OutputAnnotationStepClosed()
 153
 154   def PrintPartialResults(self, bisect_state):
 155     revision_states = bisect_state.GetRevisionStates()
 156     first_working_rev, last_broken_rev = BisectResults.FindBreakingRevRange(
 157         revision_states)
 158     self._PrintTestedCommitsTable(revision_states, first_working_rev,
 159                                   last_broken_rev, 100, final_step=False)
 160
 161   def _PrintAbortResults(self, abort_reason):
 162
 163     if self.opts.output_buildbot_annotations:
 164       bisect_utils.OutputAnnotationStepStart('Results')
 165     print ABORT_REASON_TEMPLATE % {
 166         'abort_reason': abort_reason,
 167         'bug_id': self.opts.bug_id or 'NOT SPECIFIED',
 168         'command': self.opts.command,
 169         'metric': '/'.join(self.opts.metric),
 170         'good_revision': self.opts.good_revision,
 171         'bad_revision': self.opts.bad_revision,
 172     }
 173     self._PrintThankYou()
 174     if self.opts.output_buildbot_annotations:
 175       bisect_utils.OutputAnnotationStepClosed()
 176
 177   @staticmethod
 178   def _PrintThankYou():
 179     print RESULTS_THANKYOU
 180
 181   @staticmethod
 182   def _PrintStepTime(revision_states):
 183     """Prints information about how long various steps took.
 184
 185     Args:
 186       revision_states: Ordered list of revision states."""
 187     step_perf_time_avg = 0.0
 188     step_build_time_avg = 0.0
 189     step_count = 0.0
 190     for revision_state in revision_states:
 191       if revision_state.value:
 192         step_perf_time_avg += revision_state.perf_time
 193         step_build_time_avg += revision_state.build_time
 194         step_count += 1
 195     if step_count:
 196       step_perf_time_avg = step_perf_time_avg / step_count
 197       step_build_time_avg = step_build_time_avg / step_count
 198     print
 199     print 'Average build time : %s' % datetime.timedelta(
 200         seconds=int(step_build_time_avg))
 201     print 'Average test time  : %s' % datetime.timedelta(
 202         seconds=int(step_perf_time_avg))
 203
 204   @staticmethod
 205   def _GetViewVCLinkFromDepotAndHash(git_revision, depot):
 206     """Gets link to the repository browser."""
 207     if depot and 'viewvc' in bisect_utils.DEPOT_DEPS_NAME[depot]:
 208       return bisect_utils.DEPOT_DEPS_NAME[depot]['viewvc'] + git_revision
 209     return ''
 210
 211   def _PrintRevisionInfo(self, cl, info, depot=None):
 212     commit_link = self._GetViewVCLinkFromDepotAndHash(cl, depot)
 213     if commit_link:
 214       commit_link = '\nLink    : %s' % commit_link
 215     else:
 216       commit_link = ('\Description:\n%s' % info['body'])
 217     print RESULTS_REVISION_INFO % {
 218         'subject': info['subject'],
 219         'author': info['email'],
 220         'commit_info': commit_link,
 221         'cl': cl,
 222         'cl_date': info['date']
 223     }
 224
 225   @staticmethod
 226   def _PrintTableRow(column_widths, row_data):
 227     """Prints out a row in a formatted table that has columns aligned.
 228
 229     Args:
 230       column_widths: A list of column width numbers.
 231       row_data: A list of items for each column in this row.
 232     """
 233     assert len(column_widths) == len(row_data)
 234     text = ''
 235     for i in xrange(len(column_widths)):
 236       current_row_data = row_data[i].center(column_widths[i], ' ')
 237       text += ('%%%ds' % column_widths[i]) % current_row_data
 238     print text
 239
 240   def _PrintTestedCommitsHeader(self):
 241     if self.opts.bisect_mode == bisect_utils.BISECT_MODE_MEAN:
 242       self._PrintTableRow(
 243           [20, 12, 70, 14, 12, 13],
 244           ['Depot', 'Position', 'SHA', 'Mean', 'Std. Error', 'State'])
 245     elif self.opts.bisect_mode == bisect_utils.BISECT_MODE_STD_DEV:
 246       self._PrintTableRow(
 247           [20, 12, 70, 14, 12, 13],
 248           ['Depot', 'Position', 'SHA', 'Std. Error', 'Mean', 'State'])
 249     elif self.opts.bisect_mode == bisect_utils.BISECT_MODE_RETURN_CODE:
 250       self._PrintTableRow(
 251           [20, 12, 70, 14, 13],
 252           ['Depot', 'Position', 'SHA', 'Return Code', 'State'])
 253     else:
 254       assert False, 'Invalid bisect_mode specified.'
 255
 256   def _PrintTestedCommitsEntry(self, revision_state, commit_position, cl_link,
 257                                state_str):
 258     if self.opts.bisect_mode == bisect_utils.BISECT_MODE_MEAN:
 259       std_error = '+-%.02f' % revision_state.value['std_err']
 260       mean = '%.02f' % revision_state.value['mean']
 261       self._PrintTableRow(
 262           [20, 12, 70, 12, 14, 13],
 263           [revision_state.depot, commit_position, cl_link, mean, std_error,
 264            state_str])
 265     elif self.opts.bisect_mode == bisect_utils.BISECT_MODE_STD_DEV:
 266       std_error = '+-%.02f' % revision_state.value['std_err']
 267       mean = '%.02f' % revision_state.value['mean']
 268       self._PrintTableRow(
 269           [20, 12, 70, 12, 14, 13],
 270           [revision_state.depot, commit_position, cl_link, std_error, mean,
 271            state_str])
 272     elif self.opts.bisect_mode == bisect_utils.BISECT_MODE_RETURN_CODE:
 273       mean = '%d' % revision_state.value['mean']
 274       self._PrintTableRow(
 275           [20, 12, 70, 14, 13],
 276           [revision_state.depot, commit_position, cl_link, mean,
 277            state_str])
 278
 279   def _PrintTestedCommitsTable(
 280       self, revision_states, first_working_revision, last_broken_revision,
 281       confidence, final_step=True):
 282     print
 283     if final_step:
 284       print '===== TESTED COMMITS ====='
 285     else:
 286       print '===== PARTIAL RESULTS ====='
 287     self._PrintTestedCommitsHeader()
 288     state = 0
 289     for revision_state in revision_states:
 290       if revision_state.value:
 291         if (revision_state == last_broken_revision or
 292             revision_state == first_working_revision):
 293           # If confidence is too low, don't add this empty line since it's
 294           # used to put focus on a suspected CL.
 295           if confidence and final_step:
 296             print
 297           state += 1
 298           if state == 2 and not final_step:
 299             # Just want a separation between "bad" and "good" cl's.
 300             print
 301
 302         state_str = 'Bad'
 303         if state == 1 and final_step:
 304           state_str = 'Suspected CL'
 305         elif state == 2:
 306           state_str = 'Good'
 307
 308         # If confidence is too low, don't bother outputting good/bad.
 309         if not confidence:
 310           state_str = ''
 311         state_str = state_str.center(13, ' ')
 312         commit_position = source_control.GetCommitPosition(
 313             revision_state.revision,
 314             self.depot_registry.GetDepotDir(revision_state.depot))
 315         display_commit_pos = ''
 316         if commit_position:
 317           display_commit_pos = str(commit_position)
 318         self._PrintTestedCommitsEntry(revision_state,
 319                                       display_commit_pos,
 320                                       revision_state.revision,
 321                                       state_str)
 322
 323   def _PrintRetestResults(self, bisect_results):
 324     if (not bisect_results.retest_results_tot or
 325         not bisect_results.retest_results_reverted):
 326       return
 327     print
 328     print '===== RETEST RESULTS ====='
 329     self._PrintTestedCommitsEntry(
 330         bisect_results.retest_results_tot, '', '', '')
 331     self._PrintTestedCommitsEntry(
 332         bisect_results.retest_results_reverted, '', '', '')
 333
 334   def _PrintReproSteps(self):
 335     """Prints out a section of the results explaining how to run the test.
 336
 337     This message includes the command used to run the test.
 338     """
 339     command = '$ ' + self.opts.command
 340     if bisect_utils.IsTelemetryCommand(self.opts.command):
 341       command += ('\nAlso consider passing --profiler=list to see available '
 342                   'profilers.')
 343     print REPRO_STEPS_LOCAL
 344     if bisect_utils.IsTelemetryCommand(self.opts.command):
 345       telemetry_command = re.sub(r'--browser=[^\s]+',
 346                                  '--browser=<bot-name>',
 347                                  command)
 348       print REPRO_STEPS_TRYJOB_TELEMETRY % {'command': telemetry_command}
 349     else:
 350       print REPRO_STEPS_TRYJOB
 351
 352   def _PrintOtherRegressions(self, other_regressions):
 353     """Prints a section of the results about other potential regressions."""
 354     print
 355     print 'Other regressions may have occurred:'
 356     self._PrintTableRow([8, 70, 10], ['Depot', 'Range', 'Confidence'])
 357     for regression in other_regressions:
 358       current_rev_state, prev_rev_state, confidence = regression
 359       self._PrintTableRow(
 360           [8, 70, 10],
 361           [current_rev_state.depot, current_rev_state.revision,
 362            '%d%%' % confidence])
 363       self._PrintTableRow(
 364           [8, 70], [prev_rev_state.depot, prev_rev_state.revision])
 365       print
 366
 367   @staticmethod
 368   def _ConfidenceLevelStatus(bisect_results):
 369     if not bisect_results.confidence:
 370       return None
 371     confidence_status = 'Successful with %(level)s confidence%(warning)s.'
 372     if bisect_results.confidence >= bisect_utils.HIGH_CONFIDENCE:
 373       level = 'high'
 374     else:
 375       level = 'low'
 376     warning = ' and warnings'
 377     if not bisect_results.warnings:
 378       warning = ''
 379     return confidence_status % {'level': level, 'warning': warning}
 380
 381   def _PrintBanner(self, bisect_results):
 382     if self.opts.bisect_mode == bisect_utils.BISECT_MODE_RETURN_CODE:
 383       metrics = 'N/A'
 384       change = 'Yes'
 385     else:
 386       metrics = '/'.join(self.opts.metric)
 387       change = '%.02f%% (+/-%.02f%%)' % (
 388           bisect_results.regression_size, bisect_results.regression_std_err)
 389
 390     if bisect_results.culprit_revisions and bisect_results.confidence:
 391       status = self._ConfidenceLevelStatus(bisect_results)
 392     else:
 393       status = 'Failure, could not reproduce.'
 394       change = 'Bisect could not reproduce a change.'
 395
 396     retest_text = 'Yes' if bisect_results.retest_results_tot else 'No'
 397
 398     print RESULTS_BANNER % {
 399         'status': status,
 400         'command': self.opts.command,
 401         'metrics': metrics,
 402         'change': change,
 403         'confidence': bisect_results.confidence,
 404         'retest': retest_text,
 405     }
 406
 407   @staticmethod
 408   def _PrintWarnings(warnings):
 409     """Prints a list of warning strings if there are any."""
 410     if not warnings:
 411       return
 412     print
 413     print 'WARNINGS:'
 414     for w in set(warnings):
 415       print '  ! %s' % w