tools/auto_bisect/bisect_printer.py

   1 # Copyright 2014 The Chromium Authors. All rights reserved.
   2 # Use of this source code is governed by a BSD-style license that can be
   3 # found in the LICENSE file.
   4
   5 """This file contains printing-related functionality of the bisect."""
   6
   7 import datetime
   8 import re
   9
  10 from bisect_results import BisectResults
  11 import bisect_utils
  12 import source_control
  13
  14
  15 # The perf dashboard looks for a string like "Estimated Confidence: 95%"
  16 # to decide whether or not to cc the author(s). If you change this, please
  17 # update the perf dashboard as well.
  18 RESULTS_BANNER = """
  19 ===== BISECT JOB RESULTS =====
  20 Status: %(status)s
  21
  22 Test Command: %(command)s
  23 Test Metric: %(metrics)s
  24 Relative Change: %(change)s
  25 Estimated Confidence: %(confidence).02f%%"""
  26
  27 # When the bisect was aborted without a bisect failure the following template
  28 # is used.
  29 ABORT_REASON_TEMPLATE = """
  30 ===== BISECTION ABORTED =====
  31 The bisect was aborted because %(abort_reason)s
  32 Please contact the the team (see below) if you believe this is in error.
  33
  34 Bug ID: %(bug_id)s
  35
  36 Test Command: %(command)s
  37 Test Metric: %(metric)s
  38 Good revision: %(good_revision)s
  39 Bad revision: %(bad_revision)s """
  40
  41 # The perf dashboard specifically looks for the string
  42 # "Author  : " to parse out who to cc on a bug. If you change the
  43 # formatting here, please update the perf dashboard as well.
  44 RESULTS_REVISION_INFO = """
  45 ===== SUSPECTED CL(s) =====
  46 Subject : %(subject)s
  47 Author  : %(author)s%(commit_info)s
  48 Commit  : %(cl)s
  49 Date    : %(cl_date)s"""
  50
  51 RESULTS_THANKYOU = """
  52 | O O | Visit http://www.chromium.org/developers/core-principles for Chrome's
  53 |  X  | policy on perf regressions. Contact chrome-perf-dashboard-team with any
  54 | / \\ | questions or suggestions about bisecting. THANK YOU."""
  55
  56 REPRO_STEPS_LOCAL = """
  57 ==== INSTRUCTIONS TO REPRODUCE ====
  58 To run locally:
  59  - Use the test command given under 'BISECT JOB RESULTS' above.
  60  - Consider using a profiler. Pass --profiler=list to list available profilers.
  61 """
  62
  63 REPRO_STEPS_TRYJOB = """
  64 To reproduce on a performance try bot:
  65  1. Edit run-perf-test.cfg
  66  2. $ git try -b <bot> --svn_repo='svn://svn.chromium.org/chrome-try/try-perf'
  67
  68 Notes:
  69  a) Follow the in-file instructions in run-perf-test.cfg.
  70  b) run-perf-test.cfg is under tools/ or under third_party/WebKit/Tools.
  71  c) Do your edits preferably under a new git branch.
  72  d) --browser=release and --browser=android-chromium-testshell are supported
  73     depending on the platform (desktop|android).
  74  e) Strip any src/ directories from the head of relative path names.
  75  f) Make sure to use the appropriate bot on step 3.
  76
  77 For more details please visit
  78 https://sites.google.com/a/chromium.org/dev/developers/performance-try-bots"""
  79
  80 REPRO_STEPS_TRYJOB_TELEMETRY = """
  81 To reproduce on a performance try bot:
  82 %(command)s
  83 (Where <bot-name> comes from tools/perf/run_benchmark --browser=list)
  84
  85 For more details please visit
  86 https://sites.google.com/a/chromium.org/dev/developers/performance-try-bots
  87 """
  88
  89
  90 class BisectPrinter(object):
  91
  92   def __init__(self, opts, depot_registry):
  93     self.opts = opts
  94     self.depot_registry = depot_registry
  95
  96   def FormatAndPrintResults(self, bisect_results):
  97     """Prints the results from a bisection run in a readable format.
  98
  99     Also prints annotations creating buildbot step "Results".
 100
 101     Args:
 102       bisect_results: BisectResult object containing results to be printed.
 103     """
 104     if bisect_results.abort_reason:
 105       self._PrintAbortResults(bisect_results.abort_reason)
 106       return
 107
 108     if self.opts.output_buildbot_annotations:
 109       bisect_utils.OutputAnnotationStepStart('Build Status Per Revision')
 110
 111     print
 112     print 'Full results of bisection:'
 113     for revision_state in bisect_results.state.GetRevisionStates():
 114       build_status = revision_state.passed
 115
 116       if type(build_status) is bool:
 117         if build_status:
 118           build_status = 'Good'
 119         else:
 120           build_status = 'Bad'
 121
 122       print '  %20s  %40s  %s' % (revision_state.depot, revision_state.revision,
 123                                   build_status)
 124     print
 125
 126     if self.opts.output_buildbot_annotations:
 127       bisect_utils.OutputAnnotationStepClosed()
 128       # The perf dashboard scrapes the "results" step in order to comment on
 129       # bugs. If you change this, please update the perf dashboard as well.
 130       bisect_utils.OutputAnnotationStepStart('Results')
 131
 132     self._PrintBanner(bisect_results)
 133     self._PrintWarnings(bisect_results.warnings)
 134
 135     if bisect_results.culprit_revisions and bisect_results.confidence:
 136       for culprit in bisect_results.culprit_revisions:
 137         cl, info, depot = culprit
 138         self._PrintRevisionInfo(cl, info, depot)
 139       if bisect_results.other_regressions:
 140         self._PrintOtherRegressions(bisect_results.other_regressions)
 141     self._PrintTestedCommitsTable(bisect_results.state.GetRevisionStates(),
 142                                   bisect_results.first_working_revision,
 143                                   bisect_results.last_broken_revision,
 144                                   bisect_results.confidence)
 145     self._PrintStepTime(bisect_results.state.GetRevisionStates())
 146     self._PrintReproSteps()
 147     self._PrintThankYou()
 148     if self.opts.output_buildbot_annotations:
 149       bisect_utils.OutputAnnotationStepClosed()
 150
 151   def PrintPartialResults(self, bisect_state):
 152     revision_states = bisect_state.GetRevisionStates()
 153     first_working_rev, last_broken_rev = BisectResults.FindBreakingRevRange(
 154         revision_states)
 155     self._PrintTestedCommitsTable(revision_states, first_working_rev,
 156                                   last_broken_rev, 100, final_step=False)
 157
 158   def _PrintAbortResults(self, abort_reason):
 159
 160     if self.opts.output_buildbot_annotations:
 161       bisect_utils.OutputAnnotationStepStart('Results')
 162     print ABORT_REASON_TEMPLATE % {
 163         'abort_reason': abort_reason,
 164         'bug_id': self.opts.bug_id or 'NOT SPECIFIED',
 165         'command': self.opts.command,
 166         'metric': '/'.join(self.opts.metric),
 167         'good_revision': self.opts.good_revision,
 168         'bad_revision': self.opts.bad_revision,
 169     }
 170     self._PrintThankYou()
 171     if self.opts.output_buildbot_annotations:
 172       bisect_utils.OutputAnnotationStepClosed()
 173
 174   @staticmethod
 175   def _PrintThankYou():
 176     print RESULTS_THANKYOU
 177
 178   @staticmethod
 179   def _PrintStepTime(revision_states):
 180     """Prints information about how long various steps took.
 181
 182     Args:
 183       revision_states: Ordered list of revision states."""
 184     step_perf_time_avg = 0.0
 185     step_build_time_avg = 0.0
 186     step_count = 0.0
 187     for revision_state in revision_states:
 188       if revision_state.value:
 189         step_perf_time_avg += revision_state.perf_time
 190         step_build_time_avg += revision_state.build_time
 191         step_count += 1
 192     if step_count:
 193       step_perf_time_avg = step_perf_time_avg / step_count
 194       step_build_time_avg = step_build_time_avg / step_count
 195     print
 196     print 'Average build time : %s' % datetime.timedelta(
 197         seconds=int(step_build_time_avg))
 198     print 'Average test time  : %s' % datetime.timedelta(
 199         seconds=int(step_perf_time_avg))
 200
 201   @staticmethod
 202   def _GetViewVCLinkFromDepotAndHash(git_revision, depot):
 203     """Gets link to the repository browser."""
 204     if depot and bisect_utils.DEPOT_DEPS_NAME[depot].has_key('viewvc'):
 205       return bisect_utils.DEPOT_DEPS_NAME[depot]['viewvc'] + git_revision
 206     return ''
 207
 208   def _PrintRevisionInfo(self, cl, info, depot=None):
 209     commit_link = self._GetViewVCLinkFromDepotAndHash(cl, depot)
 210     if commit_link:
 211       commit_link = '\nLink    : %s' % commit_link
 212     else:
 213       commit_link = ('\Description:\n%s' % info['body'])
 214     print RESULTS_REVISION_INFO % {
 215         'subject': info['subject'],
 216         'author': info['email'],
 217         'commit_info': commit_link,
 218         'cl': cl,
 219         'cl_date': info['date']
 220     }
 221
 222   @staticmethod
 223   def _PrintTableRow(column_widths, row_data):
 224     """Prints out a row in a formatted table that has columns aligned.
 225
 226     Args:
 227       column_widths: A list of column width numbers.
 228       row_data: A list of items for each column in this row.
 229     """
 230     assert len(column_widths) == len(row_data)
 231     text = ''
 232     for i in xrange(len(column_widths)):
 233       current_row_data = row_data[i].center(column_widths[i], ' ')
 234       text += ('%%%ds' % column_widths[i]) % current_row_data
 235     print text
 236
 237   def _PrintTestedCommitsHeader(self):
 238     if self.opts.bisect_mode == bisect_utils.BISECT_MODE_MEAN:
 239       self._PrintTableRow(
 240           [20, 12, 70, 14, 12, 13],
 241           ['Depot', 'Position', 'SHA', 'Mean', 'Std. Error', 'State'])
 242     elif self.opts.bisect_mode == bisect_utils.BISECT_MODE_STD_DEV:
 243       self._PrintTableRow(
 244           [20, 12, 70, 14, 12, 13],
 245           ['Depot', 'Position', 'SHA', 'Std. Error', 'Mean', 'State'])
 246     elif self.opts.bisect_mode == bisect_utils.BISECT_MODE_RETURN_CODE:
 247       self._PrintTableRow(
 248           [20, 12, 70, 14, 13],
 249           ['Depot', 'Position', 'SHA', 'Return Code', 'State'])
 250     else:
 251       assert False, 'Invalid bisect_mode specified.'
 252
 253   def _PrintTestedCommitsEntry(self, revision_state, commit_position, cl_link,
 254                                state_str):
 255     if self.opts.bisect_mode == bisect_utils.BISECT_MODE_MEAN:
 256       std_error = '+-%.02f' % revision_state.value['std_err']
 257       mean = '%.02f' % revision_state.value['mean']
 258       self._PrintTableRow(
 259           [20, 12, 70, 12, 14, 13],
 260           [revision_state.depot, commit_position, cl_link, mean, std_error,
 261            state_str])
 262     elif self.opts.bisect_mode == bisect_utils.BISECT_MODE_STD_DEV:
 263       std_error = '+-%.02f' % revision_state.value['std_err']
 264       mean = '%.02f' % revision_state.value['mean']
 265       self._PrintTableRow(
 266           [20, 12, 70, 12, 14, 13],
 267           [revision_state.depot, commit_position, cl_link, std_error, mean,
 268            state_str])
 269     elif self.opts.bisect_mode == bisect_utils.BISECT_MODE_RETURN_CODE:
 270       mean = '%d' % revision_state.value['mean']
 271       self._PrintTableRow(
 272           [20, 12, 70, 14, 13],
 273           [revision_state.depot, commit_position, cl_link, mean,
 274            state_str])
 275
 276   def _PrintTestedCommitsTable(
 277       self, revision_states, first_working_revision, last_broken_revision,
 278       confidence, final_step=True):
 279     print
 280     if final_step:
 281       print '===== TESTED COMMITS ====='
 282     else:
 283       print '===== PARTIAL RESULTS ====='
 284     self._PrintTestedCommitsHeader()
 285     state = 0
 286     for revision_state in revision_states:
 287       if revision_state.value:
 288         if (revision_state == last_broken_revision or
 289             revision_state == first_working_revision):
 290           # If confidence is too low, don't add this empty line since it's
 291           # used to put focus on a suspected CL.
 292           if confidence and final_step:
 293             print
 294           state += 1
 295           if state == 2 and not final_step:
 296             # Just want a separation between "bad" and "good" cl's.
 297             print
 298
 299         state_str = 'Bad'
 300         if state == 1 and final_step:
 301           state_str = 'Suspected CL'
 302         elif state == 2:
 303           state_str = 'Good'
 304
 305         # If confidence is too low, don't bother outputting good/bad.
 306         if not confidence:
 307           state_str = ''
 308         state_str = state_str.center(13, ' ')
 309         commit_position = source_control.GetCommitPosition(
 310             revision_state.revision,
 311             self.depot_registry.GetDepotDir(revision_state.depot))
 312         display_commit_pos = ''
 313         if commit_position:
 314           display_commit_pos = str(commit_position)
 315         self._PrintTestedCommitsEntry(revision_state,
 316                                       display_commit_pos,
 317                                       revision_state.revision,
 318                                       state_str)
 319
 320   def _PrintReproSteps(self):
 321     """Prints out a section of the results explaining how to run the test.
 322
 323     This message includes the command used to run the test.
 324     """
 325     command = '$ ' + self.opts.command
 326     if bisect_utils.IsTelemetryCommand(self.opts.command):
 327       command += ('\nAlso consider passing --profiler=list to see available '
 328                   'profilers.')
 329     print REPRO_STEPS_LOCAL
 330     if bisect_utils.IsTelemetryCommand(self.opts.command):
 331       telemetry_command = re.sub(r'--browser=[^\s]+',
 332                                  '--browser=<bot-name>',
 333                                  command)
 334       print REPRO_STEPS_TRYJOB_TELEMETRY % {'command': telemetry_command}
 335     else:
 336       print REPRO_STEPS_TRYJOB
 337
 338   def _PrintOtherRegressions(self, other_regressions):
 339     """Prints a section of the results about other potential regressions."""
 340     print
 341     print 'Other regressions may have occurred:'
 342     self._PrintTableRow([8, 70, 10], ['Depot', 'Range', 'Confidence'])
 343     for regression in other_regressions:
 344       current_rev_state, prev_rev_state, confidence = regression
 345       self._PrintTableRow(
 346           [8, 70, 10],
 347           [current_rev_state.depot, current_rev_state.revision,
 348            '%d%%' % confidence])
 349       self._PrintTableRow(
 350           [8, 70], [prev_rev_state.depot, prev_rev_state.revision])
 351       print
 352
 353   @staticmethod
 354   def _ConfidenceLevelStatus(bisect_results):
 355     if not bisect_results.confidence:
 356       return None
 357     confidence_status = 'Successful with %(level)s confidence%(warning)s.'
 358     if bisect_results.confidence >= bisect_utils.HIGH_CONFIDENCE:
 359       level = 'high'
 360     else:
 361       level = 'low'
 362     warning = ' and warnings'
 363     if not bisect_results.warnings:
 364       warning = ''
 365     return confidence_status % {'level': level, 'warning': warning}
 366
 367   def _PrintBanner(self, bisect_results):
 368     if self.opts.bisect_mode == bisect_utils.BISECT_MODE_RETURN_CODE:
 369       metrics = 'N/A'
 370       change = 'Yes'
 371     else:
 372       metrics = '/'.join(self.opts.metric)
 373       change = '%.02f%% (+/-%.02f%%)' % (
 374           bisect_results.regression_size, bisect_results.regression_std_err)
 375
 376     if bisect_results.culprit_revisions and bisect_results.confidence:
 377       status = self._ConfidenceLevelStatus(bisect_results)
 378     else:
 379       status = 'Failure, could not reproduce.'
 380       change = 'Bisect could not reproduce a change.'
 381
 382     print RESULTS_BANNER % {
 383         'status': status,
 384         'command': self.opts.command,
 385         'metrics': metrics,
 386         'change': change,
 387         'confidence': bisect_results.confidence,
 388     }
 389
 390   @staticmethod
 391   def _PrintWarnings(warnings):
 392     """Prints a list of warning strings if there are any."""
 393     if not warnings:
 394       return
 395     print
 396     print 'WARNINGS:'
 397     for w in set(warnings):
 398       print '  ! %s' % w