build/util/lib/common/perf_tests_results_helper.py

   1 # Copyright 2013 The Chromium Authors. All rights reserved.
   2 # Use of this source code is governed by a BSD-style license that can be
   3 # found in the LICENSE file.
   4
   5 import re
   6 import sys
   7
   8 import json
   9 import logging
  10 import math
  11
  12 import perf_result_data_type
  13
  14
  15 # Mapping from result type to test output
  16 RESULT_TYPES = {perf_result_data_type.UNIMPORTANT: 'RESULT ',
  17                 perf_result_data_type.DEFAULT: '*RESULT ',
  18                 perf_result_data_type.INFORMATIONAL: '',
  19                 perf_result_data_type.UNIMPORTANT_HISTOGRAM: 'HISTOGRAM ',
  20                 perf_result_data_type.HISTOGRAM: '*HISTOGRAM '}
  21
  22
  23 def _EscapePerfResult(s):
  24   """Escapes |s| for use in a perf result."""
  25   return re.sub('[\:|=/#&,]', '_', s)
  26
  27
  28 def FlattenList(values):
  29   """Returns a simple list without sub-lists."""
  30   ret = []
  31   for entry in values:
  32     if isinstance(entry, list):
  33       ret.extend(FlattenList(entry))
  34     else:
  35       ret.append(entry)
  36   return ret
  37
  38
  39 def GeomMeanAndStdDevFromHistogram(histogram_json):
  40   histogram = json.loads(histogram_json)
  41   # Handle empty histograms gracefully.
  42   if not 'buckets' in histogram:
  43     return 0.0, 0.0
  44   count = 0
  45   sum_of_logs = 0
  46   for bucket in histogram['buckets']:
  47     if 'high' in bucket:
  48       bucket['mean'] = (bucket['low'] + bucket['high']) / 2.0
  49     else:
  50       bucket['mean'] = bucket['low']
  51     if bucket['mean'] > 0:
  52       sum_of_logs += math.log(bucket['mean']) * bucket['count']
  53       count += bucket['count']
  54
  55   if count == 0:
  56     return 0.0, 0.0
  57
  58   sum_of_squares = 0
  59   geom_mean = math.exp(sum_of_logs / count)
  60   for bucket in histogram['buckets']:
  61     if bucket['mean'] > 0:
  62       sum_of_squares += (bucket['mean'] - geom_mean) ** 2 * bucket['count']
  63   return geom_mean, math.sqrt(sum_of_squares / count)
  64
  65
  66 def _ValueToString(v):
  67   # Special case for floats so we don't print using scientific notation.
  68   if isinstance(v, float):
  69     return '%f' % v
  70   else:
  71     return str(v)
  72
  73
  74 def _MeanAndStdDevFromList(values):
  75   avg = None
  76   sd = None
  77   if len(values) > 1:
  78     try:
  79       value = '[%s]' % ','.join([_ValueToString(v) for v in values])
  80       avg = sum([float(v) for v in values]) / len(values)
  81       sqdiffs = [(float(v) - avg) ** 2 for v in values]
  82       variance = sum(sqdiffs) / (len(values) - 1)
  83       sd = math.sqrt(variance)
  84     except ValueError:
  85       value = ', '.join(values)
  86   else:
  87     value = values[0]
  88   return value, avg, sd
  89
  90
  91 def PrintPages(page_list):
  92   """Prints list of pages to stdout in the format required by perf tests."""
  93   print 'Pages: [%s]' % ','.join([_EscapePerfResult(p) for p in page_list])
  94
  95
  96 def PrintPerfResult(measurement, trace, values, units,
  97                     result_type=perf_result_data_type.DEFAULT,
  98                     print_to_stdout=True):
  99   """Prints numerical data to stdout in the format required by perf tests.
 100
 101   The string args may be empty but they must not contain any colons (:) or
 102   equals signs (=).
 103   This is parsed by the buildbot using:
 104   http://src.chromium.org/viewvc/chrome/trunk/tools/build/scripts/slave/process_log_utils.py
 105
 106   Args:
 107     measurement: A description of the quantity being measured, e.g. "vm_peak".
 108         On the dashboard, this maps to a particular graph. Mandatory.
 109     trace: A description of the particular data point, e.g. "reference".
 110         On the dashboard, this maps to a particular "line" in the graph.
 111         Mandatory.
 112     values: A list of numeric measured values. An N-dimensional list will be
 113         flattened and treated as a simple list.
 114     units: A description of the units of measure, e.g. "bytes".
 115     result_type: Accepts values of perf_result_data_type.ALL_TYPES.
 116     print_to_stdout: If True, prints the output in stdout instead of returning
 117         the output to caller.
 118
 119     Returns:
 120       String of the formated perf result.
 121   """
 122   assert perf_result_data_type.IsValidType(result_type), \
 123          'result type: %s is invalid' % result_type
 124
 125   trace_name = _EscapePerfResult(trace)
 126
 127   if (result_type == perf_result_data_type.UNIMPORTANT or
 128       result_type == perf_result_data_type.DEFAULT or
 129       result_type == perf_result_data_type.INFORMATIONAL):
 130     assert isinstance(values, list)
 131     assert '/' not in measurement
 132     flattened_values = FlattenList(values)
 133     assert len(flattened_values)
 134     value, avg, sd = _MeanAndStdDevFromList(flattened_values)
 135     output = '%s%s: %s%s%s %s' % (
 136         RESULT_TYPES[result_type],
 137         _EscapePerfResult(measurement),
 138         trace_name,
 139         # Do not show equal sign if the trace is empty. Usually it happens when
 140         # measurement is enough clear to describe the result.
 141         '= ' if trace_name else '',
 142         value,
 143         units)
 144   else:
 145     assert perf_result_data_type.IsHistogram(result_type)
 146     assert isinstance(values, list)
 147     # The histograms can only be printed individually, there's no computation
 148     # across different histograms.
 149     assert len(values) == 1
 150     value = values[0]
 151     output = '%s%s: %s= %s %s' % (
 152         RESULT_TYPES[result_type],
 153         _EscapePerfResult(measurement),
 154         trace_name,
 155         value,
 156         units)
 157     avg, sd = GeomMeanAndStdDevFromHistogram(value)
 158
 159   if avg:
 160     output += '\nAvg %s: %f%s' % (measurement, avg, units)
 161   if sd:
 162     output += '\nSd  %s: %f%s' % (measurement, sd, units)
 163   if print_to_stdout:
 164     print output
 165     sys.stdout.flush()
 166   return output