scripts/simplebench/simplebench.py

   1 #!/usr/bin/env python
   2 #
   3 # Simple benchmarking framework
   4 #
   5 # Copyright (c) 2019 Virtuozzo International GmbH.
   6 #
   7 # This program is free software; you can redistribute it and/or modify
   8 # it under the terms of the GNU General Public License as published by
   9 # the Free Software Foundation; either version 2 of the License, or
  10 # (at your option) any later version.
  11 #
  12 # This program is distributed in the hope that it will be useful,
  13 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  14 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  15 # GNU General Public License for more details.
  16 #
  17 # You should have received a copy of the GNU General Public License
  18 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
  19 #
  20
  21
  22 def bench_one(test_func, test_env, test_case, count=5, initial_run=True):
  23     """Benchmark one test-case
  24
  25     test_func   -- benchmarking function with prototype
  26                    test_func(env, case), which takes test_env and test_case
  27                    arguments and returns {'seconds': int} (which is benchmark
  28                    result) on success and {'error': str} on error. Returned
  29                    dict may contain any other additional fields.
  30     test_env    -- test environment - opaque first argument for test_func
  31     test_case   -- test case - opaque second argument for test_func
  32     count       -- how many times to call test_func, to calculate average
  33     initial_run -- do initial run of test_func, which don't get into result
  34
  35     Returns dict with the following fields:
  36         'runs':     list of test_func results
  37         'average':  average seconds per run (exists only if at least one run
  38                     succeeded)
  39         'delta':    maximum delta between test_func result and the average
  40                     (exists only if at least one run succeeded)
  41         'n-failed': number of failed runs (exists only if at least one run
  42                     failed)
  43     """
  44     if initial_run:
  45         print('  #initial run:')
  46         print('   ', test_func(test_env, test_case))
  47
  48     runs = []
  49     for i in range(count):
  50         print('  #run {}'.format(i+1))
  51         res = test_func(test_env, test_case)
  52         print('   ', res)
  53         runs.append(res)
  54
  55     result = {'runs': runs}
  56
  57     successed = [r for r in runs if ('seconds' in r)]
  58     if successed:
  59         avg = sum(r['seconds'] for r in successed) / len(successed)
  60         result['average'] = avg
  61         result['delta'] = max(abs(r['seconds'] - avg) for r in successed)
  62
  63     if len(successed) < count:
  64         result['n-failed'] = count - len(successed)
  65
  66     return result
  67
  68
  69 def ascii_one(result):
  70     """Return ASCII representation of bench_one() returned dict."""
  71     if 'average' in result:
  72         s = '{:.2f} +- {:.2f}'.format(result['average'], result['delta'])
  73         if 'n-failed' in result:
  74             s += '\n({} failed)'.format(result['n-failed'])
  75         return s
  76     else:
  77         return 'FAILED'
  78
  79
  80 def bench(test_func, test_envs, test_cases, *args, **vargs):
  81     """Fill benchmark table
  82
  83     test_func -- benchmarking function, see bench_one for description
  84     test_envs -- list of test environments, see bench_one
  85     test_cases -- list of test cases, see bench_one
  86     args, vargs -- additional arguments for bench_one
  87
  88     Returns dict with the following fields:
  89         'envs':  test_envs
  90         'cases': test_cases
  91         'tab':   filled 2D array, where cell [i][j] is bench_one result for
  92                  test_cases[i] for test_envs[j] (i.e., rows are test cases and
  93                  columns are test environments)
  94     """
  95     tab = {}
  96     results = {
  97         'envs': test_envs,
  98         'cases': test_cases,
  99         'tab': tab
 100     }
 101     n = 1
 102     n_tests = len(test_envs) * len(test_cases)
 103     for env in test_envs:
 104         for case in test_cases:
 105             print('Testing {}/{}: {} :: {}'.format(n, n_tests,
 106                                                    env['id'], case['id']))
 107             if case['id'] not in tab:
 108                 tab[case['id']] = {}
 109             tab[case['id']][env['id']] = bench_one(test_func, env, case,
 110                                                    *args, **vargs)
 111             n += 1
 112
 113     print('Done')
 114     return results
 115
 116
 117 def ascii(results):
 118     """Return ASCII representation of bench() returned dict."""
 119     from tabulate import tabulate
 120
 121     tab = [[""] + [c['id'] for c in results['envs']]]
 122     for case in results['cases']:
 123         row = [case['id']]
 124         for env in results['envs']:
 125             row.append(ascii_one(results['tab'][case['id']][env['id']]))
 126         tab.append(row)
 127
 128     return tabulate(tab)