bench/bench.py

   1 #!/usr/bin/env python
   2 # MIT License
   3 #
   4 # Copyright The SCons Foundation
   5 #
   6 # Permission is hereby granted, free of charge, to any person obtaining
   7 # a copy of this software and associated documentation files (the
   8 # "Software"), to deal in the Software without restriction, including
   9 # without limitation the rights to use, copy, modify, merge, publish,
  10 # distribute, sublicense, and/or sell copies of the Software, and to
  11 # permit persons to whom the Software is furnished to do so, subject to
  12 # the following conditions:
  13 #
  14 # The above copyright notice and this permission notice shall be included
  15 # in all copies or substantial portions of the Software.
  16 #
  17 # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY
  18 # KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE
  19 # WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  20 # NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
  21 # LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
  22 # OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
  23 # WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
  24
  25 """
  26 A script for timing snippets of Python code.
  27
  28 By default, this script will execute a single Python file specified on
  29 the command line and time any functions in a list named "FunctionList"
  30 set by the Python file under test, or (by default) time any functions
  31 in the file whose names begin with "Func".
  32
  33 All functions are assumed to get passed the same arguments, and the
  34 inputs are specified in a list named "Data," each element of which
  35 is a list consisting of a tag name, a list of positional arguments,
  36 and a dictionary of keyword arguments.
  37
  38 Each function is expected to test a single, comparable snippet of
  39 of Python code.  IMPORTANT:  We want to test the timing of the code
  40 itself, not Python function call overhead, so every function should
  41 put its code under test within the following block:
  42
  43       for i in IterationList:
  44
  45 This will allow (as much as possible) us to time just the code itself,
  46 not Python function call overhead.
  47 """
  48 from __future__ import division, print_function
  49
  50 import getopt
  51 import os.path
  52 import sys
  53 import time
  54 import types
  55 from pprint import pprint
  56
  57
  58 Usage = """\
  59 Usage:  bench.py OPTIONS file.py
  60   --clock                       Use the time.clock function
  61   --func PREFIX                 Test functions whose names begin with PREFIX
  62   -h, --help                    Display this help and exit
  63   -i ITER, --iterations ITER    Run each code snippet ITER times
  64   --time                        Use the time.time function
  65   -r RUNS, --runs RUNS          Average times for RUNS invocations of
  66 """
  67
  68 # How many times each snippet of code will be (or should be) run by the
  69 # functions under test to gather the time (the "inner loop").
  70
  71 Iterations = 1000
  72
  73 # How many times we'll run each function to collect its aggregate time
  74 # and try to average out timing differences induced by system performance
  75 # (the "outer loop").
  76
  77 Runs = 10
  78
  79 # The prefix of the functions under test.  This will be used if
  80 # there's no explicit list defined in FunctionList.
  81
  82 FunctionPrefix = 'Func'
  83
  84 # Comment from Python2 timeit module:
  85 # The difference in default timer function is because on Windows,
  86 # clock() has microsecond granularity but time()'s granularity is 1/60th
  87 # of a second; on Unix, clock() has 1/100th of a second granularity and
  88 # time() is much more precise.  On either platform, the default timer
  89 # functions measure wall clock time, not the CPU time.  This means that
  90 # other processes running on the same computer may interfere with the
  91 # timing.  The best thing to do when accurate timing is necessary is to
  92 # repeat the timing a few times and use the best time.  The -i option is
  93 # good for this.
  94 # On Python3, a new time.perf_counter function picks the best available
  95 # timer, so we use that if we can, else fall back as per above.
  96
  97 try:
  98     Now = time.perf_counter
  99 except AttributeError:
 100     if sys.platform == 'win32':
 101         Now = time.clock
 102     else:
 103         Now = time.time
 104
 105 opts, args = getopt.getopt(sys.argv[1:], 'hi:r:',
 106                            ['clock', 'func=', 'help',
 107                             'iterations=', 'time', 'runs='])
 108
 109 for o, a in opts:
 110     if o in ['--clock']:
 111         try:
 112             Now = time.clock
 113         except AttributeError:
 114             sys.stderr.write("time.clock unavailable on this Python\n")
 115             sys.exit(1)
 116     elif o in ['--func']:
 117         FunctionPrefix = a
 118     elif o in ['-h', '--help']:
 119         sys.stdout.write(Usage)
 120         sys.exit(0)
 121     elif o in ['-i', '--iterations']:
 122         Iterations = int(a)
 123     elif o in ['--time']:
 124         Now = time.time
 125     elif o in ['-r', '--runs']:
 126         Runs = int(a)
 127
 128 if len(args) != 1:
 129     sys.stderr.write("bench.py:  only one file argument must be specified\n")
 130     sys.stderr.write(Usage)
 131     sys.exit(1)
 132
 133
 134 (bench_name,_) = os.path.splitext(args[0])
 135 print(f"Bench: {bench_name} : {args[0]}")
 136 results_filename = f"{bench_name}_results.csv"
 137
 138 with open(args[0], 'r') as f:
 139     exec(f.read())
 140
 141 try:
 142     FunctionList
 143 except NameError:
 144     function_names = sorted([x for x in list(locals().keys()) if x[:4] == FunctionPrefix])
 145     lvars = locals()
 146     l = [lvars[f] for f in function_names]
 147     FunctionList = [f for f in l if isinstance(f, types.FunctionType)]
 148
 149 IterationList = [None] * Iterations
 150
 151
 152 def timer(func, *args, **kw):
 153     results = []
 154     for i in range(Runs):
 155         start = Now()
 156         func(*args, **kw)
 157         finish = Now()
 158         results.append((finish - start) / Iterations)
 159     return results
 160
 161
 162 results_dict = {}
 163
 164
 165 def display(func, result_label, results):
 166     total = 0.0
 167     for r in results:
 168         total += r
 169
 170     test_result = (total * 1e6) / len(results)
 171     print("    %8.3f" % (test_result), ':', result_label)
 172     if results_dict.get(func.__doc__, False):
 173         results_dict[func.__doc__][result_label] = test_result
 174     else:
 175         results_dict[func.__doc__] = { result_label: test_result}
 176
 177
 178 for func in FunctionList:
 179     if func.__doc__:
 180         d = ' (' + func.__doc__ + ')'
 181     else:
 182         d = ''
 183     print(func.__name__ + d + ':')
 184
 185     for label, args, kw in Data:
 186         r = timer(func, *args, **kw)
 187         # print(f"{label} {r}")
 188         display(func, label, r)
 189
 190 py_ver_string = "%d.%d"%(sys.version_info.major, sys.version_info.minor)
 191
 192
 193 # pprint(results_dict)
 194
 195 tests = [label for label, args, kw in Data]
 196 columns = ['Python Version', 'Implementation'] + tests
 197 with open(results_filename, 'a') as r:
 198     print(",".join(columns), file=r)
 199
 200     for implementation in results_dict:
 201         print(f'{py_ver_string},"{implementation}"', file=r, end='')
 202         for test in tests:
 203             print(',%8.3f' % results_dict[implementation][test], file=r, end='')
 204         print(file=r)
 205
 206 # Local Variables:
 207 # tab-width:4
 208 # indent-tabs-mode:nil
 209 # End:
 210 # vim: set expandtab tabstop=4 shiftwidth=4: