bin/scons-time.py

   1 #!/usr/bin/env python
   2
   3 # MIT License
   4 #
   5 # Copyright The SCons Foundation
   6 #
   7 # Permission is hereby granted, free of charge, to any person obtaining
   8 # a copy of this software and associated documentation files (the
   9 # "Software"), to deal in the Software without restriction, including
  10 # without limitation the rights to use, copy, modify, merge, publish,
  11 # distribute, sublicense, and/or sell copies of the Software, and to
  12 # permit persons to whom the Software is furnished to do so, subject to
  13 # the following conditions:
  14 #
  15 # The above copyright notice and this permission notice shall be included
  16 # in all copies or substantial portions of the Software.
  17 #
  18 # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY
  19 # KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE
  20 # WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  21 # NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
  22 # LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
  23 # OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
  24 # WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
  25
  26 """
  27 scons-time - run SCons timings and collect statistics
  28
  29 A script for running a configuration through SCons with a standard set
  30 of invocations to collect timing and memory statistics and to capture
  31 the results in a consistent set of output files for display and analysis.
  32 """
  33
  34 import getopt
  35 import glob
  36 import os
  37 import re
  38 import shutil
  39 import sys
  40 import tempfile
  41 import time
  42
  43
  44 def HACK_for_exec(cmd, *args):
  45     """
  46     For some reason, Python won't allow an exec() within a function
  47     that also declares an internal function (including lambda functions).
  48     This function is a hack that calls exec() in a function with no
  49     internal functions.
  50     """
  51     if not args:
  52         exec(cmd)
  53     elif len(args) == 1:
  54         exec(cmd, args[0])
  55     else:
  56         exec(cmd, args[0], args[1])
  57
  58
  59 class Plotter:
  60     def increment_size(self, largest):
  61         """
  62         Return the size of each horizontal increment line for a specified
  63         maximum value.  This returns a value that will provide somewhere
  64         between 5 and 9 horizontal lines on the graph, on some set of
  65         boundaries that are multiples of 10/100/1000/etc.
  66         """
  67         i = largest // 5
  68         if not i:
  69             return largest
  70         multiplier = 1
  71         while i >= 10:
  72             i = i // 10
  73             multiplier = multiplier * 10
  74         return i * multiplier
  75
  76     def max_graph_value(self, largest):
  77         # Round up to next integer.
  78         largest = int(largest) + 1
  79         increment = self.increment_size(largest)
  80         return ((largest + increment - 1) // increment) * increment
  81
  82
  83 class Line:
  84     def __init__(self, points, type, title, label, comment, fmt="%s %s"):
  85         self.points = points
  86         self.type = type
  87         self.title = title
  88         self.label = label
  89         self.comment = comment
  90         self.fmt = fmt
  91
  92     def print_label(self, inx, x, y):
  93         if self.label:
  94             print('set label %s "%s" at %0.1f,%0.1f right' % (inx, self.label, x, y))
  95
  96     def plot_string(self):
  97         if self.title:
  98             title_string = 'title "%s"' % self.title
  99         else:
 100             title_string = 'notitle'
 101         return "'-' %s with lines lt %s" % (title_string, self.type)
 102
 103     def print_points(self, fmt=None):
 104         if fmt is None:
 105             fmt = self.fmt
 106         if self.comment:
 107             print('# %s' % self.comment)
 108         for x, y in self.points:
 109             # If y is None, it usually represents some kind of break
 110             # in the line's index number.  We might want to represent
 111             # this some way rather than just drawing the line straight
 112             # between the two points on either side.
 113             if y is not None:
 114                 print(fmt % (x, y))
 115         print('e')
 116
 117     def get_x_values(self):
 118         return [p[0] for p in self.points]
 119
 120     def get_y_values(self):
 121         return [p[1] for p in self.points]
 122
 123
 124 class Gnuplotter(Plotter):
 125
 126     def __init__(self, title, key_location):
 127         self.lines = []
 128         self.title = title
 129         self.key_location = key_location
 130
 131     def line(self, points, type, title=None, label=None, comment=None, fmt='%s %s'):
 132         if points:
 133             line = Line(points, type, title, label, comment, fmt)
 134             self.lines.append(line)
 135
 136     def plot_string(self, line):
 137         return line.plot_string()
 138
 139     def vertical_bar(self, x, type, label, comment):
 140         if self.get_min_x() <= x <= self.get_max_x():
 141             points = [(x, 0), (x, self.max_graph_value(self.get_max_y()))]
 142             self.line(points, type, label, comment)
 143
 144     def get_all_x_values(self):
 145         result = []
 146         for line in self.lines:
 147             result.extend(line.get_x_values())
 148         return [r for r in result if r is not None]
 149
 150     def get_all_y_values(self):
 151         result = []
 152         for line in self.lines:
 153             result.extend(line.get_y_values())
 154         return [r for r in result if r is not None]
 155
 156     def get_min_x(self):
 157         try:
 158             return self.min_x
 159         except AttributeError:
 160             try:
 161                 self.min_x = min(self.get_all_x_values())
 162             except ValueError:
 163                 self.min_x = 0
 164             return self.min_x
 165
 166     def get_max_x(self):
 167         try:
 168             return self.max_x
 169         except AttributeError:
 170             try:
 171                 self.max_x = max(self.get_all_x_values())
 172             except ValueError:
 173                 self.max_x = 0
 174             return self.max_x
 175
 176     def get_min_y(self):
 177         try:
 178             return self.min_y
 179         except AttributeError:
 180             try:
 181                 self.min_y = min(self.get_all_y_values())
 182             except ValueError:
 183                 self.min_y = 0
 184             return self.min_y
 185
 186     def get_max_y(self):
 187         try:
 188             return self.max_y
 189         except AttributeError:
 190             try:
 191                 self.max_y = max(self.get_all_y_values())
 192             except ValueError:
 193                 self.max_y = 0
 194             return self.max_y
 195
 196     def draw(self):
 197
 198         if not self.lines:
 199             return
 200
 201         if self.title:
 202             print('set title "%s"' % self.title)
 203         print('set key %s' % self.key_location)
 204
 205         min_y = self.get_min_y()
 206         max_y = self.max_graph_value(self.get_max_y())
 207         incr = (max_y - min_y) / 10.0
 208         start = min_y + (max_y / 2.0) + (2.0 * incr)
 209         position = [start - (i * incr) for i in range(5)]
 210
 211         inx = 1
 212         for line in self.lines:
 213             line.print_label(inx, line.points[0][0] - 1,
 214                              position[(inx - 1) % len(position)])
 215             inx += 1
 216
 217         plot_strings = [self.plot_string(l) for l in self.lines]
 218         print('plot ' + ', \\\n     '.join(plot_strings))
 219
 220         for line in self.lines:
 221             line.print_points()
 222
 223
 224 def untar(fname):
 225     import tarfile
 226     with tarfile.open(name=fname, mode='r') as tar:
 227         for tarinfo in tar:
 228             try:
 229                 tar.extract(tarinfo, filter="tar")
 230             except TypeError:
 231                 tar.extract(tarinfo)
 232
 233
 234 def unzip(fname):
 235     import zipfile
 236     with zipfile.ZipFile(fname, 'r') as zf:
 237         for name in zf.namelist():
 238             dir = os.path.dirname(name)
 239             try:
 240                 os.makedirs(dir)
 241             except OSError:
 242                 pass
 243             with open(name, 'wb') as f:
 244                 f.write(zf.read(name))
 245
 246
 247 def read_tree(dir):
 248     for dirpath, dirnames, filenames in os.walk(dir):
 249         for fn in filenames:
 250             fn = os.path.join(dirpath, fn)
 251             if os.path.isfile(fn):
 252                 with open(fn, 'rb') as f:
 253                     f.read()
 254
 255
 256 def redirect_to_file(command, log):
 257     return '%s > %s 2>&1' % (command, log)
 258
 259
 260 def tee_to_file(command, log):
 261     return '%s 2>&1 | tee %s' % (command, log)
 262
 263 def makedict(**kw):
 264     return kw
 265
 266 class SConsTimer:
 267     """
 268     Usage: scons-time SUBCOMMAND [ARGUMENTS]
 269     Type "scons-time help SUBCOMMAND" for help on a specific subcommand.
 270
 271     Available subcommands:
 272         func            Extract test-run data for a function
 273         help            Provides help
 274         mem             Extract --debug=memory data from test runs
 275         obj             Extract --debug=count data from test runs
 276         time            Extract --debug=time data from test runs
 277         run             Runs a test configuration
 278     """
 279
 280     name = 'scons-time'
 281     name_spaces = ' ' * len(name)
 282
 283     default_settings = makedict(
 284         chdir=None,
 285         config_file=None,
 286         initial_commands=[],
 287         key_location='bottom left',
 288         orig_cwd=os.getcwd(),
 289         outdir=None,
 290         prefix='',
 291         python='"%s"' % sys.executable,
 292         redirect=redirect_to_file,
 293         scons=None,
 294         scons_flags='--debug=count --debug=memory --debug=time --debug=memoizer',
 295         scons_lib_dir=None,
 296         scons_wrapper=None,
 297         startup_targets='--help',
 298         subdir=None,
 299         subversion_url=None,
 300         svn='svn',
 301         svn_co_flag='-q',
 302         tar='tar',
 303         targets='',
 304         targets0=None,
 305         targets1=None,
 306         targets2=None,
 307         title=None,
 308         unzip='unzip',
 309         verbose=False,
 310         vertical_bars=[],
 311
 312         unpack_map={
 313             '.tar.gz': (untar, '%(tar)s xzf %%s'),
 314             '.tgz': (untar, '%(tar)s xzf %%s'),
 315             '.tar': (untar, '%(tar)s xf %%s'),
 316             '.zip': (unzip, '%(unzip)s %%s'),
 317         },
 318     )
 319
 320     run_titles = [
 321         'Startup',
 322         'Full build',
 323         'Up-to-date build',
 324     ]
 325
 326     run_commands = [
 327         '%(python)s %(scons_wrapper)s %(scons_flags)s --profile=%(prof0)s %(targets0)s',
 328         '%(python)s %(scons_wrapper)s %(scons_flags)s --profile=%(prof1)s %(targets1)s',
 329         '%(python)s %(scons_wrapper)s %(scons_flags)s --profile=%(prof2)s %(targets2)s',
 330     ]
 331
 332     stages = [
 333         'pre-read',
 334         'post-read',
 335         'pre-build',
 336         'post-build',
 337     ]
 338
 339     stage_strings = {
 340         'pre-read': 'Memory before reading SConscript files:',
 341         'post-read': 'Memory after reading SConscript files:',
 342         'pre-build': 'Memory before building targets:',
 343         'post-build': 'Memory after building targets:',
 344     }
 345
 346     memory_string_all = 'Memory '
 347
 348     default_stage = stages[-1]
 349
 350     time_strings = {
 351         'total': 'Total build time',
 352         'SConscripts': 'Total SConscript file execution time',
 353         'SCons': 'Total SCons execution time',
 354         'commands': 'Total command execution time',
 355     }
 356
 357     time_string_all = 'Total .* time'
 358
 359     #
 360
 361     def __init__(self):
 362         self.__dict__.update(self.default_settings)
 363
 364     # Functions for displaying and executing commands.
 365
 366     def subst(self, x, dictionary):
 367         try:
 368             return x % dictionary
 369         except TypeError:
 370             # x isn't a string (it's probably a Python function),
 371             # so just return it.
 372             return x
 373
 374     def subst_variables(self, command, dictionary):
 375         """
 376         Substitutes (via the format operator) the values in the specified
 377         dictionary into the specified command.
 378
 379         The command can be an (action, string) tuple.  In all cases, we
 380         perform substitution on strings and don't worry if something isn't
 381         a string.  (It's probably a Python function to be executed.)
 382         """
 383         try:
 384             command + ''
 385         except TypeError:
 386             action = command[0]
 387             string = command[1]
 388             args = command[2:]
 389         else:
 390             action = command
 391             string = action
 392             args = (())
 393         action = self.subst(action, dictionary)
 394         string = self.subst(string, dictionary)
 395         return (action, string, args)
 396
 397     def _do_not_display(self, msg, *args):
 398         pass
 399
 400     def display(self, msg, *args):
 401         """
 402         Displays the specified message.
 403
 404         Each message is prepended with a standard prefix of our name
 405         plus the time.
 406         """
 407         if callable(msg):
 408             msg = msg(*args)
 409         else:
 410             msg = msg % args
 411         if msg is None:
 412             return
 413         fmt = '%s[%s]: %s\n'
 414         sys.stdout.write(fmt % (self.name, time.strftime('%H:%M:%S'), msg))
 415
 416     def _do_not_execute(self, action, *args):
 417         pass
 418
 419     def execute(self, action, *args):
 420         """
 421         Executes the specified action.
 422
 423         The action is called if it's a callable Python function, and
 424         otherwise passed to os.system().
 425         """
 426         if callable(action):
 427             action(*args)
 428         else:
 429             os.system(action % args)
 430
 431     def run_command_list(self, commands, dict):
 432         """
 433         Executes a list of commands, substituting values from the
 434         specified dictionary.
 435         """
 436         commands = [self.subst_variables(c, dict) for c in commands]
 437         for action, string, args in commands:
 438             self.display(string, *args)
 439             sys.stdout.flush()
 440             status = self.execute(action, *args)
 441             if status:
 442                 sys.exit(status)
 443
 444     def log_display(self, command, log):
 445         command = self.subst(command, self.__dict__)
 446         if log:
 447             command = self.redirect(command, log)
 448         return command
 449
 450     def log_execute(self, command, log):
 451         command = self.subst(command, self.__dict__)
 452         p = os.popen(command)
 453         output = p.read()
 454         p.close()
 455         # TODO: convert to subrocess, os.popen is obsolete. This didn't work:
 456         # process = subprocess.Popen(command, stdout=subprocess.PIPE, shell=True)
 457         # output = process.stdout.read()
 458         # process.stdout.close()
 459         # process.wait()
 460         if self.verbose:
 461             sys.stdout.write(output)
 462         # TODO: Figure out
 463         # Not sure we need to write binary here
 464         with open(log, 'w') as f:
 465             f.write(str(output))
 466
 467     def archive_splitext(self, path):
 468         """
 469         Splits an archive name into a filename base and extension.
 470
 471         This is like os.path.splitext() (which it calls) except that it
 472         also looks for '.tar.gz' and treats it as an atomic extensions.
 473         """
 474         if path.endswith('.tar.gz'):
 475             return path[:-7], path[-7:]
 476         else:
 477             return os.path.splitext(path)
 478
 479     def args_to_files(self, args, tail=None):
 480         """
 481         Takes a list of arguments, expands any glob patterns, and
 482         returns the last "tail" files from the list.
 483         """
 484         files = []
 485         for a in args:
 486             files.extend(sorted(glob.glob(a)))
 487
 488         if tail:
 489             files = files[-tail:]
 490
 491         return files
 492
 493     def ascii_table(self, files, columns,
 494                     line_function, file_function=lambda x: x,
 495                     *args, **kw):
 496
 497         header_fmt = ' '.join(['%12s'] * len(columns))
 498         line_fmt = header_fmt + '    %s'
 499
 500         print(header_fmt % columns)
 501
 502         for file in files:
 503             t = line_function(file, *args, **kw)
 504             if t is None:
 505                 t = []
 506             diff = len(columns) - len(t)
 507             if diff > 0:
 508                 t += [''] * diff
 509             t.append(file_function(file))
 510             print(line_fmt % tuple(t))
 511
 512     def collect_results(self, files, function, *args, **kw):
 513         results = {}
 514
 515         for file in files:
 516             base = os.path.splitext(file)[0]
 517             run, index = base.split('-')[-2:]
 518
 519             run = int(run)
 520             index = int(index)
 521
 522             value = function(file, *args, **kw)
 523
 524             try:
 525                 r = results[index]
 526             except KeyError:
 527                 r = []
 528                 results[index] = r
 529             r.append((run, value))
 530
 531         return results
 532
 533     def doc_to_help(self, obj):
 534         """
 535         Translates an object's __doc__ string into help text.
 536
 537         This strips a consistent number of spaces from each line in the
 538         help text, essentially "outdenting" the text to the left-most
 539         column.
 540         """
 541         doc = obj.__doc__
 542         if doc is None:
 543             return ''
 544         return self.outdent(doc)
 545
 546     def find_next_run_number(self, dir, prefix):
 547         """
 548         Returns the next run number in a directory for the specified prefix.
 549
 550         Examines the contents the specified directory for files with the
 551         specified prefix, extracts the run numbers from each file name,
 552         and returns the next run number after the largest it finds.
 553         """
 554         x = re.compile(re.escape(prefix) + r'-([0-9]+).*')
 555         matches = [x.match(e) for e in os.listdir(dir)]
 556         matches = [_f for _f in matches if _f]
 557         if not matches:
 558             return 0
 559         run_numbers = [int(m.group(1)) for m in matches]
 560         return int(max(run_numbers)) + 1
 561
 562     def gnuplot_results(self, results, fmt='%s %.3f'):
 563         """
 564         Prints out a set of results in Gnuplot format.
 565         """
 566         gp = Gnuplotter(self.title, self.key_location)
 567
 568         for i in sorted(results.keys()):
 569             try:
 570                 t = self.run_titles[i]
 571             except IndexError:
 572                 t = '??? %s ???' % i
 573             results[i].sort()
 574             gp.line(results[i], i + 1, t, None, t, fmt=fmt)
 575
 576         for bar_tuple in self.vertical_bars:
 577             try:
 578                 x, type, label, comment = bar_tuple
 579             except ValueError:
 580                 x, type, label = bar_tuple
 581                 comment = label
 582             gp.vertical_bar(x, type, label, comment)
 583
 584         gp.draw()
 585
 586     def logfile_name(self, invocation):
 587         """
 588         Returns the absolute path of a log file for the specificed
 589         invocation number.
 590         """
 591         name = self.prefix_run + '-%d.log' % invocation
 592         return os.path.join(self.outdir, name)
 593
 594     def outdent(self, s):
 595         """
 596         Strip as many spaces from each line as are found at the beginning
 597         of the first line in the list.
 598         """
 599         lines = s.split('\n')
 600         if lines[0] == '':
 601             lines = lines[1:]
 602         spaces = re.match(' *', lines[0]).group(0)
 603
 604         def strip_initial_spaces(line, s=spaces):
 605             if line.startswith(spaces):
 606                 line = line[len(spaces):]
 607             return line
 608
 609         return '\n'.join([strip_initial_spaces(l) for l in lines]) + '\n'
 610
 611     def profile_name(self, invocation):
 612         """
 613         Returns the absolute path of a profile file for the specified
 614         invocation number.
 615         """
 616         name = self.prefix_run + '-%d.prof' % invocation
 617         return os.path.join(self.outdir, name)
 618
 619     def set_env(self, key, value):
 620         os.environ[key] = value
 621
 622     #
 623
 624     def get_debug_times(self, file, time_string=None):
 625         """
 626         Fetch times from the --debug=time strings in the specified file.
 627         """
 628         if time_string is None:
 629             search_string = self.time_string_all
 630         else:
 631             search_string = time_string
 632         with open(file) as f:
 633             contents = f.read()
 634         if not contents:
 635             sys.stderr.write('file %s has no contents!\n' % repr(file))
 636             return None
 637         result = re.findall(r'%s: ([\d.]*)' % search_string, contents)[-4:]
 638         result = [float(r) for r in result]
 639         if time_string is not None:
 640             try:
 641                 result = result[0]
 642             except IndexError:
 643                 sys.stderr.write('file %s has no results!\n' % repr(file))
 644                 return None
 645         return result
 646
 647     def get_function_profile(self, file, function):
 648         """
 649         Returns the file, line number, function name, and cumulative time.
 650         """
 651         try:
 652             import pstats
 653         except ImportError as e:
 654             sys.stderr.write('%s: func: %s\n' % (self.name, e))
 655             sys.stderr.write('%s  This version of Python is missing the profiler.\n' % self.name_spaces)
 656             sys.stderr.write('%s  Cannot use the "func" subcommand.\n' % self.name_spaces)
 657             sys.exit(1)
 658         statistics = pstats.Stats(file).stats
 659         matches = [e for e in statistics.items() if e[0][2] == function]
 660         r = matches[0]
 661         return r[0][0], r[0][1], r[0][2], r[1][3]
 662
 663     def get_function_time(self, file, function):
 664         """
 665         Returns just the cumulative time for the specified function.
 666         """
 667         return self.get_function_profile(file, function)[3]
 668
 669     def get_memory(self, file, memory_string=None):
 670         """
 671         Returns a list of integers of the amount of memory used.  The
 672         default behavior is to return all the stages.
 673         """
 674         if memory_string is None:
 675             search_string = self.memory_string_all
 676         else:
 677             search_string = memory_string
 678         with open(file) as f:
 679             lines = f.readlines()
 680         lines = [l for l in lines if l.startswith(search_string)][-4:]
 681         result = [int(l.split()[-1]) for l in lines[-4:]]
 682         if len(result) == 1:
 683             result = result[0]
 684         return result
 685
 686     def get_object_counts(self, file, object_name, index=None):
 687         """
 688         Returns the counts of the specified object_name.
 689         """
 690         object_string = ' ' + object_name + '\n'
 691         with open(file) as f:
 692             lines = f.readlines()
 693         line = [l for l in lines if l.endswith(object_string)][0]
 694         result = [int(field) for field in line.split()[:4]]
 695         if index is not None:
 696             result = result[index]
 697         return result
 698
 699     command_alias = {}
 700
 701     def execute_subcommand(self, argv):
 702         """
 703         Executes the do_*() function for the specified subcommand (argv[0]).
 704         """
 705         if not argv:
 706             return
 707         cmdName = self.command_alias.get(argv[0], argv[0])
 708         try:
 709             func = getattr(self, 'do_' + cmdName)
 710         except AttributeError:
 711             return self.default(argv)
 712         try:
 713             return func(argv)
 714         except TypeError as e:
 715             sys.stderr.write("%s %s: %s\n" % (self.name, cmdName, e))
 716             import traceback
 717             traceback.print_exc(file=sys.stderr)
 718             sys.stderr.write("Try '%s help %s'\n" % (self.name, cmdName))
 719
 720     def default(self, argv):
 721         """
 722         The default behavior for an unknown subcommand.  Prints an
 723         error message and exits.
 724         """
 725         sys.stderr.write('%s: Unknown subcommand "%s".\n' % (self.name, argv[0]))
 726         sys.stderr.write('Type "%s help" for usage.\n' % self.name)
 727         sys.exit(1)
 728
 729     #
 730
 731     def do_help(self, argv):
 732         """
 733         """
 734         if argv[1:]:
 735             for arg in argv[1:]:
 736                 try:
 737                     func = getattr(self, 'do_' + arg)
 738                 except AttributeError:
 739                     sys.stderr.write('%s: No help for "%s"\n' % (self.name, arg))
 740                 else:
 741                     try:
 742                         help = getattr(self, 'help_' + arg)
 743                     except AttributeError:
 744                         sys.stdout.write(self.doc_to_help(func))
 745                         sys.stdout.flush()
 746                     else:
 747                         help()
 748         else:
 749             doc = self.doc_to_help(self.__class__)
 750             if doc:
 751                 sys.stdout.write(doc)
 752             sys.stdout.flush()
 753             return None
 754
 755     #
 756
 757     def help_func(self):
 758         help = """\
 759         Usage: scons-time func [OPTIONS] FILE [...]
 760
 761           -C DIR, --chdir=DIR           Change to DIR before looking for files
 762           -f FILE, --file=FILE          Read configuration from specified FILE
 763           --fmt=FORMAT, --format=FORMAT Print data in specified FORMAT
 764           --func=NAME, --function=NAME  Report time for function NAME
 765           -h, --help                    Print this help and exit
 766           -p STRING, --prefix=STRING    Use STRING as log file/profile prefix
 767           -t NUMBER, --tail=NUMBER      Only report the last NUMBER files
 768           --title=TITLE                 Specify the output plot TITLE
 769         """
 770         sys.stdout.write(self.outdent(help))
 771         sys.stdout.flush()
 772
 773     def do_func(self, argv):
 774         """
 775         """
 776         format = 'ascii'
 777         function_name = '_main'
 778         tail = None
 779
 780         short_opts = '?C:f:hp:t:'
 781
 782         long_opts = [
 783             'chdir=',
 784             'file=',
 785             'fmt=',
 786             'format=',
 787             'func=',
 788             'function=',
 789             'help',
 790             'prefix=',
 791             'tail=',
 792             'title=',
 793         ]
 794
 795         opts, args = getopt.getopt(argv[1:], short_opts, long_opts)
 796
 797         for o, a in opts:
 798             if o in ('-C', '--chdir'):
 799                 self.chdir = a
 800             elif o in ('-f', '--file'):
 801                 self.config_file = a
 802             elif o in ('--fmt', '--format'):
 803                 format = a
 804             elif o in ('--func', '--function'):
 805                 function_name = a
 806             elif o in ('-?', '-h', '--help'):
 807                 self.do_help(['help', 'func'])
 808                 sys.exit(0)
 809             elif o in ('--max',):
 810                 max_time = int(a)
 811             elif o in ('-p', '--prefix'):
 812                 self.prefix = a
 813             elif o in ('-t', '--tail'):
 814                 tail = int(a)
 815             elif o in ('--title',):
 816                 self.title = a
 817
 818         if self.config_file:
 819             with open(self.config_file, 'r') as f:
 820                 config = f.read()
 821             exec(config, self.__dict__)
 822
 823         if self.chdir:
 824             os.chdir(self.chdir)
 825
 826         if not args:
 827
 828             pattern = '%s*.prof' % self.prefix
 829             args = self.args_to_files([pattern], tail)
 830
 831             if not args:
 832                 if self.chdir:
 833                     directory = self.chdir
 834                 else:
 835                     directory = os.getcwd()
 836
 837                 sys.stderr.write('%s: func: No arguments specified.\n' % self.name)
 838                 sys.stderr.write('%s  No %s*.prof files found in "%s".\n' % (self.name_spaces, self.prefix, directory))
 839                 sys.stderr.write('%s  Type "%s help func" for help.\n' % (self.name_spaces, self.name))
 840                 sys.exit(1)
 841
 842         else:
 843
 844             args = self.args_to_files(args, tail)
 845
 846         cwd_ = os.getcwd() + os.sep
 847
 848         if format == 'ascii':
 849
 850             for file in args:
 851                 try:
 852                     f, line, func, time = \
 853                         self.get_function_profile(file, function_name)
 854                 except ValueError as e:
 855                     sys.stderr.write("%s: func: %s: %s\n" %
 856                                      (self.name, file, e))
 857                 else:
 858                     if f.startswith(cwd_):
 859                         f = f[len(cwd_):]
 860                     print("%.3f %s:%d(%s)" % (time, f, line, func))
 861
 862         elif format == 'gnuplot':
 863
 864             results = self.collect_results(args, self.get_function_time,
 865                                            function_name)
 866
 867             self.gnuplot_results(results)
 868
 869         else:
 870
 871             sys.stderr.write('%s: func: Unknown format "%s".\n' % (self.name, format))
 872             sys.exit(1)
 873
 874     #
 875
 876     def help_mem(self):
 877         help = """\
 878         Usage: scons-time mem [OPTIONS] FILE [...]
 879
 880           -C DIR, --chdir=DIR           Change to DIR before looking for files
 881           -f FILE, --file=FILE          Read configuration from specified FILE
 882           --fmt=FORMAT, --format=FORMAT Print data in specified FORMAT
 883           -h, --help                    Print this help and exit
 884           -p STRING, --prefix=STRING    Use STRING as log file/profile prefix
 885           --stage=STAGE                 Plot memory at the specified stage:
 886                                           pre-read, post-read, pre-build,
 887                                           post-build (default: post-build)
 888           -t NUMBER, --tail=NUMBER      Only report the last NUMBER files
 889           --title=TITLE                 Specify the output plot TITLE
 890         """
 891         sys.stdout.write(self.outdent(help))
 892         sys.stdout.flush()
 893
 894     def do_mem(self, argv):
 895
 896         format = 'ascii'
 897         def _logfile_path(x):
 898             return x
 899
 900         logfile_path = _logfile_path
 901
 902         stage = self.default_stage
 903         tail = None
 904
 905         short_opts = '?C:f:hp:t:'
 906
 907         long_opts = [
 908             'chdir=',
 909             'file=',
 910             'fmt=',
 911             'format=',
 912             'help',
 913             'prefix=',
 914             'stage=',
 915             'tail=',
 916             'title=',
 917         ]
 918
 919         opts, args = getopt.getopt(argv[1:], short_opts, long_opts)
 920
 921         for o, a in opts:
 922             if o in ('-C', '--chdir'):
 923                 self.chdir = a
 924             elif o in ('-f', '--file'):
 925                 self.config_file = a
 926             elif o in ('--fmt', '--format'):
 927                 format = a
 928             elif o in ('-?', '-h', '--help'):
 929                 self.do_help(['help', 'mem'])
 930                 sys.exit(0)
 931             elif o in ('-p', '--prefix'):
 932                 self.prefix = a
 933             elif o in ('--stage',):
 934                 if a not in self.stages:
 935                     sys.stderr.write('%s: mem: Unrecognized stage "%s".\n' % (self.name, a))
 936                     sys.exit(1)
 937                 stage = a
 938             elif o in ('-t', '--tail'):
 939                 tail = int(a)
 940             elif o in ('--title',):
 941                 self.title = a
 942
 943         if self.config_file:
 944             with open(self.config_file, 'r') as f:
 945                 config = f.read()
 946             HACK_for_exec(config, self.__dict__)
 947
 948         if self.chdir:
 949             os.chdir(self.chdir)
 950             def _logfile_path_join(x):
 951                 return os.path.join(self.chdir, x)
 952
 953             logfile_path = _logfile_path_join
 954
 955         if not args:
 956             pattern = '%s*.log' % self.prefix
 957             args = self.args_to_files([pattern], tail)
 958
 959             if not args:
 960                 if self.chdir:
 961                     directory = self.chdir
 962                 else:
 963                     directory = os.getcwd()
 964
 965                 sys.stderr.write('%s: mem: No arguments specified.\n' % self.name)
 966                 sys.stderr.write('%s  No %s*.log files found in "%s".\n' % (self.name_spaces, self.prefix, directory))
 967                 sys.stderr.write('%s  Type "%s help mem" for help.\n' % (self.name_spaces, self.name))
 968                 sys.exit(1)
 969
 970         else:
 971
 972             args = self.args_to_files(args, tail)
 973
 974         # cwd_ = os.getcwd() + os.sep
 975
 976         if format == 'ascii':
 977
 978             self.ascii_table(args, tuple(self.stages), self.get_memory, logfile_path)
 979
 980         elif format == 'gnuplot':
 981
 982             results = self.collect_results(args, self.get_memory,
 983                                            self.stage_strings[stage])
 984
 985             self.gnuplot_results(results)
 986
 987         else:
 988
 989             sys.stderr.write('%s: mem: Unknown format "%s".\n' % (self.name, format))
 990             sys.exit(1)
 991
 992         return 0
 993
 994     #
 995
 996     def help_obj(self):
 997         help = """\
 998         Usage: scons-time obj [OPTIONS] OBJECT FILE [...]
 999
1000           -C DIR, --chdir=DIR           Change to DIR before looking for files
1001           -f FILE, --file=FILE          Read configuration from specified FILE
1002           --fmt=FORMAT, --format=FORMAT Print data in specified FORMAT
1003           -h, --help                    Print this help and exit
1004           -p STRING, --prefix=STRING    Use STRING as log file/profile prefix
1005           --stage=STAGE                 Plot memory at the specified stage:
1006                                           pre-read, post-read, pre-build,
1007                                           post-build (default: post-build)
1008           -t NUMBER, --tail=NUMBER      Only report the last NUMBER files
1009           --title=TITLE                 Specify the output plot TITLE
1010         """
1011         sys.stdout.write(self.outdent(help))
1012         sys.stdout.flush()
1013
1014     def do_obj(self, argv):
1015
1016         format = 'ascii'
1017
1018         def _logfile_path(x):
1019             return x
1020
1021         logfile_path = _logfile_path
1022
1023         stage = self.default_stage
1024         tail = None
1025
1026         short_opts = '?C:f:hp:t:'
1027
1028         long_opts = [
1029             'chdir=',
1030             'file=',
1031             'fmt=',
1032             'format=',
1033             'help',
1034             'prefix=',
1035             'stage=',
1036             'tail=',
1037             'title=',
1038         ]
1039
1040         opts, args = getopt.getopt(argv[1:], short_opts, long_opts)
1041
1042         for o, a in opts:
1043             if o in ('-C', '--chdir'):
1044                 self.chdir = a
1045             elif o in ('-f', '--file'):
1046                 self.config_file = a
1047             elif o in ('--fmt', '--format'):
1048                 format = a
1049             elif o in ('-?', '-h', '--help'):
1050                 self.do_help(['help', 'obj'])
1051                 sys.exit(0)
1052             elif o in ('-p', '--prefix'):
1053                 self.prefix = a
1054             elif o in ('--stage',):
1055                 if a not in self.stages:
1056                     sys.stderr.write('%s: obj: Unrecognized stage "%s".\n' % (self.name, a))
1057                     sys.stderr.write('%s       Type "%s help obj" for help.\n' % (self.name_spaces, self.name))
1058                     sys.exit(1)
1059                 stage = a
1060             elif o in ('-t', '--tail'):
1061                 tail = int(a)
1062             elif o in ('--title',):
1063                 self.title = a
1064
1065         if not args:
1066             sys.stderr.write('%s: obj: Must specify an object name.\n' % self.name)
1067             sys.stderr.write('%s       Type "%s help obj" for help.\n' % (self.name_spaces, self.name))
1068             sys.exit(1)
1069
1070         object_name = args.pop(0)
1071
1072         if self.config_file:
1073             with open(self.config_file, 'r') as f:
1074                 config = f.read()
1075             HACK_for_exec(config, self.__dict__)
1076
1077         if self.chdir:
1078             os.chdir(self.chdir)
1079
1080             def _logfile_path_join(x):
1081                 return os.path.join(self.chdir, x)
1082
1083             logfile_path = _logfile_path_join
1084
1085         if not args:
1086             pattern = '%s*.log' % self.prefix
1087             args = self.args_to_files([pattern], tail)
1088
1089             if not args:
1090                 if self.chdir:
1091                     directory = self.chdir
1092                 else:
1093                     directory = os.getcwd()
1094
1095                 sys.stderr.write('%s: obj: No arguments specified.\n' % self.name)
1096                 sys.stderr.write('%s  No %s*.log files found in "%s".\n' % (self.name_spaces, self.prefix, directory))
1097                 sys.stderr.write('%s  Type "%s help obj" for help.\n' % (self.name_spaces, self.name))
1098                 sys.exit(1)
1099
1100         else:
1101
1102             args = self.args_to_files(args, tail)
1103
1104         cwd_ = os.getcwd() + os.sep
1105
1106         if format == 'ascii':
1107
1108             self.ascii_table(args, tuple(self.stages), self.get_object_counts, logfile_path, object_name)
1109
1110         elif format == 'gnuplot':
1111
1112             stage_index = 0
1113             for s in self.stages:
1114                 if stage == s:
1115                     break
1116                 stage_index = stage_index + 1
1117
1118             results = self.collect_results(args, self.get_object_counts,
1119                                            object_name, stage_index)
1120
1121             self.gnuplot_results(results)
1122
1123         else:
1124
1125             sys.stderr.write('%s: obj: Unknown format "%s".\n' % (self.name, format))
1126             sys.exit(1)
1127
1128         return 0
1129
1130     #
1131
1132     def help_run(self):
1133         help = """\
1134         Usage: scons-time run [OPTIONS] [FILE ...]
1135
1136           --chdir=DIR                   Name of unpacked directory for chdir
1137           -f FILE, --file=FILE          Read configuration from specified FILE
1138           -h, --help                    Print this help and exit
1139           -n, --no-exec                 No execute, just print command lines
1140           --number=NUMBER               Put output in files for run NUMBER
1141           --outdir=OUTDIR               Put output files in OUTDIR
1142           -p STRING, --prefix=STRING    Use STRING as log file/profile prefix
1143           --python=PYTHON               Time using the specified PYTHON
1144           -q, --quiet                   Don't print command lines
1145           --scons=SCONS                 Time using the specified SCONS
1146           --svn=URL, --subversion=URL   Use SCons from Subversion URL
1147           -v, --verbose                 Display output of commands
1148         """
1149         sys.stdout.write(self.outdent(help))
1150         sys.stdout.flush()
1151
1152     def do_run(self, argv):
1153         """
1154         """
1155         run_number_list = [None]
1156
1157         short_opts = '?f:hnp:qs:v'
1158
1159         long_opts = [
1160             'file=',
1161             'help',
1162             'no-exec',
1163             'number=',
1164             'outdir=',
1165             'prefix=',
1166             'python=',
1167             'quiet',
1168             'scons=',
1169             'svn=',
1170             'subdir=',
1171             'subversion=',
1172             'verbose',
1173         ]
1174
1175         opts, args = getopt.getopt(argv[1:], short_opts, long_opts)
1176
1177         for o, a in opts:
1178             if o in ('-f', '--file'):
1179                 self.config_file = a
1180             elif o in ('-?', '-h', '--help'):
1181                 self.do_help(['help', 'run'])
1182                 sys.exit(0)
1183             elif o in ('-n', '--no-exec'):
1184                 self.execute = self._do_not_execute
1185             elif o in ('--number',):
1186                 run_number_list = self.split_run_numbers(a)
1187             elif o in ('--outdir',):
1188                 self.outdir = a
1189             elif o in ('-p', '--prefix'):
1190                 self.prefix = a
1191             elif o in ('--python',):
1192                 self.python = a
1193             elif o in ('-q', '--quiet'):
1194                 self.display = self._do_not_display
1195             elif o in ('-s', '--subdir'):
1196                 self.subdir = a
1197             elif o in ('--scons',):
1198                 self.scons = a
1199             elif o in ('--svn', '--subversion'):
1200                 self.subversion_url = a
1201             elif o in ('-v', '--verbose'):
1202                 self.redirect = tee_to_file
1203                 self.verbose = True
1204                 self.svn_co_flag = ''
1205
1206         if not args and not self.config_file:
1207             sys.stderr.write('%s: run: No arguments or -f config file specified.\n' % self.name)
1208             sys.stderr.write('%s  Type "%s help run" for help.\n' % (self.name_spaces, self.name))
1209             sys.exit(1)
1210
1211         if self.config_file:
1212             with open(self.config_file, 'r') as f:
1213                 config = f.read()
1214             exec(config, self.__dict__)
1215
1216         if args:
1217             self.archive_list = args
1218
1219         archive_file_name = os.path.split(self.archive_list[0])[1]
1220
1221         if not self.subdir:
1222             self.subdir = self.archive_splitext(archive_file_name)[0]
1223
1224         if not self.prefix:
1225             self.prefix = self.archive_splitext(archive_file_name)[0]
1226
1227         prepare = None
1228         if self.subversion_url:
1229             prepare = self.prep_subversion_run
1230
1231         for run_number in run_number_list:
1232             self.individual_run(run_number, self.archive_list, prepare)
1233
1234     def split_run_numbers(self, s):
1235         result = []
1236         for n in s.split(','):
1237             try:
1238                 x, y = n.split('-')
1239             except ValueError:
1240                 result.append(int(n))
1241             else:
1242                 result.extend(list(range(int(x), int(y) + 1)))
1243         return result
1244
1245     def scons_path(self, dir):
1246         return os.path.join(dir, 'scripts', 'scons.py')
1247
1248     def scons_lib_dir_path(self, dir):
1249         """build the path to the engine.
1250
1251         this used to join src/engine, but no longer.
1252         """
1253         return dir
1254
1255     def prep_subversion_run(self, commands, removals):
1256         self.svn_tmpdir = tempfile.mkdtemp(prefix=self.name + '-svn-')
1257         removals.append((shutil.rmtree, 'rm -rf %%s', self.svn_tmpdir))
1258
1259         self.scons = self.scons_path(self.svn_tmpdir)
1260         self.scons_lib_dir = self.scons_lib_dir_path(self.svn_tmpdir)
1261
1262         commands.extend([
1263             '%(svn)s co %(svn_co_flag)s -r %(run_number)s %(subversion_url)s %(svn_tmpdir)s',
1264         ])
1265
1266     def individual_run(self, run_number, archive_list, prepare=None):
1267         """
1268         Performs an individual run of the default SCons invocations.
1269         """
1270
1271         commands = []
1272         removals = []
1273
1274         if prepare:
1275             prepare(commands, removals)
1276
1277         save_scons = self.scons
1278         save_scons_wrapper = self.scons_wrapper
1279         save_scons_lib_dir = self.scons_lib_dir
1280
1281         if self.outdir is None:
1282             self.outdir = self.orig_cwd
1283         elif not os.path.isabs(self.outdir):
1284             self.outdir = os.path.join(self.orig_cwd, self.outdir)
1285
1286         if self.scons is None:
1287             self.scons = self.scons_path(self.orig_cwd)
1288
1289         if self.scons_lib_dir is None:
1290             self.scons_lib_dir = self.scons_lib_dir_path(self.orig_cwd)
1291
1292         if self.scons_wrapper is None:
1293             self.scons_wrapper = self.scons
1294
1295         if not run_number:
1296             run_number = self.find_next_run_number(self.outdir, self.prefix)
1297
1298         self.run_number = str(run_number)
1299
1300         self.prefix_run = self.prefix + '-%03d' % run_number
1301
1302         if self.targets0 is None:
1303             self.targets0 = self.startup_targets
1304         if self.targets1 is None:
1305             self.targets1 = self.targets
1306         if self.targets2 is None:
1307             self.targets2 = self.targets
1308
1309         self.tmpdir = tempfile.mkdtemp(prefix=self.name + '-')
1310
1311         commands.extend([
1312             (os.chdir, 'cd %%s', self.tmpdir),
1313         ])
1314
1315         for archive in archive_list:
1316             if not os.path.isabs(archive):
1317                 archive = os.path.join(self.orig_cwd, archive)
1318             if os.path.isdir(archive):
1319                 dest = os.path.split(archive)[1]
1320                 commands.append((shutil.copytree, 'cp -r %%s %%s', archive, dest))
1321             else:
1322                 suffix = self.archive_splitext(archive)[1]
1323                 unpack_command = self.unpack_map.get(suffix)
1324                 if not unpack_command:
1325                     dest = os.path.split(archive)[1]
1326                     commands.append((shutil.copyfile, 'cp %%s %%s', archive, dest))
1327                 else:
1328                     commands.append(unpack_command + (archive,))
1329
1330         commands.extend([
1331             (os.chdir, 'cd %%s', self.subdir),
1332         ])
1333
1334         commands.extend(self.initial_commands)
1335
1336         commands.extend([
1337             (lambda: read_tree('.'),
1338              'find * -type f | xargs cat > /dev/null'),
1339
1340             (self.set_env, 'export %%s=%%s',
1341              'SCONS_LIB_DIR', self.scons_lib_dir),
1342
1343             '%(python)s %(scons_wrapper)s --version',
1344         ])
1345
1346         index = 0
1347         for run_command in self.run_commands:
1348             setattr(self, 'prof%d' % index, self.profile_name(index))
1349             c = (
1350                 self.log_execute,
1351                 self.log_display,
1352                 run_command,
1353                 self.logfile_name(index),
1354             )
1355             commands.append(c)
1356             index = index + 1
1357
1358         commands.extend([
1359             (os.chdir, 'cd %%s', self.orig_cwd),
1360         ])
1361
1362         if not os.environ.get('PRESERVE'):
1363             commands.extend(removals)
1364             commands.append((shutil.rmtree, 'rm -rf %%s', self.tmpdir))
1365
1366         self.run_command_list(commands, self.__dict__)
1367
1368         self.scons = save_scons
1369         self.scons_lib_dir = save_scons_lib_dir
1370         self.scons_wrapper = save_scons_wrapper
1371
1372     #
1373
1374     def help_time(self):
1375         help = """\
1376         Usage: scons-time time [OPTIONS] FILE [...]
1377
1378           -C DIR, --chdir=DIR           Change to DIR before looking for files
1379           -f FILE, --file=FILE          Read configuration from specified FILE
1380           --fmt=FORMAT, --format=FORMAT Print data in specified FORMAT
1381           -h, --help                    Print this help and exit
1382           -p STRING, --prefix=STRING    Use STRING as log file/profile prefix
1383           -t NUMBER, --tail=NUMBER      Only report the last NUMBER files
1384           --which=TIMER                 Plot timings for TIMER:  total,
1385                                           SConscripts, SCons, commands.
1386         """
1387         sys.stdout.write(self.outdent(help))
1388         sys.stdout.flush()
1389
1390     def do_time(self, argv):
1391
1392         format = 'ascii'
1393
1394         def _logfile_path(x):
1395             return x
1396
1397         logfile_path = _logfile_path
1398
1399         tail = None
1400         which = 'total'
1401
1402         short_opts = '?C:f:hp:t:'
1403
1404         long_opts = [
1405             'chdir=',
1406             'file=',
1407             'fmt=',
1408             'format=',
1409             'help',
1410             'prefix=',
1411             'tail=',
1412             'title=',
1413             'which=',
1414         ]
1415
1416         opts, args = getopt.getopt(argv[1:], short_opts, long_opts)
1417
1418         for o, a in opts:
1419             if o in ('-C', '--chdir'):
1420                 self.chdir = a
1421             elif o in ('-f', '--file'):
1422                 self.config_file = a
1423             elif o in ('--fmt', '--format'):
1424                 format = a
1425             elif o in ('-?', '-h', '--help'):
1426                 self.do_help(['help', 'time'])
1427                 sys.exit(0)
1428             elif o in ('-p', '--prefix'):
1429                 self.prefix = a
1430             elif o in ('-t', '--tail'):
1431                 tail = int(a)
1432             elif o in ('--title',):
1433                 self.title = a
1434             elif o in ('--which',):
1435                 if a not in list(self.time_strings.keys()):
1436                     sys.stderr.write('%s: time: Unrecognized timer "%s".\n' % (self.name, a))
1437                     sys.stderr.write('%s  Type "%s help time" for help.\n' % (self.name_spaces, self.name))
1438                     sys.exit(1)
1439                 which = a
1440
1441         if self.config_file:
1442             with open(self.config_file, 'r') as f:
1443                 config = f.read()
1444             HACK_for_exec(config, self.__dict__)
1445
1446         if self.chdir:
1447             os.chdir(self.chdir)
1448
1449             def _logfile_path_join(x):
1450                 return os.path.join(self.chdir, x)
1451
1452             logfile_path = _logfile_path_join
1453
1454         if not args:
1455             pattern = '%s*.log' % self.prefix
1456             args = self.args_to_files([pattern], tail)
1457
1458             if not args:
1459                 if self.chdir:
1460                     directory = self.chdir
1461                 else:
1462                     directory = os.getcwd()
1463
1464                 sys.stderr.write('%s: time: No arguments specified.\n' % self.name)
1465                 sys.stderr.write('%s  No %s*.log files found in "%s".\n' % (self.name_spaces, self.prefix, directory))
1466                 sys.stderr.write('%s  Type "%s help time" for help.\n' % (self.name_spaces, self.name))
1467                 sys.exit(1)
1468
1469         else:
1470
1471             args = self.args_to_files(args, tail)
1472
1473         cwd_ = os.getcwd() + os.sep
1474
1475         if format == 'ascii':
1476
1477             columns = ("Total", "SConscripts", "SCons", "commands")
1478             self.ascii_table(args, columns, self.get_debug_times, logfile_path)
1479
1480         elif format == 'gnuplot':
1481
1482             results = self.collect_results(args, self.get_debug_times,
1483                                            self.time_strings[which])
1484
1485             self.gnuplot_results(results, fmt='%s %.6f')
1486
1487         else:
1488
1489             sys.stderr.write('%s: time: Unknown format "%s".\n' % (self.name, format))
1490             sys.exit(1)
1491
1492
1493 if __name__ == '__main__':
1494     opts, args = getopt.getopt(sys.argv[1:], 'h?V', ['help', 'version'])
1495
1496     ST = SConsTimer()
1497
1498     for o, a in opts:
1499         if o in ('-?', '-h', '--help'):
1500             ST.do_help(['help'])
1501             sys.exit(0)
1502         elif o in ('-V', '--version'):
1503             sys.stdout.write('scons-time version\n')
1504             sys.exit(0)
1505
1506     if not args:
1507         sys.stderr.write('Type "%s help" for usage.\n' % ST.name)
1508         sys.exit(1)
1509
1510     ST.execute_subcommand(args)
1511
1512 # Local Variables:
1513 # tab-width:4
1514 # indent-tabs-mode:nil
1515 # End:
1516 # vim: set expandtab tabstop=4 shiftwidth=4: