bin/scons-time.py

   1 #!/usr/bin/env python
   2
   3 # MIT License
   4 #
   5 # Copyright The SCons Foundation
   6 #
   7 # Permission is hereby granted, free of charge, to any person obtaining
   8 # a copy of this software and associated documentation files (the
   9 # "Software"), to deal in the Software without restriction, including
  10 # without limitation the rights to use, copy, modify, merge, publish,
  11 # distribute, sublicense, and/or sell copies of the Software, and to
  12 # permit persons to whom the Software is furnished to do so, subject to
  13 # the following conditions:
  14 #
  15 # The above copyright notice and this permission notice shall be included
  16 # in all copies or substantial portions of the Software.
  17 #
  18 # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY
  19 # KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE
  20 # WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  21 # NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
  22 # LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
  23 # OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
  24 # WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
  25
  26 """
  27 scons-time - run SCons timings and collect statistics
  28
  29 A script for running a configuration through SCons with a standard set
  30 of invocations to collect timing and memory statistics and to capture
  31 the results in a consistent set of output files for display and analysis.
  32 """
  33
  34 import getopt
  35 import glob
  36 import os
  37 import re
  38 import shutil
  39 import sys
  40 import tempfile
  41 import time
  42
  43
  44 def HACK_for_exec(cmd, *args):
  45     """
  46     For some reason, Python won't allow an exec() within a function
  47     that also declares an internal function (including lambda functions).
  48     This function is a hack that calls exec() in a function with no
  49     internal functions.
  50     """
  51     if not args:
  52         exec(cmd)
  53     elif len(args) == 1:
  54         exec(cmd, args[0])
  55     else:
  56         exec(cmd, args[0], args[1])
  57
  58
  59 class Plotter:
  60     def increment_size(self, largest):
  61         """
  62         Return the size of each horizontal increment line for a specified
  63         maximum value.  This returns a value that will provide somewhere
  64         between 5 and 9 horizontal lines on the graph, on some set of
  65         boundaries that are multiples of 10/100/1000/etc.
  66         """
  67         i = largest // 5
  68         if not i:
  69             return largest
  70         multiplier = 1
  71         while i >= 10:
  72             i = i // 10
  73             multiplier = multiplier * 10
  74         return i * multiplier
  75
  76     def max_graph_value(self, largest):
  77         # Round up to next integer.
  78         largest = int(largest) + 1
  79         increment = self.increment_size(largest)
  80         return ((largest + increment - 1) // increment) * increment
  81
  82
  83 class Line:
  84     def __init__(self, points, type, title, label, comment, fmt="%s %s"):
  85         self.points = points
  86         self.type = type
  87         self.title = title
  88         self.label = label
  89         self.comment = comment
  90         self.fmt = fmt
  91
  92     def print_label(self, inx, x, y):
  93         if self.label:
  94             print('set label %s "%s" at %0.1f,%0.1f right' % (inx, self.label, x, y))
  95
  96     def plot_string(self):
  97         if self.title:
  98             title_string = 'title "%s"' % self.title
  99         else:
 100             title_string = 'notitle'
 101         return "'-' %s with lines lt %s" % (title_string, self.type)
 102
 103     def print_points(self, fmt=None):
 104         if fmt is None:
 105             fmt = self.fmt
 106         if self.comment:
 107             print('# %s' % self.comment)
 108         for x, y in self.points:
 109             # If y is None, it usually represents some kind of break
 110             # in the line's index number.  We might want to represent
 111             # this some way rather than just drawing the line straight
 112             # between the two points on either side.
 113             if y is not None:
 114                 print(fmt % (x, y))
 115         print('e')
 116
 117     def get_x_values(self):
 118         return [p[0] for p in self.points]
 119
 120     def get_y_values(self):
 121         return [p[1] for p in self.points]
 122
 123
 124 class Gnuplotter(Plotter):
 125
 126     def __init__(self, title, key_location):
 127         self.lines = []
 128         self.title = title
 129         self.key_location = key_location
 130
 131     def line(self, points, type, title=None, label=None, comment=None, fmt='%s %s'):
 132         if points:
 133             line = Line(points, type, title, label, comment, fmt)
 134             self.lines.append(line)
 135
 136     def plot_string(self, line):
 137         return line.plot_string()
 138
 139     def vertical_bar(self, x, type, label, comment):
 140         if self.get_min_x() <= x <= self.get_max_x():
 141             points = [(x, 0), (x, self.max_graph_value(self.get_max_y()))]
 142             self.line(points, type, label, comment)
 143
 144     def get_all_x_values(self):
 145         result = []
 146         for line in self.lines:
 147             result.extend(line.get_x_values())
 148         return [r for r in result if r is not None]
 149
 150     def get_all_y_values(self):
 151         result = []
 152         for line in self.lines:
 153             result.extend(line.get_y_values())
 154         return [r for r in result if r is not None]
 155
 156     def get_min_x(self):
 157         try:
 158             return self.min_x
 159         except AttributeError:
 160             try:
 161                 self.min_x = min(self.get_all_x_values())
 162             except ValueError:
 163                 self.min_x = 0
 164             return self.min_x
 165
 166     def get_max_x(self):
 167         try:
 168             return self.max_x
 169         except AttributeError:
 170             try:
 171                 self.max_x = max(self.get_all_x_values())
 172             except ValueError:
 173                 self.max_x = 0
 174             return self.max_x
 175
 176     def get_min_y(self):
 177         try:
 178             return self.min_y
 179         except AttributeError:
 180             try:
 181                 self.min_y = min(self.get_all_y_values())
 182             except ValueError:
 183                 self.min_y = 0
 184             return self.min_y
 185
 186     def get_max_y(self):
 187         try:
 188             return self.max_y
 189         except AttributeError:
 190             try:
 191                 self.max_y = max(self.get_all_y_values())
 192             except ValueError:
 193                 self.max_y = 0
 194             return self.max_y
 195
 196     def draw(self):
 197
 198         if not self.lines:
 199             return
 200
 201         if self.title:
 202             print('set title "%s"' % self.title)
 203         print('set key %s' % self.key_location)
 204
 205         min_y = self.get_min_y()
 206         max_y = self.max_graph_value(self.get_max_y())
 207         incr = (max_y - min_y) / 10.0
 208         start = min_y + (max_y / 2.0) + (2.0 * incr)
 209         position = [start - (i * incr) for i in range(5)]
 210
 211         inx = 1
 212         for line in self.lines:
 213             line.print_label(inx, line.points[0][0] - 1,
 214                              position[(inx - 1) % len(position)])
 215             inx += 1
 216
 217         plot_strings = [self.plot_string(l) for l in self.lines]
 218         print('plot ' + ', \\\n     '.join(plot_strings))
 219
 220         for line in self.lines:
 221             line.print_points()
 222
 223
 224 def untar(fname):
 225     import tarfile
 226     tar = tarfile.open(name=fname, mode='r')
 227     for tarinfo in tar:
 228         tar.extract(tarinfo)
 229     tar.close()
 230
 231
 232 def unzip(fname):
 233     import zipfile
 234     zf = zipfile.ZipFile(fname, 'r')
 235     for name in zf.namelist():
 236         dir = os.path.dirname(name)
 237         try:
 238             os.makedirs(dir)
 239         except OSError:
 240             pass
 241         with open(name, 'wb') as f:
 242             f.write(zf.read(name))
 243
 244
 245 def read_tree(dir):
 246     for dirpath, dirnames, filenames in os.walk(dir):
 247         for fn in filenames:
 248             fn = os.path.join(dirpath, fn)
 249             if os.path.isfile(fn):
 250                 with open(fn, 'rb') as f:
 251                     f.read()
 252
 253
 254 def redirect_to_file(command, log):
 255     return '%s > %s 2>&1' % (command, log)
 256
 257
 258 def tee_to_file(command, log):
 259     return '%s 2>&1 | tee %s' % (command, log)
 260
 261
 262 class SConsTimer:
 263     """
 264     Usage: scons-time SUBCOMMAND [ARGUMENTS]
 265     Type "scons-time help SUBCOMMAND" for help on a specific subcommand.
 266
 267     Available subcommands:
 268         func            Extract test-run data for a function
 269         help            Provides help
 270         mem             Extract --debug=memory data from test runs
 271         obj             Extract --debug=count data from test runs
 272         time            Extract --debug=time data from test runs
 273         run             Runs a test configuration
 274     """
 275
 276     name = 'scons-time'
 277     name_spaces = ' ' * len(name)
 278
 279     def makedict(**kw):
 280         return kw
 281
 282     default_settings = makedict(
 283         chdir=None,
 284         config_file=None,
 285         initial_commands=[],
 286         key_location='bottom left',
 287         orig_cwd=os.getcwd(),
 288         outdir=None,
 289         prefix='',
 290         python='"%s"' % sys.executable,
 291         redirect=redirect_to_file,
 292         scons=None,
 293         scons_flags='--debug=count --debug=memory --debug=time --debug=memoizer',
 294         scons_lib_dir=None,
 295         scons_wrapper=None,
 296         startup_targets='--help',
 297         subdir=None,
 298         subversion_url=None,
 299         svn='svn',
 300         svn_co_flag='-q',
 301         tar='tar',
 302         targets='',
 303         targets0=None,
 304         targets1=None,
 305         targets2=None,
 306         title=None,
 307         unzip='unzip',
 308         verbose=False,
 309         vertical_bars=[],
 310
 311         unpack_map={
 312             '.tar.gz': (untar, '%(tar)s xzf %%s'),
 313             '.tgz': (untar, '%(tar)s xzf %%s'),
 314             '.tar': (untar, '%(tar)s xf %%s'),
 315             '.zip': (unzip, '%(unzip)s %%s'),
 316         },
 317     )
 318
 319     run_titles = [
 320         'Startup',
 321         'Full build',
 322         'Up-to-date build',
 323     ]
 324
 325     run_commands = [
 326         '%(python)s %(scons_wrapper)s %(scons_flags)s --profile=%(prof0)s %(targets0)s',
 327         '%(python)s %(scons_wrapper)s %(scons_flags)s --profile=%(prof1)s %(targets1)s',
 328         '%(python)s %(scons_wrapper)s %(scons_flags)s --profile=%(prof2)s %(targets2)s',
 329     ]
 330
 331     stages = [
 332         'pre-read',
 333         'post-read',
 334         'pre-build',
 335         'post-build',
 336     ]
 337
 338     stage_strings = {
 339         'pre-read': 'Memory before reading SConscript files:',
 340         'post-read': 'Memory after reading SConscript files:',
 341         'pre-build': 'Memory before building targets:',
 342         'post-build': 'Memory after building targets:',
 343     }
 344
 345     memory_string_all = 'Memory '
 346
 347     default_stage = stages[-1]
 348
 349     time_strings = {
 350         'total': 'Total build time',
 351         'SConscripts': 'Total SConscript file execution time',
 352         'SCons': 'Total SCons execution time',
 353         'commands': 'Total command execution time',
 354     }
 355
 356     time_string_all = 'Total .* time'
 357
 358     #
 359
 360     def __init__(self):
 361         self.__dict__.update(self.default_settings)
 362
 363     # Functions for displaying and executing commands.
 364
 365     def subst(self, x, dictionary):
 366         try:
 367             return x % dictionary
 368         except TypeError:
 369             # x isn't a string (it's probably a Python function),
 370             # so just return it.
 371             return x
 372
 373     def subst_variables(self, command, dictionary):
 374         """
 375         Substitutes (via the format operator) the values in the specified
 376         dictionary into the specified command.
 377
 378         The command can be an (action, string) tuple.  In all cases, we
 379         perform substitution on strings and don't worry if something isn't
 380         a string.  (It's probably a Python function to be executed.)
 381         """
 382         try:
 383             command + ''
 384         except TypeError:
 385             action = command[0]
 386             string = command[1]
 387             args = command[2:]
 388         else:
 389             action = command
 390             string = action
 391             args = (())
 392         action = self.subst(action, dictionary)
 393         string = self.subst(string, dictionary)
 394         return (action, string, args)
 395
 396     def _do_not_display(self, msg, *args):
 397         pass
 398
 399     def display(self, msg, *args):
 400         """
 401         Displays the specified message.
 402
 403         Each message is prepended with a standard prefix of our name
 404         plus the time.
 405         """
 406         if callable(msg):
 407             msg = msg(*args)
 408         else:
 409             msg = msg % args
 410         if msg is None:
 411             return
 412         fmt = '%s[%s]: %s\n'
 413         sys.stdout.write(fmt % (self.name, time.strftime('%H:%M:%S'), msg))
 414
 415     def _do_not_execute(self, action, *args):
 416         pass
 417
 418     def execute(self, action, *args):
 419         """
 420         Executes the specified action.
 421
 422         The action is called if it's a callable Python function, and
 423         otherwise passed to os.system().
 424         """
 425         if callable(action):
 426             action(*args)
 427         else:
 428             os.system(action % args)
 429
 430     def run_command_list(self, commands, dict):
 431         """
 432         Executes a list of commands, substituting values from the
 433         specified dictionary.
 434         """
 435         commands = [self.subst_variables(c, dict) for c in commands]
 436         for action, string, args in commands:
 437             self.display(string, *args)
 438             sys.stdout.flush()
 439             status = self.execute(action, *args)
 440             if status:
 441                 sys.exit(status)
 442
 443     def log_display(self, command, log):
 444         command = self.subst(command, self.__dict__)
 445         if log:
 446             command = self.redirect(command, log)
 447         return command
 448
 449     def log_execute(self, command, log):
 450         command = self.subst(command, self.__dict__)
 451         p = os.popen(command)
 452         output = p.read()
 453         p.close()
 454         # TODO: convert to subrocess, os.popen is obsolete. This didn't work:
 455         # process = subprocess.Popen(command, stdout=subprocess.PIPE, shell=True)
 456         # output = process.stdout.read()
 457         # process.stdout.close()
 458         # process.wait()
 459         if self.verbose:
 460             sys.stdout.write(output)
 461         # TODO: Figure out
 462         # Not sure we need to write binary here
 463         with open(log, 'w') as f:
 464             f.write(str(output))
 465
 466     def archive_splitext(self, path):
 467         """
 468         Splits an archive name into a filename base and extension.
 469
 470         This is like os.path.splitext() (which it calls) except that it
 471         also looks for '.tar.gz' and treats it as an atomic extensions.
 472         """
 473         if path.endswith('.tar.gz'):
 474             return path[:-7], path[-7:]
 475         else:
 476             return os.path.splitext(path)
 477
 478     def args_to_files(self, args, tail=None):
 479         """
 480         Takes a list of arguments, expands any glob patterns, and
 481         returns the last "tail" files from the list.
 482         """
 483         files = []
 484         for a in args:
 485             files.extend(sorted(glob.glob(a)))
 486
 487         if tail:
 488             files = files[-tail:]
 489
 490         return files
 491
 492     def ascii_table(self, files, columns,
 493                     line_function, file_function=lambda x: x,
 494                     *args, **kw):
 495
 496         header_fmt = ' '.join(['%12s'] * len(columns))
 497         line_fmt = header_fmt + '    %s'
 498
 499         print(header_fmt % columns)
 500
 501         for file in files:
 502             t = line_function(file, *args, **kw)
 503             if t is None:
 504                 t = []
 505             diff = len(columns) - len(t)
 506             if diff > 0:
 507                 t += [''] * diff
 508             t.append(file_function(file))
 509             print(line_fmt % tuple(t))
 510
 511     def collect_results(self, files, function, *args, **kw):
 512         results = {}
 513
 514         for file in files:
 515             base = os.path.splitext(file)[0]
 516             run, index = base.split('-')[-2:]
 517
 518             run = int(run)
 519             index = int(index)
 520
 521             value = function(file, *args, **kw)
 522
 523             try:
 524                 r = results[index]
 525             except KeyError:
 526                 r = []
 527                 results[index] = r
 528             r.append((run, value))
 529
 530         return results
 531
 532     def doc_to_help(self, obj):
 533         """
 534         Translates an object's __doc__ string into help text.
 535
 536         This strips a consistent number of spaces from each line in the
 537         help text, essentially "outdenting" the text to the left-most
 538         column.
 539         """
 540         doc = obj.__doc__
 541         if doc is None:
 542             return ''
 543         return self.outdent(doc)
 544
 545     def find_next_run_number(self, dir, prefix):
 546         """
 547         Returns the next run number in a directory for the specified prefix.
 548
 549         Examines the contents the specified directory for files with the
 550         specified prefix, extracts the run numbers from each file name,
 551         and returns the next run number after the largest it finds.
 552         """
 553         x = re.compile(re.escape(prefix) + '-([0-9]+).*')
 554         matches = [x.match(e) for e in os.listdir(dir)]
 555         matches = [_f for _f in matches if _f]
 556         if not matches:
 557             return 0
 558         run_numbers = [int(m.group(1)) for m in matches]
 559         return int(max(run_numbers)) + 1
 560
 561     def gnuplot_results(self, results, fmt='%s %.3f'):
 562         """
 563         Prints out a set of results in Gnuplot format.
 564         """
 565         gp = Gnuplotter(self.title, self.key_location)
 566
 567         for i in sorted(results.keys()):
 568             try:
 569                 t = self.run_titles[i]
 570             except IndexError:
 571                 t = '??? %s ???' % i
 572             results[i].sort()
 573             gp.line(results[i], i + 1, t, None, t, fmt=fmt)
 574
 575         for bar_tuple in self.vertical_bars:
 576             try:
 577                 x, type, label, comment = bar_tuple
 578             except ValueError:
 579                 x, type, label = bar_tuple
 580                 comment = label
 581             gp.vertical_bar(x, type, label, comment)
 582
 583         gp.draw()
 584
 585     def logfile_name(self, invocation):
 586         """
 587         Returns the absolute path of a log file for the specificed
 588         invocation number.
 589         """
 590         name = self.prefix_run + '-%d.log' % invocation
 591         return os.path.join(self.outdir, name)
 592
 593     def outdent(self, s):
 594         """
 595         Strip as many spaces from each line as are found at the beginning
 596         of the first line in the list.
 597         """
 598         lines = s.split('\n')
 599         if lines[0] == '':
 600             lines = lines[1:]
 601         spaces = re.match(' *', lines[0]).group(0)
 602
 603         def strip_initial_spaces(line, s=spaces):
 604             if line.startswith(spaces):
 605                 line = line[len(spaces):]
 606             return line
 607
 608         return '\n'.join([strip_initial_spaces(l) for l in lines]) + '\n'
 609
 610     def profile_name(self, invocation):
 611         """
 612         Returns the absolute path of a profile file for the specified
 613         invocation number.
 614         """
 615         name = self.prefix_run + '-%d.prof' % invocation
 616         return os.path.join(self.outdir, name)
 617
 618     def set_env(self, key, value):
 619         os.environ[key] = value
 620
 621     #
 622
 623     def get_debug_times(self, file, time_string=None):
 624         """
 625         Fetch times from the --debug=time strings in the specified file.
 626         """
 627         if time_string is None:
 628             search_string = self.time_string_all
 629         else:
 630             search_string = time_string
 631         with open(file) as f:
 632             contents = f.read()
 633         if not contents:
 634             sys.stderr.write('file %s has no contents!\n' % repr(file))
 635             return None
 636         result = re.findall(r'%s: ([\d.]*)' % search_string, contents)[-4:]
 637         result = [float(r) for r in result]
 638         if time_string is not None:
 639             try:
 640                 result = result[0]
 641             except IndexError:
 642                 sys.stderr.write('file %s has no results!\n' % repr(file))
 643                 return None
 644         return result
 645
 646     def get_function_profile(self, file, function):
 647         """
 648         Returns the file, line number, function name, and cumulative time.
 649         """
 650         try:
 651             import pstats
 652         except ImportError as e:
 653             sys.stderr.write('%s: func: %s\n' % (self.name, e))
 654             sys.stderr.write('%s  This version of Python is missing the profiler.\n' % self.name_spaces)
 655             sys.stderr.write('%s  Cannot use the "func" subcommand.\n' % self.name_spaces)
 656             sys.exit(1)
 657         statistics = pstats.Stats(file).stats
 658         matches = [e for e in statistics.items() if e[0][2] == function]
 659         r = matches[0]
 660         return r[0][0], r[0][1], r[0][2], r[1][3]
 661
 662     def get_function_time(self, file, function):
 663         """
 664         Returns just the cumulative time for the specified function.
 665         """
 666         return self.get_function_profile(file, function)[3]
 667
 668     def get_memory(self, file, memory_string=None):
 669         """
 670         Returns a list of integers of the amount of memory used.  The
 671         default behavior is to return all the stages.
 672         """
 673         if memory_string is None:
 674             search_string = self.memory_string_all
 675         else:
 676             search_string = memory_string
 677         with open(file) as f:
 678             lines = f.readlines()
 679         lines = [l for l in lines if l.startswith(search_string)][-4:]
 680         result = [int(l.split()[-1]) for l in lines[-4:]]
 681         if len(result) == 1:
 682             result = result[0]
 683         return result
 684
 685     def get_object_counts(self, file, object_name, index=None):
 686         """
 687         Returns the counts of the specified object_name.
 688         """
 689         object_string = ' ' + object_name + '\n'
 690         with open(file) as f:
 691             lines = f.readlines()
 692         line = [l for l in lines if l.endswith(object_string)][0]
 693         result = [int(field) for field in line.split()[:4]]
 694         if index is not None:
 695             result = result[index]
 696         return result
 697
 698     command_alias = {}
 699
 700     def execute_subcommand(self, argv):
 701         """
 702         Executes the do_*() function for the specified subcommand (argv[0]).
 703         """
 704         if not argv:
 705             return
 706         cmdName = self.command_alias.get(argv[0], argv[0])
 707         try:
 708             func = getattr(self, 'do_' + cmdName)
 709         except AttributeError:
 710             return self.default(argv)
 711         try:
 712             return func(argv)
 713         except TypeError as e:
 714             sys.stderr.write("%s %s: %s\n" % (self.name, cmdName, e))
 715             import traceback
 716             traceback.print_exc(file=sys.stderr)
 717             sys.stderr.write("Try '%s help %s'\n" % (self.name, cmdName))
 718
 719     def default(self, argv):
 720         """
 721         The default behavior for an unknown subcommand.  Prints an
 722         error message and exits.
 723         """
 724         sys.stderr.write('%s: Unknown subcommand "%s".\n' % (self.name, argv[0]))
 725         sys.stderr.write('Type "%s help" for usage.\n' % self.name)
 726         sys.exit(1)
 727
 728     #
 729
 730     def do_help(self, argv):
 731         """
 732         """
 733         if argv[1:]:
 734             for arg in argv[1:]:
 735                 try:
 736                     func = getattr(self, 'do_' + arg)
 737                 except AttributeError:
 738                     sys.stderr.write('%s: No help for "%s"\n' % (self.name, arg))
 739                 else:
 740                     try:
 741                         help = getattr(self, 'help_' + arg)
 742                     except AttributeError:
 743                         sys.stdout.write(self.doc_to_help(func))
 744                         sys.stdout.flush()
 745                     else:
 746                         help()
 747         else:
 748             doc = self.doc_to_help(self.__class__)
 749             if doc:
 750                 sys.stdout.write(doc)
 751             sys.stdout.flush()
 752             return None
 753
 754     #
 755
 756     def help_func(self):
 757         help = """\
 758         Usage: scons-time func [OPTIONS] FILE [...]
 759
 760           -C DIR, --chdir=DIR           Change to DIR before looking for files
 761           -f FILE, --file=FILE          Read configuration from specified FILE
 762           --fmt=FORMAT, --format=FORMAT Print data in specified FORMAT
 763           --func=NAME, --function=NAME  Report time for function NAME
 764           -h, --help                    Print this help and exit
 765           -p STRING, --prefix=STRING    Use STRING as log file/profile prefix
 766           -t NUMBER, --tail=NUMBER      Only report the last NUMBER files
 767           --title=TITLE                 Specify the output plot TITLE
 768         """
 769         sys.stdout.write(self.outdent(help))
 770         sys.stdout.flush()
 771
 772     def do_func(self, argv):
 773         """
 774         """
 775         format = 'ascii'
 776         function_name = '_main'
 777         tail = None
 778
 779         short_opts = '?C:f:hp:t:'
 780
 781         long_opts = [
 782             'chdir=',
 783             'file=',
 784             'fmt=',
 785             'format=',
 786             'func=',
 787             'function=',
 788             'help',
 789             'prefix=',
 790             'tail=',
 791             'title=',
 792         ]
 793
 794         opts, args = getopt.getopt(argv[1:], short_opts, long_opts)
 795
 796         for o, a in opts:
 797             if o in ('-C', '--chdir'):
 798                 self.chdir = a
 799             elif o in ('-f', '--file'):
 800                 self.config_file = a
 801             elif o in ('--fmt', '--format'):
 802                 format = a
 803             elif o in ('--func', '--function'):
 804                 function_name = a
 805             elif o in ('-?', '-h', '--help'):
 806                 self.do_help(['help', 'func'])
 807                 sys.exit(0)
 808             elif o in ('--max',):
 809                 max_time = int(a)
 810             elif o in ('-p', '--prefix'):
 811                 self.prefix = a
 812             elif o in ('-t', '--tail'):
 813                 tail = int(a)
 814             elif o in ('--title',):
 815                 self.title = a
 816
 817         if self.config_file:
 818             with open(self.config_file, 'r') as f:
 819                 config = f.read()
 820             exec(config, self.__dict__)
 821
 822         if self.chdir:
 823             os.chdir(self.chdir)
 824
 825         if not args:
 826
 827             pattern = '%s*.prof' % self.prefix
 828             args = self.args_to_files([pattern], tail)
 829
 830             if not args:
 831                 if self.chdir:
 832                     directory = self.chdir
 833                 else:
 834                     directory = os.getcwd()
 835
 836                 sys.stderr.write('%s: func: No arguments specified.\n' % self.name)
 837                 sys.stderr.write('%s  No %s*.prof files found in "%s".\n' % (self.name_spaces, self.prefix, directory))
 838                 sys.stderr.write('%s  Type "%s help func" for help.\n' % (self.name_spaces, self.name))
 839                 sys.exit(1)
 840
 841         else:
 842
 843             args = self.args_to_files(args, tail)
 844
 845         cwd_ = os.getcwd() + os.sep
 846
 847         if format == 'ascii':
 848
 849             for file in args:
 850                 try:
 851                     f, line, func, time = \
 852                         self.get_function_profile(file, function_name)
 853                 except ValueError as e:
 854                     sys.stderr.write("%s: func: %s: %s\n" %
 855                                      (self.name, file, e))
 856                 else:
 857                     if f.startswith(cwd_):
 858                         f = f[len(cwd_):]
 859                     print("%.3f %s:%d(%s)" % (time, f, line, func))
 860
 861         elif format == 'gnuplot':
 862
 863             results = self.collect_results(args, self.get_function_time,
 864                                            function_name)
 865
 866             self.gnuplot_results(results)
 867
 868         else:
 869
 870             sys.stderr.write('%s: func: Unknown format "%s".\n' % (self.name, format))
 871             sys.exit(1)
 872
 873     #
 874
 875     def help_mem(self):
 876         help = """\
 877         Usage: scons-time mem [OPTIONS] FILE [...]
 878
 879           -C DIR, --chdir=DIR           Change to DIR before looking for files
 880           -f FILE, --file=FILE          Read configuration from specified FILE
 881           --fmt=FORMAT, --format=FORMAT Print data in specified FORMAT
 882           -h, --help                    Print this help and exit
 883           -p STRING, --prefix=STRING    Use STRING as log file/profile prefix
 884           --stage=STAGE                 Plot memory at the specified stage:
 885                                           pre-read, post-read, pre-build,
 886                                           post-build (default: post-build)
 887           -t NUMBER, --tail=NUMBER      Only report the last NUMBER files
 888           --title=TITLE                 Specify the output plot TITLE
 889         """
 890         sys.stdout.write(self.outdent(help))
 891         sys.stdout.flush()
 892
 893     def do_mem(self, argv):
 894
 895         format = 'ascii'
 896         def _logfile_path(x):
 897             return x
 898
 899         logfile_path = _logfile_path
 900
 901         stage = self.default_stage
 902         tail = None
 903
 904         short_opts = '?C:f:hp:t:'
 905
 906         long_opts = [
 907             'chdir=',
 908             'file=',
 909             'fmt=',
 910             'format=',
 911             'help',
 912             'prefix=',
 913             'stage=',
 914             'tail=',
 915             'title=',
 916         ]
 917
 918         opts, args = getopt.getopt(argv[1:], short_opts, long_opts)
 919
 920         for o, a in opts:
 921             if o in ('-C', '--chdir'):
 922                 self.chdir = a
 923             elif o in ('-f', '--file'):
 924                 self.config_file = a
 925             elif o in ('--fmt', '--format'):
 926                 format = a
 927             elif o in ('-?', '-h', '--help'):
 928                 self.do_help(['help', 'mem'])
 929                 sys.exit(0)
 930             elif o in ('-p', '--prefix'):
 931                 self.prefix = a
 932             elif o in ('--stage',):
 933                 if a not in self.stages:
 934                     sys.stderr.write('%s: mem: Unrecognized stage "%s".\n' % (self.name, a))
 935                     sys.exit(1)
 936                 stage = a
 937             elif o in ('-t', '--tail'):
 938                 tail = int(a)
 939             elif o in ('--title',):
 940                 self.title = a
 941
 942         if self.config_file:
 943             with open(self.config_file, 'r') as f:
 944                 config = f.read()
 945             HACK_for_exec(config, self.__dict__)
 946
 947         if self.chdir:
 948             os.chdir(self.chdir)
 949             def _logfile_path_join(x):
 950                 return os.path.join(self.chdir, x)
 951
 952             logfile_path = _logfile_path_join
 953
 954         if not args:
 955             pattern = '%s*.log' % self.prefix
 956             args = self.args_to_files([pattern], tail)
 957
 958             if not args:
 959                 if self.chdir:
 960                     directory = self.chdir
 961                 else:
 962                     directory = os.getcwd()
 963
 964                 sys.stderr.write('%s: mem: No arguments specified.\n' % self.name)
 965                 sys.stderr.write('%s  No %s*.log files found in "%s".\n' % (self.name_spaces, self.prefix, directory))
 966                 sys.stderr.write('%s  Type "%s help mem" for help.\n' % (self.name_spaces, self.name))
 967                 sys.exit(1)
 968
 969         else:
 970
 971             args = self.args_to_files(args, tail)
 972
 973         # cwd_ = os.getcwd() + os.sep
 974
 975         if format == 'ascii':
 976
 977             self.ascii_table(args, tuple(self.stages), self.get_memory, logfile_path)
 978
 979         elif format == 'gnuplot':
 980
 981             results = self.collect_results(args, self.get_memory,
 982                                            self.stage_strings[stage])
 983
 984             self.gnuplot_results(results)
 985
 986         else:
 987
 988             sys.stderr.write('%s: mem: Unknown format "%s".\n' % (self.name, format))
 989             sys.exit(1)
 990
 991         return 0
 992
 993     #
 994
 995     def help_obj(self):
 996         help = """\
 997         Usage: scons-time obj [OPTIONS] OBJECT FILE [...]
 998
 999           -C DIR, --chdir=DIR           Change to DIR before looking for files
1000           -f FILE, --file=FILE          Read configuration from specified FILE
1001           --fmt=FORMAT, --format=FORMAT Print data in specified FORMAT
1002           -h, --help                    Print this help and exit
1003           -p STRING, --prefix=STRING    Use STRING as log file/profile prefix
1004           --stage=STAGE                 Plot memory at the specified stage:
1005                                           pre-read, post-read, pre-build,
1006                                           post-build (default: post-build)
1007           -t NUMBER, --tail=NUMBER      Only report the last NUMBER files
1008           --title=TITLE                 Specify the output plot TITLE
1009         """
1010         sys.stdout.write(self.outdent(help))
1011         sys.stdout.flush()
1012
1013     def do_obj(self, argv):
1014
1015         format = 'ascii'
1016
1017         def _logfile_path(x):
1018             return x
1019
1020         logfile_path = _logfile_path
1021
1022         stage = self.default_stage
1023         tail = None
1024
1025         short_opts = '?C:f:hp:t:'
1026
1027         long_opts = [
1028             'chdir=',
1029             'file=',
1030             'fmt=',
1031             'format=',
1032             'help',
1033             'prefix=',
1034             'stage=',
1035             'tail=',
1036             'title=',
1037         ]
1038
1039         opts, args = getopt.getopt(argv[1:], short_opts, long_opts)
1040
1041         for o, a in opts:
1042             if o in ('-C', '--chdir'):
1043                 self.chdir = a
1044             elif o in ('-f', '--file'):
1045                 self.config_file = a
1046             elif o in ('--fmt', '--format'):
1047                 format = a
1048             elif o in ('-?', '-h', '--help'):
1049                 self.do_help(['help', 'obj'])
1050                 sys.exit(0)
1051             elif o in ('-p', '--prefix'):
1052                 self.prefix = a
1053             elif o in ('--stage',):
1054                 if a not in self.stages:
1055                     sys.stderr.write('%s: obj: Unrecognized stage "%s".\n' % (self.name, a))
1056                     sys.stderr.write('%s       Type "%s help obj" for help.\n' % (self.name_spaces, self.name))
1057                     sys.exit(1)
1058                 stage = a
1059             elif o in ('-t', '--tail'):
1060                 tail = int(a)
1061             elif o in ('--title',):
1062                 self.title = a
1063
1064         if not args:
1065             sys.stderr.write('%s: obj: Must specify an object name.\n' % self.name)
1066             sys.stderr.write('%s       Type "%s help obj" for help.\n' % (self.name_spaces, self.name))
1067             sys.exit(1)
1068
1069         object_name = args.pop(0)
1070
1071         if self.config_file:
1072             with open(self.config_file, 'r') as f:
1073                 config = f.read()
1074             HACK_for_exec(config, self.__dict__)
1075
1076         if self.chdir:
1077             os.chdir(self.chdir)
1078
1079             def _logfile_path_join(x):
1080                 return os.path.join(self.chdir, x)
1081
1082             logfile_path = _logfile_path_join
1083
1084         if not args:
1085             pattern = '%s*.log' % self.prefix
1086             args = self.args_to_files([pattern], tail)
1087
1088             if not args:
1089                 if self.chdir:
1090                     directory = self.chdir
1091                 else:
1092                     directory = os.getcwd()
1093
1094                 sys.stderr.write('%s: obj: No arguments specified.\n' % self.name)
1095                 sys.stderr.write('%s  No %s*.log files found in "%s".\n' % (self.name_spaces, self.prefix, directory))
1096                 sys.stderr.write('%s  Type "%s help obj" for help.\n' % (self.name_spaces, self.name))
1097                 sys.exit(1)
1098
1099         else:
1100
1101             args = self.args_to_files(args, tail)
1102
1103         cwd_ = os.getcwd() + os.sep
1104
1105         if format == 'ascii':
1106
1107             self.ascii_table(args, tuple(self.stages), self.get_object_counts, logfile_path, object_name)
1108
1109         elif format == 'gnuplot':
1110
1111             stage_index = 0
1112             for s in self.stages:
1113                 if stage == s:
1114                     break
1115                 stage_index = stage_index + 1
1116
1117             results = self.collect_results(args, self.get_object_counts,
1118                                            object_name, stage_index)
1119
1120             self.gnuplot_results(results)
1121
1122         else:
1123
1124             sys.stderr.write('%s: obj: Unknown format "%s".\n' % (self.name, format))
1125             sys.exit(1)
1126
1127         return 0
1128
1129     #
1130
1131     def help_run(self):
1132         help = """\
1133         Usage: scons-time run [OPTIONS] [FILE ...]
1134
1135           --chdir=DIR                   Name of unpacked directory for chdir
1136           -f FILE, --file=FILE          Read configuration from specified FILE
1137           -h, --help                    Print this help and exit
1138           -n, --no-exec                 No execute, just print command lines
1139           --number=NUMBER               Put output in files for run NUMBER
1140           --outdir=OUTDIR               Put output files in OUTDIR
1141           -p STRING, --prefix=STRING    Use STRING as log file/profile prefix
1142           --python=PYTHON               Time using the specified PYTHON
1143           -q, --quiet                   Don't print command lines
1144           --scons=SCONS                 Time using the specified SCONS
1145           --svn=URL, --subversion=URL   Use SCons from Subversion URL
1146           -v, --verbose                 Display output of commands
1147         """
1148         sys.stdout.write(self.outdent(help))
1149         sys.stdout.flush()
1150
1151     def do_run(self, argv):
1152         """
1153         """
1154         run_number_list = [None]
1155
1156         short_opts = '?f:hnp:qs:v'
1157
1158         long_opts = [
1159             'file=',
1160             'help',
1161             'no-exec',
1162             'number=',
1163             'outdir=',
1164             'prefix=',
1165             'python=',
1166             'quiet',
1167             'scons=',
1168             'svn=',
1169             'subdir=',
1170             'subversion=',
1171             'verbose',
1172         ]
1173
1174         opts, args = getopt.getopt(argv[1:], short_opts, long_opts)
1175
1176         for o, a in opts:
1177             if o in ('-f', '--file'):
1178                 self.config_file = a
1179             elif o in ('-?', '-h', '--help'):
1180                 self.do_help(['help', 'run'])
1181                 sys.exit(0)
1182             elif o in ('-n', '--no-exec'):
1183                 self.execute = self._do_not_execute
1184             elif o in ('--number',):
1185                 run_number_list = self.split_run_numbers(a)
1186             elif o in ('--outdir',):
1187                 self.outdir = a
1188             elif o in ('-p', '--prefix'):
1189                 self.prefix = a
1190             elif o in ('--python',):
1191                 self.python = a
1192             elif o in ('-q', '--quiet'):
1193                 self.display = self._do_not_display
1194             elif o in ('-s', '--subdir'):
1195                 self.subdir = a
1196             elif o in ('--scons',):
1197                 self.scons = a
1198             elif o in ('--svn', '--subversion'):
1199                 self.subversion_url = a
1200             elif o in ('-v', '--verbose'):
1201                 self.redirect = tee_to_file
1202                 self.verbose = True
1203                 self.svn_co_flag = ''
1204
1205         if not args and not self.config_file:
1206             sys.stderr.write('%s: run: No arguments or -f config file specified.\n' % self.name)
1207             sys.stderr.write('%s  Type "%s help run" for help.\n' % (self.name_spaces, self.name))
1208             sys.exit(1)
1209
1210         if self.config_file:
1211             with open(self.config_file, 'r') as f:
1212                 config = f.read()
1213             exec(config, self.__dict__)
1214
1215         if args:
1216             self.archive_list = args
1217
1218         archive_file_name = os.path.split(self.archive_list[0])[1]
1219
1220         if not self.subdir:
1221             self.subdir = self.archive_splitext(archive_file_name)[0]
1222
1223         if not self.prefix:
1224             self.prefix = self.archive_splitext(archive_file_name)[0]
1225
1226         prepare = None
1227         if self.subversion_url:
1228             prepare = self.prep_subversion_run
1229
1230         for run_number in run_number_list:
1231             self.individual_run(run_number, self.archive_list, prepare)
1232
1233     def split_run_numbers(self, s):
1234         result = []
1235         for n in s.split(','):
1236             try:
1237                 x, y = n.split('-')
1238             except ValueError:
1239                 result.append(int(n))
1240             else:
1241                 result.extend(list(range(int(x), int(y) + 1)))
1242         return result
1243
1244     def scons_path(self, dir):
1245         return os.path.join(dir, 'scripts', 'scons.py')
1246
1247     def scons_lib_dir_path(self, dir):
1248         """build the path to the engine.
1249
1250         this used to join src/engine, but no longer.
1251         """
1252         return dir
1253
1254     def prep_subversion_run(self, commands, removals):
1255         self.svn_tmpdir = tempfile.mkdtemp(prefix=self.name + '-svn-')
1256         removals.append((shutil.rmtree, 'rm -rf %%s', self.svn_tmpdir))
1257
1258         self.scons = self.scons_path(self.svn_tmpdir)
1259         self.scons_lib_dir = self.scons_lib_dir_path(self.svn_tmpdir)
1260
1261         commands.extend([
1262             '%(svn)s co %(svn_co_flag)s -r %(run_number)s %(subversion_url)s %(svn_tmpdir)s',
1263         ])
1264
1265     def individual_run(self, run_number, archive_list, prepare=None):
1266         """
1267         Performs an individual run of the default SCons invocations.
1268         """
1269
1270         commands = []
1271         removals = []
1272
1273         if prepare:
1274             prepare(commands, removals)
1275
1276         save_scons = self.scons
1277         save_scons_wrapper = self.scons_wrapper
1278         save_scons_lib_dir = self.scons_lib_dir
1279
1280         if self.outdir is None:
1281             self.outdir = self.orig_cwd
1282         elif not os.path.isabs(self.outdir):
1283             self.outdir = os.path.join(self.orig_cwd, self.outdir)
1284
1285         if self.scons is None:
1286             self.scons = self.scons_path(self.orig_cwd)
1287
1288         if self.scons_lib_dir is None:
1289             self.scons_lib_dir = self.scons_lib_dir_path(self.orig_cwd)
1290
1291         if self.scons_wrapper is None:
1292             self.scons_wrapper = self.scons
1293
1294         if not run_number:
1295             run_number = self.find_next_run_number(self.outdir, self.prefix)
1296
1297         self.run_number = str(run_number)
1298
1299         self.prefix_run = self.prefix + '-%03d' % run_number
1300
1301         if self.targets0 is None:
1302             self.targets0 = self.startup_targets
1303         if self.targets1 is None:
1304             self.targets1 = self.targets
1305         if self.targets2 is None:
1306             self.targets2 = self.targets
1307
1308         self.tmpdir = tempfile.mkdtemp(prefix=self.name + '-')
1309
1310         commands.extend([
1311             (os.chdir, 'cd %%s', self.tmpdir),
1312         ])
1313
1314         for archive in archive_list:
1315             if not os.path.isabs(archive):
1316                 archive = os.path.join(self.orig_cwd, archive)
1317             if os.path.isdir(archive):
1318                 dest = os.path.split(archive)[1]
1319                 commands.append((shutil.copytree, 'cp -r %%s %%s', archive, dest))
1320             else:
1321                 suffix = self.archive_splitext(archive)[1]
1322                 unpack_command = self.unpack_map.get(suffix)
1323                 if not unpack_command:
1324                     dest = os.path.split(archive)[1]
1325                     commands.append((shutil.copyfile, 'cp %%s %%s', archive, dest))
1326                 else:
1327                     commands.append(unpack_command + (archive,))
1328
1329         commands.extend([
1330             (os.chdir, 'cd %%s', self.subdir),
1331         ])
1332
1333         commands.extend(self.initial_commands)
1334
1335         commands.extend([
1336             (lambda: read_tree('.'),
1337              'find * -type f | xargs cat > /dev/null'),
1338
1339             (self.set_env, 'export %%s=%%s',
1340              'SCONS_LIB_DIR', self.scons_lib_dir),
1341
1342             '%(python)s %(scons_wrapper)s --version',
1343         ])
1344
1345         index = 0
1346         for run_command in self.run_commands:
1347             setattr(self, 'prof%d' % index, self.profile_name(index))
1348             c = (
1349                 self.log_execute,
1350                 self.log_display,
1351                 run_command,
1352                 self.logfile_name(index),
1353             )
1354             commands.append(c)
1355             index = index + 1
1356
1357         commands.extend([
1358             (os.chdir, 'cd %%s', self.orig_cwd),
1359         ])
1360
1361         if not os.environ.get('PRESERVE'):
1362             commands.extend(removals)
1363             commands.append((shutil.rmtree, 'rm -rf %%s', self.tmpdir))
1364
1365         self.run_command_list(commands, self.__dict__)
1366
1367         self.scons = save_scons
1368         self.scons_lib_dir = save_scons_lib_dir
1369         self.scons_wrapper = save_scons_wrapper
1370
1371     #
1372
1373     def help_time(self):
1374         help = """\
1375         Usage: scons-time time [OPTIONS] FILE [...]
1376
1377           -C DIR, --chdir=DIR           Change to DIR before looking for files
1378           -f FILE, --file=FILE          Read configuration from specified FILE
1379           --fmt=FORMAT, --format=FORMAT Print data in specified FORMAT
1380           -h, --help                    Print this help and exit
1381           -p STRING, --prefix=STRING    Use STRING as log file/profile prefix
1382           -t NUMBER, --tail=NUMBER      Only report the last NUMBER files
1383           --which=TIMER                 Plot timings for TIMER:  total,
1384                                           SConscripts, SCons, commands.
1385         """
1386         sys.stdout.write(self.outdent(help))
1387         sys.stdout.flush()
1388
1389     def do_time(self, argv):
1390
1391         format = 'ascii'
1392
1393         def _logfile_path(x):
1394             return x
1395
1396         logfile_path = _logfile_path
1397
1398         tail = None
1399         which = 'total'
1400
1401         short_opts = '?C:f:hp:t:'
1402
1403         long_opts = [
1404             'chdir=',
1405             'file=',
1406             'fmt=',
1407             'format=',
1408             'help',
1409             'prefix=',
1410             'tail=',
1411             'title=',
1412             'which=',
1413         ]
1414
1415         opts, args = getopt.getopt(argv[1:], short_opts, long_opts)
1416
1417         for o, a in opts:
1418             if o in ('-C', '--chdir'):
1419                 self.chdir = a
1420             elif o in ('-f', '--file'):
1421                 self.config_file = a
1422             elif o in ('--fmt', '--format'):
1423                 format = a
1424             elif o in ('-?', '-h', '--help'):
1425                 self.do_help(['help', 'time'])
1426                 sys.exit(0)
1427             elif o in ('-p', '--prefix'):
1428                 self.prefix = a
1429             elif o in ('-t', '--tail'):
1430                 tail = int(a)
1431             elif o in ('--title',):
1432                 self.title = a
1433             elif o in ('--which',):
1434                 if a not in list(self.time_strings.keys()):
1435                     sys.stderr.write('%s: time: Unrecognized timer "%s".\n' % (self.name, a))
1436                     sys.stderr.write('%s  Type "%s help time" for help.\n' % (self.name_spaces, self.name))
1437                     sys.exit(1)
1438                 which = a
1439
1440         if self.config_file:
1441             with open(self.config_file, 'r') as f:
1442                 config = f.read()
1443             HACK_for_exec(config, self.__dict__)
1444
1445         if self.chdir:
1446             os.chdir(self.chdir)
1447
1448             def _logfile_path_join(x):
1449                 return os.path.join(self.chdir, x)
1450
1451             logfile_path = _logfile_path_join
1452
1453         if not args:
1454             pattern = '%s*.log' % self.prefix
1455             args = self.args_to_files([pattern], tail)
1456
1457             if not args:
1458                 if self.chdir:
1459                     directory = self.chdir
1460                 else:
1461                     directory = os.getcwd()
1462
1463                 sys.stderr.write('%s: time: No arguments specified.\n' % self.name)
1464                 sys.stderr.write('%s  No %s*.log files found in "%s".\n' % (self.name_spaces, self.prefix, directory))
1465                 sys.stderr.write('%s  Type "%s help time" for help.\n' % (self.name_spaces, self.name))
1466                 sys.exit(1)
1467
1468         else:
1469
1470             args = self.args_to_files(args, tail)
1471
1472         cwd_ = os.getcwd() + os.sep
1473
1474         if format == 'ascii':
1475
1476             columns = ("Total", "SConscripts", "SCons", "commands")
1477             self.ascii_table(args, columns, self.get_debug_times, logfile_path)
1478
1479         elif format == 'gnuplot':
1480
1481             results = self.collect_results(args, self.get_debug_times,
1482                                            self.time_strings[which])
1483
1484             self.gnuplot_results(results, fmt='%s %.6f')
1485
1486         else:
1487
1488             sys.stderr.write('%s: time: Unknown format "%s".\n' % (self.name, format))
1489             sys.exit(1)
1490
1491
1492 if __name__ == '__main__':
1493     opts, args = getopt.getopt(sys.argv[1:], 'h?V', ['help', 'version'])
1494
1495     ST = SConsTimer()
1496
1497     for o, a in opts:
1498         if o in ('-?', '-h', '--help'):
1499             ST.do_help(['help'])
1500             sys.exit(0)
1501         elif o in ('-V', '--version'):
1502             sys.stdout.write('scons-time version\n')
1503             sys.exit(0)
1504
1505     if not args:
1506         sys.stderr.write('Type "%s help" for usage.\n' % ST.name)
1507         sys.exit(1)
1508
1509     ST.execute_subcommand(args)
1510
1511 # Local Variables:
1512 # tab-width:4
1513 # indent-tabs-mode:nil
1514 # End:
1515 # vim: set expandtab tabstop=4 shiftwidth=4: