Lib/pstats.py

   1 """Class for printing reports on profiled python code."""
   2
   3 # Class for printing reports on profiled python code. rev 1.0  4/1/94
   4 #
   5 # Based on prior profile module by Sjoerd Mullender...
   6 #   which was hacked somewhat by: Guido van Rossum
   7 #
   8 # see profile.doc and profile.py for more info.
   9
  10 # Copyright 1994, by InfoSeek Corporation, all rights reserved.
  11 # Written by James Roskind
  12 #
  13 # Permission to use, copy, modify, and distribute this Python software
  14 # and its associated documentation for any purpose (subject to the
  15 # restriction in the following sentence) without fee is hereby granted,
  16 # provided that the above copyright notice appears in all copies, and
  17 # that both that copyright notice and this permission notice appear in
  18 # supporting documentation, and that the name of InfoSeek not be used in
  19 # advertising or publicity pertaining to distribution of the software
  20 # without specific, written prior permission.  This permission is
  21 # explicitly restricted to the copying and modification of the software
  22 # to remain in Python, compiled Python, or other languages (such as C)
  23 # wherein the modified or derived code is exclusively imported into a
  24 # Python module.
  25 #
  26 # INFOSEEK CORPORATION DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS
  27 # SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND
  28 # FITNESS. IN NO EVENT SHALL INFOSEEK CORPORATION BE LIABLE FOR ANY
  29 # SPECIAL, INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER
  30 # RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF
  31 # CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN
  32 # CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
  33
  34
  35 import os
  36 import time
  37 import marshal
  38 import re
  39
  40 __all__ = ["Stats"]
  41
  42 class Stats:
  43     """This class is used for creating reports from data generated by the
  44     Profile class.  It is a "friend" of that class, and imports data either
  45     by direct access to members of Profile class, or by reading in a dictionary
  46     that was emitted (via marshal) from the Profile class.
  47
  48     The big change from the previous Profiler (in terms of raw functionality)
  49     is that an "add()" method has been provided to combine Stats from
  50     several distinct profile runs.  Both the constructor and the add()
  51     method now take arbitrarily many file names as arguments.
  52
  53     All the print methods now take an argument that indicates how many lines
  54     to print.  If the arg is a floating point number between 0 and 1.0, then
  55     it is taken as a decimal percentage of the available lines to be printed
  56     (e.g., .1 means print 10% of all available lines).  If it is an integer,
  57     it is taken to mean the number of lines of data that you wish to have
  58     printed.
  59
  60     The sort_stats() method now processes some additional options (i.e., in
  61     addition to the old -1, 0, 1, or 2).  It takes an arbitrary number of quoted
  62     strings to select the sort order.  For example sort_stats('time', 'name')
  63     sorts on the major key of "internal function time", and on the minor
  64     key of 'the name of the function'.  Look at the two tables in sort_stats()
  65     and get_sort_arg_defs(self) for more examples.
  66
  67     All methods now return "self",  so you can string together commands like:
  68         Stats('foo', 'goo').strip_dirs().sort_stats('calls').\
  69                             print_stats(5).print_callers(5)
  70     """
  71
  72     def __init__(self, *args):
  73         if not len(args):
  74             arg = None
  75         else:
  76             arg = args[0]
  77             args = args[1:]
  78         self.init(arg)
  79         self.add(*args)
  80
  81     def init(self, arg):
  82         self.all_callees = None  # calc only if needed
  83         self.files = []
  84         self.fcn_list = None
  85         self.total_tt = 0
  86         self.total_calls = 0
  87         self.prim_calls = 0
  88         self.max_name_len = 0
  89         self.top_level = {}
  90         self.stats = {}
  91         self.sort_arg_dict = {}
  92         self.load_stats(arg)
  93         trouble = 1
  94         try:
  95             self.get_top_level_stats()
  96             trouble = 0
  97         finally:
  98             if trouble:
  99                 print "Invalid timing data",
 100                 if self.files: print self.files[-1],
 101                 print
 102
 103     def load_stats(self, arg):
 104         if not arg:  self.stats = {}
 105         elif type(arg) == type(""):
 106             f = open(arg, 'rb')
 107             self.stats = marshal.load(f)
 108             f.close()
 109             try:
 110                 file_stats = os.stat(arg)
 111                 arg = time.ctime(file_stats.st_mtime) + "    " + arg
 112             except:  # in case this is not unix
 113                 pass
 114             self.files = [ arg ]
 115         elif hasattr(arg, 'create_stats'):
 116             arg.create_stats()
 117             self.stats = arg.stats
 118             arg.stats = {}
 119         if not self.stats:
 120             raise TypeError,  "Cannot create or construct a " \
 121                       + `self.__class__` \
 122                       + " object from '" + `arg` + "'"
 123         return
 124
 125     def get_top_level_stats(self):
 126         for func, (cc, nc, tt, ct, callers) in self.stats.items():
 127             self.total_calls += nc
 128             self.prim_calls  += cc
 129             self.total_tt    += tt
 130             if callers.has_key(("jprofile", 0, "profiler")):
 131                 self.top_level[func] = None
 132             if len(func_std_string(func)) > self.max_name_len:
 133                 self.max_name_len = len(func_std_string(func))
 134
 135     def add(self, *arg_list):
 136         if not arg_list: return self
 137         if len(arg_list) > 1: self.add(*arg_list[1:])
 138         other = arg_list[0]
 139         if type(self) != type(other) or self.__class__ != other.__class__:
 140             other = Stats(other)
 141         self.files += other.files
 142         self.total_calls += other.total_calls
 143         self.prim_calls += other.prim_calls
 144         self.total_tt += other.total_tt
 145         for func in other.top_level:
 146             self.top_level[func] = None
 147
 148         if self.max_name_len < other.max_name_len:
 149             self.max_name_len = other.max_name_len
 150
 151         self.fcn_list = None
 152
 153         for func, stat in other.stats.iteritems():
 154             if func in self.stats:
 155                 old_func_stat = self.stats[func]
 156             else:
 157                 old_func_stat = (0, 0, 0, 0, {},)
 158             self.stats[func] = add_func_stats(old_func_stat, stat)
 159         return self
 160
 161     def dump_stats(self, filename):
 162         """Write the profile data to a file we know how to load back."""
 163         f = file(filename, 'wb')
 164         try:
 165             marshal.dump(self.stats, f)
 166         finally:
 167             f.close()
 168
 169     # list the tuple indices and directions for sorting,
 170     # along with some printable description
 171     sort_arg_dict_default = {
 172               "calls"     : (((1,-1),              ), "call count"),
 173               "cumulative": (((3,-1),              ), "cumulative time"),
 174               "file"      : (((4, 1),              ), "file name"),
 175               "line"      : (((5, 1),              ), "line number"),
 176               "module"    : (((4, 1),              ), "file name"),
 177               "name"      : (((6, 1),              ), "function name"),
 178               "nfl"       : (((6, 1),(4, 1),(5, 1),), "name/file/line"),
 179               "pcalls"    : (((0,-1),              ), "call count"),
 180               "stdname"   : (((7, 1),              ), "standard name"),
 181               "time"      : (((2,-1),              ), "internal time"),
 182               }
 183
 184     def get_sort_arg_defs(self):
 185         """Expand all abbreviations that are unique."""
 186         if not self.sort_arg_dict:
 187             self.sort_arg_dict = dict = {}
 188             bad_list = {}
 189             for word, tup in self.sort_arg_dict_default.iteritems():
 190                 fragment = word
 191                 while fragment:
 192                     if not fragment:
 193                         break
 194                     if fragment in dict:
 195                         bad_list[fragment] = 0
 196                         break
 197                     dict[fragment] = tup
 198                     fragment = fragment[:-1]
 199             for word in bad_list:
 200                 del dict[word]
 201         return self.sort_arg_dict
 202
 203     def sort_stats(self, *field):
 204         if not field:
 205             self.fcn_list = 0
 206             return self
 207         if len(field) == 1 and type(field[0]) == type(1):
 208             # Be compatible with old profiler
 209             field = [ {-1: "stdname",
 210                       0:"calls",
 211                       1:"time",
 212                       2: "cumulative" }  [ field[0] ] ]
 213
 214         sort_arg_defs = self.get_sort_arg_defs()
 215         sort_tuple = ()
 216         self.sort_type = ""
 217         connector = ""
 218         for word in field:
 219             sort_tuple = sort_tuple + sort_arg_defs[word][0]
 220             self.sort_type += connector + sort_arg_defs[word][1]
 221             connector = ", "
 222
 223         stats_list = []
 224         for func, (cc, nc, tt, ct, callers) in self.stats.iteritems():
 225             stats_list.append((cc, nc, tt, ct) + func +
 226                               (func_std_string(func), func))
 227
 228         stats_list.sort(TupleComp(sort_tuple).compare)
 229
 230         self.fcn_list = fcn_list = []
 231         for tuple in stats_list:
 232             fcn_list.append(tuple[-1])
 233         return self
 234
 235     def reverse_order(self):
 236         if self.fcn_list:
 237             self.fcn_list.reverse()
 238         return self
 239
 240     def strip_dirs(self):
 241         oldstats = self.stats
 242         self.stats = newstats = {}
 243         max_name_len = 0
 244         for func, (cc, nc, tt, ct, callers) in oldstats.iteritems():
 245             newfunc = func_strip_path(func)
 246             if len(func_std_string(newfunc)) > max_name_len:
 247                 max_name_len = len(func_std_string(newfunc))
 248             newcallers = {}
 249             for func2, caller in callers.iteritems():
 250                 newcallers[func_strip_path(func2)] = caller
 251
 252             if newfunc in newstats:
 253                 newstats[newfunc] = add_func_stats(
 254                                         newstats[newfunc],
 255                                         (cc, nc, tt, ct, newcallers))
 256             else:
 257                 newstats[newfunc] = (cc, nc, tt, ct, newcallers)
 258         old_top = self.top_level
 259         self.top_level = new_top = {}
 260         for func in old_top:
 261             new_top[func_strip_path(func)] = None
 262
 263         self.max_name_len = max_name_len
 264
 265         self.fcn_list = None
 266         self.all_callees = None
 267         return self
 268
 269     def calc_callees(self):
 270         if self.all_callees: return
 271         self.all_callees = all_callees = {}
 272         for func, (cc, nc, tt, ct, callers) in self.stats.iteritems():
 273             if not func in all_callees:
 274                 all_callees[func] = {}
 275             for func2, caller in callers.iteritems():
 276                 if not func2 in all_callees:
 277                     all_callees[func2] = {}
 278                 all_callees[func2][func]  = caller
 279         return
 280
 281     #******************************************************************
 282     # The following functions support actual printing of reports
 283     #******************************************************************
 284
 285     # Optional "amount" is either a line count, or a percentage of lines.
 286
 287     def eval_print_amount(self, sel, list, msg):
 288         new_list = list
 289         if type(sel) == type(""):
 290             new_list = []
 291             for func in list:
 292                 if re.search(sel, func_std_string(func)):
 293                     new_list.append(func)
 294         else:
 295             count = len(list)
 296             if type(sel) == type(1.0) and 0.0 <= sel < 1.0:
 297                 count = int(count * sel + .5)
 298                 new_list = list[:count]
 299             elif type(sel) == type(1) and 0 <= sel < count:
 300                 count = sel
 301                 new_list = list[:count]
 302         if len(list) != len(new_list):
 303             msg = msg + "   List reduced from " + `len(list)` \
 304                       + " to " + `len(new_list)` + \
 305                       " due to restriction <" + `sel` + ">\n"
 306
 307         return new_list, msg
 308
 309     def get_print_list(self, sel_list):
 310         width = self.max_name_len
 311         if self.fcn_list:
 312             list = self.fcn_list[:]
 313             msg = "   Ordered by: " + self.sort_type + '\n'
 314         else:
 315             list = self.stats.keys()
 316             msg = "   Random listing order was used\n"
 317
 318         for selection in sel_list:
 319             list, msg = self.eval_print_amount(selection, list, msg)
 320
 321         count = len(list)
 322
 323         if not list:
 324             return 0, list
 325         print msg
 326         if count < len(self.stats):
 327             width = 0
 328             for func in list:
 329                 if  len(func_std_string(func)) > width:
 330                     width = len(func_std_string(func))
 331         return width+2, list
 332
 333     def print_stats(self, *amount):
 334         for filename in self.files:
 335             print filename
 336         if self.files: print
 337         indent = ' ' * 8
 338         for func in self.top_level:
 339             print indent, func_get_function_name(func)
 340
 341         print indent, self.total_calls, "function calls",
 342         if self.total_calls != self.prim_calls:
 343             print "(%d primitive calls)" % self.prim_calls,
 344         print "in %.3f CPU seconds" % self.total_tt
 345         print
 346         width, list = self.get_print_list(amount)
 347         if list:
 348             self.print_title()
 349             for func in list:
 350                 self.print_line(func)
 351             print
 352             print
 353         return self
 354
 355     def print_callees(self, *amount):
 356         width, list = self.get_print_list(amount)
 357         if list:
 358             self.calc_callees()
 359
 360             self.print_call_heading(width, "called...")
 361             for func in list:
 362                 if func in self.all_callees:
 363                     self.print_call_line(width, func, self.all_callees[func])
 364                 else:
 365                     self.print_call_line(width, func, {})
 366             print
 367             print
 368         return self
 369
 370     def print_callers(self, *amount):
 371         width, list = self.get_print_list(amount)
 372         if list:
 373             self.print_call_heading(width, "was called by...")
 374             for func in list:
 375                 cc, nc, tt, ct, callers = self.stats[func]
 376                 self.print_call_line(width, func, callers)
 377             print
 378             print
 379         return self
 380
 381     def print_call_heading(self, name_size, column_title):
 382         print "Function ".ljust(name_size) + column_title
 383
 384     def print_call_line(self, name_size, source, call_dict):
 385         print func_std_string(source).ljust(name_size),
 386         if not call_dict:
 387             print "--"
 388             return
 389         clist = call_dict.keys()
 390         clist.sort()
 391         name_size = name_size + 1
 392         indent = ""
 393         for func in clist:
 394             name = func_std_string(func)
 395             print indent*name_size + name + '(' \
 396                       + `call_dict[func]`+')', \
 397                       f8(self.stats[func][3])
 398             indent = " "
 399
 400     def print_title(self):
 401         print '   ncalls  tottime  percall  cumtime  percall', \
 402               'filename:lineno(function)'
 403
 404     def print_line(self, func):  # hack : should print percentages
 405         cc, nc, tt, ct, callers = self.stats[func]
 406         c = str(nc)
 407         if nc != cc:
 408             c = c + '/' + str(cc)
 409         print c.rjust(9),
 410         print f8(tt),
 411         if nc == 0:
 412             print ' '*8,
 413         else:
 414             print f8(tt/nc),
 415         print f8(ct),
 416         if cc == 0:
 417             print ' '*8,
 418         else:
 419             print f8(ct/cc),
 420         print func_std_string(func)
 421
 422     def ignore(self):
 423         # Deprecated since 1.5.1 -- see the docs.
 424         pass # has no return value, so use at end of line :-)
 425
 426 class TupleComp:
 427     """This class provides a generic function for comparing any two tuples.
 428     Each instance records a list of tuple-indices (from most significant
 429     to least significant), and sort direction (ascending or decending) for
 430     each tuple-index.  The compare functions can then be used as the function
 431     argument to the system sort() function when a list of tuples need to be
 432     sorted in the instances order."""
 433
 434     def __init__(self, comp_select_list):
 435         self.comp_select_list = comp_select_list
 436
 437     def compare (self, left, right):
 438         for index, direction in self.comp_select_list:
 439             l = left[index]
 440             r = right[index]
 441             if l < r:
 442                 return -direction
 443             if l > r:
 444                 return direction
 445         return 0
 446
 447 #**************************************************************************
 448 # func_name is a triple (file:string, line:int, name:string)
 449
 450 def func_strip_path(func_name):
 451     filename, line, name = func_name
 452     return os.path.basename(filename), line, name
 453
 454 def func_get_function_name(func):
 455     return func[2]
 456
 457 def func_std_string(func_name): # match what old profile produced
 458     return "%s:%d(%s)" % func_name
 459
 460 #**************************************************************************
 461 # The following functions combine statists for pairs functions.
 462 # The bulk of the processing involves correctly handling "call" lists,
 463 # such as callers and callees.
 464 #**************************************************************************
 465
 466 def add_func_stats(target, source):
 467     """Add together all the stats for two profile entries."""
 468     cc, nc, tt, ct, callers = source
 469     t_cc, t_nc, t_tt, t_ct, t_callers = target
 470     return (cc+t_cc, nc+t_nc, tt+t_tt, ct+t_ct,
 471               add_callers(t_callers, callers))
 472
 473 def add_callers(target, source):
 474     """Combine two caller lists in a single list."""
 475     new_callers = {}
 476     for func, caller in target.iteritems():
 477         new_callers[func] = caller
 478     for func, caller in source.iteritems():
 479         if func in new_callers:
 480             new_callers[func] = caller + new_callers[func]
 481         else:
 482             new_callers[func] = caller
 483     return new_callers
 484
 485 def count_calls(callers):
 486     """Sum the caller statistics to get total number of calls received."""
 487     nc = 0
 488     for calls in callers.itervalues():
 489         nc += calls
 490     return nc
 491
 492 #**************************************************************************
 493 # The following functions support printing of reports
 494 #**************************************************************************
 495
 496 def f8(x):
 497     return "%8.3f" % x
 498
 499 #**************************************************************************
 500 # Statistics browser added by ESR, April 2001
 501 #**************************************************************************
 502
 503 if __name__ == '__main__':
 504     import cmd
 505     try:
 506         import readline
 507     except ImportError:
 508         pass
 509
 510     class ProfileBrowser(cmd.Cmd):
 511         def __init__(self, profile=None):
 512             cmd.Cmd.__init__(self)
 513             self.prompt = "% "
 514             if profile is not None:
 515                 self.stats = Stats(profile)
 516             else:
 517                 self.stats = None
 518
 519         def generic(self, fn, line):
 520             args = line.split()
 521             processed = []
 522             for term in args:
 523                 try:
 524                     processed.append(int(term))
 525                     continue
 526                 except ValueError:
 527                     pass
 528                 try:
 529                     frac = float(term)
 530                     if frac > 1 or frac < 0:
 531                         print "Fraction argument mus be in [0, 1]"
 532                         continue
 533                     processed.append(frac)
 534                     continue
 535                 except ValueError:
 536                     pass
 537                 processed.append(term)
 538             if self.stats:
 539                 getattr(self.stats, fn)(*processed)
 540             else:
 541                 print "No statistics object is loaded."
 542             return 0
 543         def generic_help(self):
 544             print "Arguments may be:"
 545             print "* An integer maximum number of entries to print."
 546             print "* A decimal fractional number between 0 and 1, controlling"
 547             print "  what fraction of selected entries to print."
 548             print "* A regular expression; only entries with function names"
 549             print "  that match it are printed."
 550
 551         def do_add(self, line):
 552             self.stats.add(line)
 553             return 0
 554         def help_add(self):
 555             print "Add profile info from given file to current statistics object."
 556
 557         def do_callees(self, line):
 558             return self.generic('print_callees', line)
 559         def help_callees(self):
 560             print "Print callees statistics from the current stat object."
 561             self.generic_help()
 562
 563         def do_callers(self, line):
 564             return self.generic('print_callers', line)
 565         def help_callers(self):
 566             print "Print callers statistics from the current stat object."
 567             self.generic_help()
 568
 569         def do_EOF(self, line):
 570             print ""
 571             return 1
 572         def help_EOF(self):
 573             print "Leave the profile brower."
 574
 575         def do_quit(self, line):
 576             return 1
 577         def help_quit(self):
 578             print "Leave the profile brower."
 579
 580         def do_read(self, line):
 581             if line:
 582                 try:
 583                     self.stats = Stats(line)
 584                 except IOError, args:
 585                     print args[1]
 586                     return
 587                 self.prompt = line + "% "
 588             elif len(self.prompt) > 2:
 589                 line = self.prompt[-2:]
 590             else:
 591                 print "No statistics object is current -- cannot reload."
 592             return 0
 593         def help_read(self):
 594             print "Read in profile data from a specified file."
 595
 596         def do_reverse(self, line):
 597             self.stats.reverse_order()
 598             return 0
 599         def help_reverse(self):
 600             print "Reverse the sort order of the profiling report."
 601
 602         def do_sort(self, line):
 603             abbrevs = self.stats.get_sort_arg_defs()
 604             if line and not filter(lambda x,a=abbrevs: x not in a,line.split()):
 605                 self.stats.sort_stats(*line.split())
 606             else:
 607                 print "Valid sort keys (unique prefixes are accepted):"
 608                 for (key, value) in Stats.sort_arg_dict_default.iteritems():
 609                     print "%s -- %s" % (key, value[1])
 610             return 0
 611         def help_sort(self):
 612             print "Sort profile data according to specified keys."
 613             print "(Typing `sort' without arguments lists valid keys.)"
 614         def complete_sort(self, text, *args):
 615             return [a for a in Stats.sort_arg_dict_default if a.startswith(text)]
 616
 617         def do_stats(self, line):
 618             return self.generic('print_stats', line)
 619         def help_stats(self):
 620             print "Print statistics from the current stat object."
 621             self.generic_help()
 622
 623         def do_strip(self, line):
 624             self.stats.strip_dirs()
 625             return 0
 626         def help_strip(self):
 627             print "Strip leading path information from filenames in the report."
 628
 629         def postcmd(self, stop, line):
 630             if stop:
 631                 return stop
 632             return None
 633
 634     import sys
 635     print "Welcome to the profile statistics browser."
 636     if len(sys.argv) > 1:
 637         initprofile = sys.argv[1]
 638     else:
 639         initprofile = None
 640     try:
 641         ProfileBrowser(initprofile).cmdloop()
 642         print "Goodbye."
 643     except KeyboardInterrupt:
 644         pass
 645
 646 # That's all, folks.