Lib/pyclbr.py

   1 """Parse a Python file and retrieve classes and methods.
   2
   3 Parse enough of a Python file to recognize class and method
   4 definitions and to find out the superclasses of a class.
   5
   6 The interface consists of a single function:
   7         readmodule(module, path)
   8 module is the name of a Python module, path is an optional list of
   9 directories where the module is to be searched.  If present, path is
  10 prepended to the system search path sys.path.
  11 The return value is a dictionary.  The keys of the dictionary are
  12 the names of the classes defined in the module (including classes
  13 that are defined via the from XXX import YYY construct).  The values
  14 are class instances of the class Class defined here.
  15
  16 A class is described by the class Class in this module.  Instances
  17 of this class have the following instance variables:
  18         name -- the name of the class
  19         super -- a list of super classes (Class instances)
  20         methods -- a dictionary of methods
  21         file -- the file in which the class was defined
  22         lineno -- the line in the file on which the class statement occurred
  23 The dictionary of methods uses the method names as keys and the line
  24 numbers on which the method was defined as values.
  25 If the name of a super class is not recognized, the corresponding
  26 entry in the list of super classes is not a class instance but a
  27 string giving the name of the super class.  Since import statements
  28 are recognized and imported modules are scanned as well, this
  29 shouldn't happen often.
  30
  31 BUGS
  32 - Continuation lines are not dealt with at all.
  33 - While triple-quoted strings won't confuse it, lines that look like
  34   def, class, import or "from ... import" stmts inside backslash-continued
  35   single-quoted strings are treated like code.  The expense of stopping
  36   that isn't worth it.
  37 - Code that doesn't pass tabnanny or python -t will confuse it, unless
  38   you set the module TABWIDTH vrbl (default 8) to the correct tab width
  39   for the file.
  40
  41 PACKAGE RELATED BUGS
  42 - If you have a package and a module inside that or another package
  43   with the same name, module caching doesn't work properly since the
  44   key is the base name of the module/package.
  45 - The only entry that is returned when you readmodule a package is a
  46   __path__ whose value is a list which confuses certain class browsers.
  47 - When code does:
  48   from package import subpackage
  49   class MyClass(subpackage.SuperClass):
  50     ...
  51   It can't locate the parent.  It probably needs to have the same
  52   hairy logic that the import locator already does.  (This logic
  53   exists coded in Python in the freeze package.)
  54 """
  55
  56 import os
  57 import sys
  58 import imp
  59 import re
  60 import string
  61
  62 __all__ = ["readmodule"]
  63
  64 TABWIDTH = 8
  65
  66 _getnext = re.compile(r"""
  67     (?P<String>
  68        \""" [^"\\]* (?:
  69                         (?: \\. | "(?!"") )
  70                         [^"\\]*
  71                     )*
  72        \"""
  73
  74     |   ''' [^'\\]* (?:
  75                         (?: \\. | '(?!'') )
  76                         [^'\\]*
  77                     )*
  78         '''
  79     )
  80
  81 |   (?P<Method>
  82         ^
  83         (?P<MethodIndent> [ \t]* )
  84         def [ \t]+
  85         (?P<MethodName> [a-zA-Z_] \w* )
  86         [ \t]* \(
  87     )
  88
  89 |   (?P<Class>
  90         ^
  91         (?P<ClassIndent> [ \t]* )
  92         class [ \t]+
  93         (?P<ClassName> [a-zA-Z_] \w* )
  94         [ \t]*
  95         (?P<ClassSupers> \( [^)\n]* \) )?
  96         [ \t]* :
  97     )
  98
  99 |   (?P<Import>
 100         ^ import [ \t]+
 101         (?P<ImportList> [^#;\n]+ )
 102     )
 103
 104 |   (?P<ImportFrom>
 105         ^ from [ \t]+
 106         (?P<ImportFromPath>
 107             [a-zA-Z_] \w*
 108             (?:
 109                 [ \t]* \. [ \t]* [a-zA-Z_] \w*
 110             )*
 111         )
 112         [ \t]+
 113         import [ \t]+
 114         (?P<ImportFromList> [^#;\n]+ )
 115     )
 116 """, re.VERBOSE | re.DOTALL | re.MULTILINE).search
 117
 118 _modules = {}                           # cache of modules we've seen
 119
 120 # each Python class is represented by an instance of this class
 121 class Class:
 122     '''Class to represent a Python class.'''
 123     def __init__(self, module, name, super, file, lineno):
 124         self.module = module
 125         self.name = name
 126         if super is None:
 127             super = []
 128         self.super = super
 129         self.methods = {}
 130         self.file = file
 131         self.lineno = lineno
 132
 133     def _addmethod(self, name, lineno):
 134         self.methods[name] = lineno
 135
 136 class Function(Class):
 137     '''Class to represent a top-level Python function'''
 138     def __init__(self, module, name, file, lineno):
 139         Class.__init__(self, module, name, None, file, lineno)
 140     def _addmethod(self, name, lineno):
 141         assert 0, "Function._addmethod() shouldn't be called"
 142
 143 def readmodule(module, path=[], inpackage=0):
 144     '''Backwards compatible interface.
 145
 146     Like readmodule_ex() but strips Function objects from the
 147     resulting dictionary.'''
 148
 149     dict = readmodule_ex(module, path, inpackage)
 150     res = {}
 151     for key, value in dict.items():
 152         if not isinstance(value, Function):
 153             res[key] = value
 154     return res
 155
 156 def readmodule_ex(module, path=[], inpackage=0):
 157     '''Read a module file and return a dictionary of classes.
 158
 159     Search for MODULE in PATH and sys.path, read and parse the
 160     module and return a dictionary with one entry for each class
 161     found in the module.'''
 162
 163     dict = {}
 164
 165     i = module.rfind('.')
 166     if i >= 0:
 167         # Dotted module name
 168         package = module[:i].strip()
 169         submodule = module[i+1:].strip()
 170         parent = readmodule(package, path, inpackage)
 171         child = readmodule(submodule, parent['__path__'], 1)
 172         return child
 173
 174     if _modules.has_key(module):
 175         # we've seen this module before...
 176         return _modules[module]
 177     if module in sys.builtin_module_names:
 178         # this is a built-in module
 179         _modules[module] = dict
 180         return dict
 181
 182     # search the path for the module
 183     f = None
 184     if inpackage:
 185         try:
 186             f, file, (suff, mode, type) = \
 187                     imp.find_module(module, path)
 188         except ImportError:
 189             f = None
 190     if f is None:
 191         fullpath = list(path) + sys.path
 192         f, file, (suff, mode, type) = imp.find_module(module, fullpath)
 193     if type == imp.PKG_DIRECTORY:
 194         dict['__path__'] = [file]
 195         _modules[module] = dict
 196         path = [file] + path
 197         f, file, (suff, mode, type) = \
 198                         imp.find_module('__init__', [file])
 199     if type != imp.PY_SOURCE:
 200         # not Python source, can't do anything with this module
 201         f.close()
 202         _modules[module] = dict
 203         return dict
 204
 205     _modules[module] = dict
 206     imports = []
 207     classstack = [] # stack of (class, indent) pairs
 208     src = f.read()
 209     f.close()
 210
 211     # To avoid having to stop the regexp at each newline, instead
 212     # when we need a line number we simply string.count the number of
 213     # newlines in the string since the last time we did this; i.e.,
 214     #    lineno = lineno + \
 215     #             string.count(src, '\n', last_lineno_pos, here)
 216     #    last_lineno_pos = here
 217     countnl = string.count
 218     lineno, last_lineno_pos = 1, 0
 219     i = 0
 220     while 1:
 221         m = _getnext(src, i)
 222         if not m:
 223             break
 224         start, i = m.span()
 225
 226         if m.start("Method") >= 0:
 227             # found a method definition or function
 228             thisindent = _indent(m.group("MethodIndent"))
 229             meth_name = m.group("MethodName")
 230             lineno = lineno + \
 231                      countnl(src, '\n',
 232                              last_lineno_pos, start)
 233             last_lineno_pos = start
 234             # close all classes indented at least as much
 235             while classstack and \
 236                   classstack[-1][1] >= thisindent:
 237                 del classstack[-1]
 238             if classstack:
 239                 # it's a class method
 240                 cur_class = classstack[-1][0]
 241                 cur_class._addmethod(meth_name, lineno)
 242             else:
 243                 # it's a function
 244                 f = Function(module, meth_name,
 245                              file, lineno)
 246                 dict[meth_name] = f
 247
 248         elif m.start("String") >= 0:
 249             pass
 250
 251         elif m.start("Class") >= 0:
 252             # we found a class definition
 253             thisindent = _indent(m.group("ClassIndent"))
 254             # close all classes indented at least as much
 255             while classstack and \
 256                   classstack[-1][1] >= thisindent:
 257                 del classstack[-1]
 258             lineno = lineno + \
 259                      countnl(src, '\n', last_lineno_pos, start)
 260             last_lineno_pos = start
 261             class_name = m.group("ClassName")
 262             inherit = m.group("ClassSupers")
 263             if inherit:
 264                 # the class inherits from other classes
 265                 inherit = inherit[1:-1].strip()
 266                 names = []
 267                 for n in inherit.split(','):
 268                     n = n.strip()
 269                     if dict.has_key(n):
 270                         # we know this super class
 271                         n = dict[n]
 272                     else:
 273                         c = n.split('.')
 274                         if len(c) > 1:
 275                             # super class
 276                             # is of the
 277                             # form module.class:
 278                             # look in
 279                             # module for class
 280                             m = c[-2]
 281                             c = c[-1]
 282                             if _modules.has_key(m):
 283                                 d = _modules[m]
 284                                 if d.has_key(c):
 285                                     n = d[c]
 286                     names.append(n)
 287                 inherit = names
 288             # remember this class
 289             cur_class = Class(module, class_name, inherit,
 290                               file, lineno)
 291             dict[class_name] = cur_class
 292             classstack.append((cur_class, thisindent))
 293
 294         elif m.start("Import") >= 0:
 295             # import module
 296             for n in m.group("ImportList").split(','):
 297                 n = n.strip()
 298                 try:
 299                     # recursively read the imported module
 300                     d = readmodule(n, path, inpackage)
 301                 except:
 302                     ##print 'module', n, 'not found'
 303                     pass
 304
 305         elif m.start("ImportFrom") >= 0:
 306             # from module import stuff
 307             mod = m.group("ImportFromPath")
 308             names = m.group("ImportFromList").split(',')
 309             try:
 310                 # recursively read the imported module
 311                 d = readmodule(mod, path, inpackage)
 312             except:
 313                 ##print 'module', mod, 'not found'
 314                 continue
 315             # add any classes that were defined in the
 316             # imported module to our name space if they
 317             # were mentioned in the list
 318             for n in names:
 319                 n = n.strip()
 320                 if d.has_key(n):
 321                     dict[n] = d[n]
 322                 elif n == '*':
 323                     # only add a name if not
 324                     # already there (to mimic what
 325                     # Python does internally)
 326                     # also don't add names that
 327                     # start with _
 328                     for n in d.keys():
 329                         if n[0] != '_' and \
 330                            not dict.has_key(n):
 331                             dict[n] = d[n]
 332         else:
 333             assert 0, "regexp _getnext found something unexpected"
 334
 335     return dict
 336
 337 def _indent(ws, _expandtabs=string.expandtabs):
 338     return len(_expandtabs(ws, TABWIDTH))