Lib/pyclbr.py

   1 """Parse a Python file and retrieve classes and methods.
   2
   3 Parse enough of a Python file to recognize class and method
   4 definitions and to find out the superclasses of a class.
   5
   6 The interface consists of a single function:
   7         readmodule(module, path)
   8 module is the name of a Python module, path is an optional list of
   9 directories where the module is to be searched.  If present, path is
  10 prepended to the system search path sys.path.
  11 The return value is a dictionary.  The keys of the dictionary are
  12 the names of the classes defined in the module (including classes
  13 that are defined via the from XXX import YYY construct).  The values
  14 are class instances of the class Class defined here.
  15
  16 A class is described by the class Class in this module.  Instances
  17 of this class have the following instance variables:
  18         name -- the name of the class
  19         super -- a list of super classes (Class instances)
  20         methods -- a dictionary of methods
  21         file -- the file in which the class was defined
  22         lineno -- the line in the file on which the class statement occurred
  23 The dictionary of methods uses the method names as keys and the line
  24 numbers on which the method was defined as values.
  25 If the name of a super class is not recognized, the corresponding
  26 entry in the list of super classes is not a class instance but a
  27 string giving the name of the super class.  Since import statements
  28 are recognized and imported modules are scanned as well, this
  29 shouldn't happen often.
  30
  31 BUGS
  32 - Continuation lines are not dealt with at all, except inside strings.
  33 - Nested classes and functions can confuse it.
  34 - Code that doesn't pass tabnanny or python -t will confuse it, unless
  35   you set the module TABWIDTH vrbl (default 8) to the correct tab width
  36   for the file.
  37
  38 PACKAGE RELATED BUGS
  39 - If you have a package and a module inside that or another package
  40   with the same name, module caching doesn't work properly since the
  41   key is the base name of the module/package.
  42 - The only entry that is returned when you readmodule a package is a
  43   __path__ whose value is a list which confuses certain class browsers.
  44 - When code does:
  45   from package import subpackage
  46   class MyClass(subpackage.SuperClass):
  47     ...
  48   It can't locate the parent.  It probably needs to have the same
  49   hairy logic that the import locator already does.  (This logic
  50   exists coded in Python in the freeze package.)
  51 """
  52
  53 import sys
  54 import imp
  55 import re
  56 import string
  57
  58 __all__ = ["readmodule"]
  59
  60 TABWIDTH = 8
  61
  62 _getnext = re.compile(r"""
  63     (?P<String>
  64        \""" [^"\\]* (?:
  65                         (?: \\. | "(?!"") )
  66                         [^"\\]*
  67                     )*
  68        \"""
  69
  70     |   ''' [^'\\]* (?:
  71                         (?: \\. | '(?!'') )
  72                         [^'\\]*
  73                     )*
  74         '''
  75
  76     |   " [^"\\\n]* (?: \\. [^"\\\n]*)* "
  77
  78     |   ' [^'\\\n]* (?: \\. [^'\\\n]*)* '
  79     )
  80
  81 |   (?P<Method>
  82         ^
  83         (?P<MethodIndent> [ \t]* )
  84         def [ \t]+
  85         (?P<MethodName> [a-zA-Z_] \w* )
  86         [ \t]* \(
  87     )
  88
  89 |   (?P<Class>
  90         ^
  91         (?P<ClassIndent> [ \t]* )
  92         class [ \t]+
  93         (?P<ClassName> [a-zA-Z_] \w* )
  94         [ \t]*
  95         (?P<ClassSupers> \( [^)\n]* \) )?
  96         [ \t]* :
  97     )
  98
  99 |   (?P<Import>
 100         ^ import [ \t]+
 101         (?P<ImportList> [^#;\n]+ )
 102     )
 103
 104 |   (?P<ImportFrom>
 105         ^ from [ \t]+
 106         (?P<ImportFromPath>
 107             [a-zA-Z_] \w*
 108             (?:
 109                 [ \t]* \. [ \t]* [a-zA-Z_] \w*
 110             )*
 111         )
 112         [ \t]+
 113         import [ \t]+
 114         (?P<ImportFromList> [^#;\n]+ )
 115     )
 116 """, re.VERBOSE | re.DOTALL | re.MULTILINE).search
 117
 118 _modules = {}                           # cache of modules we've seen
 119
 120 # each Python class is represented by an instance of this class
 121 class Class:
 122     '''Class to represent a Python class.'''
 123     def __init__(self, module, name, super, file, lineno):
 124         self.module = module
 125         self.name = name
 126         if super is None:
 127             super = []
 128         self.super = super
 129         self.methods = {}
 130         self.file = file
 131         self.lineno = lineno
 132
 133     def _addmethod(self, name, lineno):
 134         self.methods[name] = lineno
 135
 136 class Function(Class):
 137     '''Class to represent a top-level Python function'''
 138     def __init__(self, module, name, file, lineno):
 139         Class.__init__(self, module, name, None, file, lineno)
 140     def _addmethod(self, name, lineno):
 141         assert 0, "Function._addmethod() shouldn't be called"
 142
 143 def readmodule(module, path=[], inpackage=0):
 144     '''Backwards compatible interface.
 145
 146     Like readmodule_ex() but strips Function objects from the
 147     resulting dictionary.'''
 148
 149     dict = readmodule_ex(module, path, inpackage)
 150     res = {}
 151     for key, value in dict.items():
 152         if not isinstance(value, Function):
 153             res[key] = value
 154     return res
 155
 156 def readmodule_ex(module, path=[], inpackage=0):
 157     '''Read a module file and return a dictionary of classes.
 158
 159     Search for MODULE in PATH and sys.path, read and parse the
 160     module and return a dictionary with one entry for each class
 161     found in the module.'''
 162
 163     dict = {}
 164
 165     i = module.rfind('.')
 166     if i >= 0:
 167         # Dotted module name
 168         package = module[:i].strip()
 169         submodule = module[i+1:].strip()
 170         parent = readmodule_ex(package, path, inpackage)
 171         child = readmodule_ex(submodule, parent['__path__'], 1)
 172         return child
 173
 174     if _modules.has_key(module):
 175         # we've seen this module before...
 176         return _modules[module]
 177     if module in sys.builtin_module_names:
 178         # this is a built-in module
 179         _modules[module] = dict
 180         return dict
 181
 182     # search the path for the module
 183     f = None
 184     if inpackage:
 185         try:
 186             f, file, (suff, mode, type) = \
 187                     imp.find_module(module, path)
 188         except ImportError:
 189             f = None
 190     if f is None:
 191         fullpath = list(path) + sys.path
 192         f, file, (suff, mode, type) = imp.find_module(module, fullpath)
 193     if type == imp.PKG_DIRECTORY:
 194         dict['__path__'] = [file]
 195         _modules[module] = dict
 196         path = [file] + path
 197         f, file, (suff, mode, type) = \
 198                         imp.find_module('__init__', [file])
 199     if type != imp.PY_SOURCE:
 200         # not Python source, can't do anything with this module
 201         f.close()
 202         _modules[module] = dict
 203         return dict
 204
 205     _modules[module] = dict
 206     classstack = [] # stack of (class, indent) pairs
 207     src = f.read()
 208     f.close()
 209
 210     # To avoid having to stop the regexp at each newline, instead
 211     # when we need a line number we simply string.count the number of
 212     # newlines in the string since the last time we did this; i.e.,
 213     #    lineno = lineno + \
 214     #             string.count(src, '\n', last_lineno_pos, here)
 215     #    last_lineno_pos = here
 216     countnl = string.count
 217     lineno, last_lineno_pos = 1, 0
 218     i = 0
 219     while 1:
 220         m = _getnext(src, i)
 221         if not m:
 222             break
 223         start, i = m.span()
 224
 225         if m.start("Method") >= 0:
 226             # found a method definition or function
 227             thisindent = _indent(m.group("MethodIndent"))
 228             meth_name = m.group("MethodName")
 229             lineno = lineno + \
 230                      countnl(src, '\n',
 231                              last_lineno_pos, start)
 232             last_lineno_pos = start
 233             # close all classes indented at least as much
 234             while classstack and \
 235                   classstack[-1][1] >= thisindent:
 236                 del classstack[-1]
 237             if classstack:
 238                 # it's a class method
 239                 cur_class = classstack[-1][0]
 240                 cur_class._addmethod(meth_name, lineno)
 241             else:
 242                 # it's a function
 243                 f = Function(module, meth_name,
 244                              file, lineno)
 245                 dict[meth_name] = f
 246
 247         elif m.start("String") >= 0:
 248             pass
 249
 250         elif m.start("Class") >= 0:
 251             # we found a class definition
 252             thisindent = _indent(m.group("ClassIndent"))
 253             # close all classes indented at least as much
 254             while classstack and \
 255                   classstack[-1][1] >= thisindent:
 256                 del classstack[-1]
 257             lineno = lineno + \
 258                      countnl(src, '\n', last_lineno_pos, start)
 259             last_lineno_pos = start
 260             class_name = m.group("ClassName")
 261             inherit = m.group("ClassSupers")
 262             if inherit:
 263                 # the class inherits from other classes
 264                 inherit = inherit[1:-1].strip()
 265                 names = []
 266                 for n in inherit.split(','):
 267                     n = n.strip()
 268                     if dict.has_key(n):
 269                         # we know this super class
 270                         n = dict[n]
 271                     else:
 272                         c = n.split('.')
 273                         if len(c) > 1:
 274                             # super class
 275                             # is of the
 276                             # form module.class:
 277                             # look in
 278                             # module for class
 279                             m = c[-2]
 280                             c = c[-1]
 281                             if _modules.has_key(m):
 282                                 d = _modules[m]
 283                                 if d.has_key(c):
 284                                     n = d[c]
 285                     names.append(n)
 286                 inherit = names
 287             # remember this class
 288             cur_class = Class(module, class_name, inherit,
 289                               file, lineno)
 290             dict[class_name] = cur_class
 291             classstack.append((cur_class, thisindent))
 292
 293         elif m.start("Import") >= 0:
 294             # import module
 295             for n in m.group("ImportList").split(','):
 296                 n = n.strip()
 297                 try:
 298                     # recursively read the imported module
 299                     d = readmodule_ex(n, path, inpackage)
 300                 except:
 301                     ##print 'module', n, 'not found'
 302                     pass
 303
 304         elif m.start("ImportFrom") >= 0:
 305             # from module import stuff
 306             mod = m.group("ImportFromPath")
 307             names = m.group("ImportFromList").split(',')
 308             try:
 309                 # recursively read the imported module
 310                 d = readmodule_ex(mod, path, inpackage)
 311             except:
 312                 ##print 'module', mod, 'not found'
 313                 continue
 314             # add any classes that were defined in the
 315             # imported module to our name space if they
 316             # were mentioned in the list
 317             for n in names:
 318                 n = n.strip()
 319                 if d.has_key(n):
 320                     dict[n] = d[n]
 321                 elif n == '*':
 322                     # only add a name if not
 323                     # already there (to mimic what
 324                     # Python does internally)
 325                     # also don't add names that
 326                     # start with _
 327                     for n in d.keys():
 328                         if n[0] != '_' and \
 329                            not dict.has_key(n):
 330                             dict[n] = d[n]
 331         else:
 332             assert 0, "regexp _getnext found something unexpected"
 333
 334     return dict
 335
 336 def _indent(ws, _expandtabs=string.expandtabs):
 337     return len(_expandtabs(ws, TABWIDTH))