Lib/pyclbr.py

   1 """Parse a Python file and retrieve classes and methods.
   2
   3 Parse enough of a Python file to recognize class and method
   4 definitions and to find out the superclasses of a class.
   5
   6 The interface consists of a single function:
   7         readmodule(module, path)
   8 module is the name of a Python module, path is an optional list of
   9 directories where the module is to be searched.  If present, path is
  10 prepended to the system search path sys.path.
  11 The return value is a dictionary.  The keys of the dictionary are
  12 the names of the classes defined in the module (including classes
  13 that are defined via the from XXX import YYY construct).  The values
  14 are class instances of the class Class defined here.
  15
  16 A class is described by the class Class in this module.  Instances
  17 of this class have the following instance variables:
  18         name -- the name of the class
  19         super -- a list of super classes (Class instances)
  20         methods -- a dictionary of methods
  21         file -- the file in which the class was defined
  22         lineno -- the line in the file on which the class statement occurred
  23 The dictionary of methods uses the method names as keys and the line
  24 numbers on which the method was defined as values.
  25 If the name of a super class is not recognized, the corresponding
  26 entry in the list of super classes is not a class instance but a
  27 string giving the name of the super class.  Since import statements
  28 are recognized and imported modules are scanned as well, this
  29 shouldn't happen often.
  30
  31 BUGS
  32 - Continuation lines are not dealt with at all.
  33 - While triple-quoted strings won't confuse it, lines that look like
  34   def, class, import or "from ... import" stmts inside backslash-continued
  35   single-quoted strings are treated like code.  The expense of stopping
  36   that isn't worth it.
  37 - Code that doesn't pass tabnanny or python -t will confuse it, unless
  38   you set the module TABWIDTH vrbl (default 8) to the correct tab width
  39   for the file.
  40
  41 PACKAGE RELATED BUGS
  42 - If you have a package and a module inside that or another package
  43   with the same name, module caching doesn't work properly since the
  44   key is the base name of the module/package.
  45 - The only entry that is returned when you readmodule a package is a
  46   __path__ whose value is a list which confuses certain class browsers.
  47 - When code does:
  48   from package import subpackage
  49   class MyClass(subpackage.SuperClass):
  50     ...
  51   It can't locate the parent.  It probably needs to have the same
  52   hairy logic that the import locator already does.  (This logic
  53   exists coded in Python in the freeze package.)
  54 """
  55
  56 import sys
  57 import imp
  58 import re
  59 import string
  60
  61 __all__ = ["readmodule"]
  62
  63 TABWIDTH = 8
  64
  65 _getnext = re.compile(r"""
  66     (?P<String>
  67        \""" [^"\\]* (?:
  68                         (?: \\. | "(?!"") )
  69                         [^"\\]*
  70                     )*
  71        \"""
  72
  73     |   ''' [^'\\]* (?:
  74                         (?: \\. | '(?!'') )
  75                         [^'\\]*
  76                     )*
  77         '''
  78     )
  79
  80 |   (?P<Method>
  81         ^
  82         (?P<MethodIndent> [ \t]* )
  83         def [ \t]+
  84         (?P<MethodName> [a-zA-Z_] \w* )
  85         [ \t]* \(
  86     )
  87
  88 |   (?P<Class>
  89         ^
  90         (?P<ClassIndent> [ \t]* )
  91         class [ \t]+
  92         (?P<ClassName> [a-zA-Z_] \w* )
  93         [ \t]*
  94         (?P<ClassSupers> \( [^)\n]* \) )?
  95         [ \t]* :
  96     )
  97
  98 |   (?P<Import>
  99         ^ import [ \t]+
 100         (?P<ImportList> [^#;\n]+ )
 101     )
 102
 103 |   (?P<ImportFrom>
 104         ^ from [ \t]+
 105         (?P<ImportFromPath>
 106             [a-zA-Z_] \w*
 107             (?:
 108                 [ \t]* \. [ \t]* [a-zA-Z_] \w*
 109             )*
 110         )
 111         [ \t]+
 112         import [ \t]+
 113         (?P<ImportFromList> [^#;\n]+ )
 114     )
 115 """, re.VERBOSE | re.DOTALL | re.MULTILINE).search
 116
 117 _modules = {}                           # cache of modules we've seen
 118
 119 # each Python class is represented by an instance of this class
 120 class Class:
 121     '''Class to represent a Python class.'''
 122     def __init__(self, module, name, super, file, lineno):
 123         self.module = module
 124         self.name = name
 125         if super is None:
 126             super = []
 127         self.super = super
 128         self.methods = {}
 129         self.file = file
 130         self.lineno = lineno
 131
 132     def _addmethod(self, name, lineno):
 133         self.methods[name] = lineno
 134
 135 class Function(Class):
 136     '''Class to represent a top-level Python function'''
 137     def __init__(self, module, name, file, lineno):
 138         Class.__init__(self, module, name, None, file, lineno)
 139     def _addmethod(self, name, lineno):
 140         assert 0, "Function._addmethod() shouldn't be called"
 141
 142 def readmodule(module, path=[], inpackage=0):
 143     '''Backwards compatible interface.
 144
 145     Like readmodule_ex() but strips Function objects from the
 146     resulting dictionary.'''
 147
 148     dict = readmodule_ex(module, path, inpackage)
 149     res = {}
 150     for key, value in dict.items():
 151         if not isinstance(value, Function):
 152             res[key] = value
 153     return res
 154
 155 def readmodule_ex(module, path=[], inpackage=0):
 156     '''Read a module file and return a dictionary of classes.
 157
 158     Search for MODULE in PATH and sys.path, read and parse the
 159     module and return a dictionary with one entry for each class
 160     found in the module.'''
 161
 162     dict = {}
 163
 164     i = module.rfind('.')
 165     if i >= 0:
 166         # Dotted module name
 167         package = module[:i].strip()
 168         submodule = module[i+1:].strip()
 169         parent = readmodule_ex(package, path, inpackage)
 170         child = readmodule_ex(submodule, parent['__path__'], 1)
 171         return child
 172
 173     if _modules.has_key(module):
 174         # we've seen this module before...
 175         return _modules[module]
 176     if module in sys.builtin_module_names:
 177         # this is a built-in module
 178         _modules[module] = dict
 179         return dict
 180
 181     # search the path for the module
 182     f = None
 183     if inpackage:
 184         try:
 185             f, file, (suff, mode, type) = \
 186                     imp.find_module(module, path)
 187         except ImportError:
 188             f = None
 189     if f is None:
 190         fullpath = list(path) + sys.path
 191         f, file, (suff, mode, type) = imp.find_module(module, fullpath)
 192     if type == imp.PKG_DIRECTORY:
 193         dict['__path__'] = [file]
 194         _modules[module] = dict
 195         path = [file] + path
 196         f, file, (suff, mode, type) = \
 197                         imp.find_module('__init__', [file])
 198     if type != imp.PY_SOURCE:
 199         # not Python source, can't do anything with this module
 200         f.close()
 201         _modules[module] = dict
 202         return dict
 203
 204     _modules[module] = dict
 205     classstack = [] # stack of (class, indent) pairs
 206     src = f.read()
 207     f.close()
 208
 209     # To avoid having to stop the regexp at each newline, instead
 210     # when we need a line number we simply string.count the number of
 211     # newlines in the string since the last time we did this; i.e.,
 212     #    lineno = lineno + \
 213     #             string.count(src, '\n', last_lineno_pos, here)
 214     #    last_lineno_pos = here
 215     countnl = string.count
 216     lineno, last_lineno_pos = 1, 0
 217     i = 0
 218     while 1:
 219         m = _getnext(src, i)
 220         if not m:
 221             break
 222         start, i = m.span()
 223
 224         if m.start("Method") >= 0:
 225             # found a method definition or function
 226             thisindent = _indent(m.group("MethodIndent"))
 227             meth_name = m.group("MethodName")
 228             lineno = lineno + \
 229                      countnl(src, '\n',
 230                              last_lineno_pos, start)
 231             last_lineno_pos = start
 232             # close all classes indented at least as much
 233             while classstack and \
 234                   classstack[-1][1] >= thisindent:
 235                 del classstack[-1]
 236             if classstack:
 237                 # it's a class method
 238                 cur_class = classstack[-1][0]
 239                 cur_class._addmethod(meth_name, lineno)
 240             else:
 241                 # it's a function
 242                 f = Function(module, meth_name,
 243                              file, lineno)
 244                 dict[meth_name] = f
 245
 246         elif m.start("String") >= 0:
 247             pass
 248
 249         elif m.start("Class") >= 0:
 250             # we found a class definition
 251             thisindent = _indent(m.group("ClassIndent"))
 252             # close all classes indented at least as much
 253             while classstack and \
 254                   classstack[-1][1] >= thisindent:
 255                 del classstack[-1]
 256             lineno = lineno + \
 257                      countnl(src, '\n', last_lineno_pos, start)
 258             last_lineno_pos = start
 259             class_name = m.group("ClassName")
 260             inherit = m.group("ClassSupers")
 261             if inherit:
 262                 # the class inherits from other classes
 263                 inherit = inherit[1:-1].strip()
 264                 names = []
 265                 for n in inherit.split(','):
 266                     n = n.strip()
 267                     if dict.has_key(n):
 268                         # we know this super class
 269                         n = dict[n]
 270                     else:
 271                         c = n.split('.')
 272                         if len(c) > 1:
 273                             # super class
 274                             # is of the
 275                             # form module.class:
 276                             # look in
 277                             # module for class
 278                             m = c[-2]
 279                             c = c[-1]
 280                             if _modules.has_key(m):
 281                                 d = _modules[m]
 282                                 if d.has_key(c):
 283                                     n = d[c]
 284                     names.append(n)
 285                 inherit = names
 286             # remember this class
 287             cur_class = Class(module, class_name, inherit,
 288                               file, lineno)
 289             dict[class_name] = cur_class
 290             classstack.append((cur_class, thisindent))
 291
 292         elif m.start("Import") >= 0:
 293             # import module
 294             for n in m.group("ImportList").split(','):
 295                 n = n.strip()
 296                 try:
 297                     # recursively read the imported module
 298                     d = readmodule_ex(n, path, inpackage)
 299                 except:
 300                     ##print 'module', n, 'not found'
 301                     pass
 302
 303         elif m.start("ImportFrom") >= 0:
 304             # from module import stuff
 305             mod = m.group("ImportFromPath")
 306             names = m.group("ImportFromList").split(',')
 307             try:
 308                 # recursively read the imported module
 309                 d = readmodule_ex(mod, path, inpackage)
 310             except:
 311                 ##print 'module', mod, 'not found'
 312                 continue
 313             # add any classes that were defined in the
 314             # imported module to our name space if they
 315             # were mentioned in the list
 316             for n in names:
 317                 n = n.strip()
 318                 if d.has_key(n):
 319                     dict[n] = d[n]
 320                 elif n == '*':
 321                     # only add a name if not
 322                     # already there (to mimic what
 323                     # Python does internally)
 324                     # also don't add names that
 325                     # start with _
 326                     for n in d.keys():
 327                         if n[0] != '_' and \
 328                            not dict.has_key(n):
 329                             dict[n] = d[n]
 330         else:
 331             assert 0, "regexp _getnext found something unexpected"
 332
 333     return dict
 334
 335 def _indent(ws, _expandtabs=string.expandtabs):
 336     return len(_expandtabs(ws, TABWIDTH))