Lib/idlelib/pyclbr.py

   1 """Parse a Python file and retrieve classes and methods.
   2
   3 Parse enough of a Python file to recognize class and method
   4 definitions and to find out the superclasses of a class.
   5
   6 The interface consists of a single function:
   7         readmodule(module, path)
   8 module is the name of a Python module, path is an optional list of
   9 directories where the module is to be searched.  If present, path is
  10 prepended to the system search path sys.path.
  11 The return value is a dictionary.  The keys of the dictionary are
  12 the names of the classes defined in the module (including classes
  13 that are defined via the from XXX import YYY construct).  The values
  14 are class instances of the class Class defined here.
  15
  16 A class is described by the class Class in this module.  Instances
  17 of this class have the following instance variables:
  18         name -- the name of the class
  19         super -- a list of super classes (Class instances)
  20         methods -- a dictionary of methods
  21         file -- the file in which the class was defined
  22         lineno -- the line in the file on which the class statement occurred
  23 The dictionary of methods uses the method names as keys and the line
  24 numbers on which the method was defined as values.
  25 If the name of a super class is not recognized, the corresponding
  26 entry in the list of super classes is not a class instance but a
  27 string giving the name of the super class.  Since import statements
  28 are recognized and imported modules are scanned as well, this
  29 shouldn't happen often.
  30
  31 BUGS
  32 - Continuation lines are not dealt with at all.
  33 - While triple-quoted strings won't confuse it, lines that look like
  34   def, class, import or "from ... import" stmts inside backslash-continued
  35   single-quoted strings are treated like code.  The expense of stopping
  36   that isn't worth it.
  37 - Code that doesn't pass tabnanny or python -t will confuse it, unless
  38   you set the module TABWIDTH vrbl (default 8) to the correct tab width
  39   for the file.
  40
  41 PACKAGE RELATED BUGS
  42 - If you have a package and a module inside that or another package
  43   with the same name, module caching doesn't work properly since the
  44   key is the base name of the module/package.
  45 - The only entry that is returned when you readmodule a package is a
  46   __path__ whose value is a list which confuses certain class browsers.
  47 - When code does:
  48   from package import subpackage
  49   class MyClass(subpackage.SuperClass):
  50     ...
  51   It can't locate the parent.  It probably needs to have the same
  52   hairy logic that the import locator already does.  (This logic
  53   exists coded in Python in the freeze package.)
  54 """
  55
  56 import os
  57 import sys
  58 import imp
  59 import re
  60 import string
  61
  62 TABWIDTH = 8
  63
  64 _getnext = re.compile(r"""
  65     (?P<String>
  66        \""" [^"\\]* (?:
  67                         (?: \\. | "(?!"") )
  68                         [^"\\]*
  69                     )*
  70        \"""
  71
  72     |   ''' [^'\\]* (?:
  73                         (?: \\. | '(?!'') )
  74                         [^'\\]*
  75                     )*
  76         '''
  77     )
  78
  79 |   (?P<Method>
  80         ^
  81         (?P<MethodIndent> [ \t]* )
  82         def [ \t]+
  83         (?P<MethodName> [a-zA-Z_] \w* )
  84         [ \t]* \(
  85     )
  86
  87 |   (?P<Class>
  88         ^
  89         (?P<ClassIndent> [ \t]* )
  90         class [ \t]+
  91         (?P<ClassName> [a-zA-Z_] \w* )
  92         [ \t]*
  93         (?P<ClassSupers> \( [^)\n]* \) )?
  94         [ \t]* :
  95     )
  96
  97 |   (?P<Import>
  98         ^ import [ \t]+
  99         (?P<ImportList> [^#;\n]+ )
 100     )
 101
 102 |   (?P<ImportFrom>
 103         ^ from [ \t]+
 104         (?P<ImportFromPath>
 105             [a-zA-Z_] \w*
 106             (?:
 107                 [ \t]* \. [ \t]* [a-zA-Z_] \w*
 108             )*
 109         )
 110         [ \t]+
 111         import [ \t]+
 112         (?P<ImportFromList> [^#;\n]+ )
 113     )
 114 """, re.VERBOSE | re.DOTALL | re.MULTILINE).search
 115
 116 _modules = {}                           # cache of modules we've seen
 117
 118 # each Python class is represented by an instance of this class
 119 class Class:
 120         '''Class to represent a Python class.'''
 121         def __init__(self, module, name, super, file, lineno):
 122                 self.module = module
 123                 self.name = name
 124                 if super is None:
 125                         super = []
 126                 self.super = super
 127                 self.methods = {}
 128                 self.file = file
 129                 self.lineno = lineno
 130
 131         def _addmethod(self, name, lineno):
 132                 self.methods[name] = lineno
 133
 134 class Function(Class):
 135         '''Class to represent a top-level Python function'''
 136         def __init__(self, module, name, file, lineno):
 137                 Class.__init__(self, module, name, None, file, lineno)
 138         def _addmethod(self, name, lineno):
 139                 assert 0, "Function._addmethod() shouldn't be called"
 140
 141 def readmodule(module, path=[], inpackage=0):
 142         '''Backwards compatible interface.
 143
 144         Like readmodule_ex() but strips Function objects from the
 145         resulting dictionary.'''
 146
 147         dict = readmodule_ex(module, path, inpackage)
 148         res = {}
 149         for key, value in dict.items():
 150                 if not isinstance(value, Function):
 151                         res[key] = value
 152         return res
 153
 154 def readmodule_ex(module, path=[], inpackage=0):
 155         '''Read a module file and return a dictionary of classes.
 156
 157         Search for MODULE in PATH and sys.path, read and parse the
 158         module and return a dictionary with one entry for each class
 159         found in the module.'''
 160
 161         dict = {}
 162
 163         i = string.rfind(module, '.')
 164         if i >= 0:
 165                 # Dotted module name
 166                 package = string.strip(module[:i])
 167                 submodule = string.strip(module[i+1:])
 168                 parent = readmodule(package, path, inpackage)
 169                 child = readmodule(submodule, parent['__path__'], 1)
 170                 return child
 171
 172         if _modules.has_key(module):
 173                 # we've seen this module before...
 174                 return _modules[module]
 175         if module in sys.builtin_module_names:
 176                 # this is a built-in module
 177                 _modules[module] = dict
 178                 return dict
 179
 180         # search the path for the module
 181         f = None
 182         if inpackage:
 183                 try:
 184                         f, file, (suff, mode, type) = \
 185                                 imp.find_module(module, path)
 186                 except ImportError:
 187                         f = None
 188         if f is None:
 189                 fullpath = list(path) + sys.path
 190                 f, file, (suff, mode, type) = imp.find_module(module, fullpath)
 191         if type == imp.PKG_DIRECTORY:
 192                 dict['__path__'] = [file]
 193                 _modules[module] = dict
 194                 path = [file] + path
 195                 f, file, (suff, mode, type) = \
 196                                 imp.find_module('__init__', [file])
 197         if type != imp.PY_SOURCE:
 198                 # not Python source, can't do anything with this module
 199                 f.close()
 200                 _modules[module] = dict
 201                 return dict
 202
 203         _modules[module] = dict
 204         imports = []
 205         classstack = [] # stack of (class, indent) pairs
 206         src = f.read()
 207         f.close()
 208
 209         # To avoid having to stop the regexp at each newline, instead
 210         # when we need a line number we simply string.count the number of
 211         # newlines in the string since the last time we did this; i.e.,
 212         #    lineno = lineno + \
 213         #             string.count(src, '\n', last_lineno_pos, here)
 214         #    last_lineno_pos = here
 215         countnl = string.count
 216         lineno, last_lineno_pos = 1, 0
 217         i = 0
 218         while 1:
 219                 m = _getnext(src, i)
 220                 if not m:
 221                         break
 222                 start, i = m.span()
 223
 224                 if m.start("Method") >= 0:
 225                         # found a method definition or function
 226                         thisindent = _indent(m.group("MethodIndent"))
 227                         meth_name = m.group("MethodName")
 228                         lineno = lineno + \
 229                                  countnl(src, '\n',
 230                                          last_lineno_pos, start)
 231                         last_lineno_pos = start
 232                         # close all classes indented at least as much
 233                         while classstack and \
 234                               classstack[-1][1] >= thisindent:
 235                                 del classstack[-1]
 236                         if classstack:
 237                                 # it's a class method
 238                                 cur_class = classstack[-1][0]
 239                                 cur_class._addmethod(meth_name, lineno)
 240                         else:
 241                                 # it's a function
 242                                 f = Function(module, meth_name,
 243                                              file, lineno)
 244                                 dict[meth_name] = f
 245
 246                 elif m.start("String") >= 0:
 247                         pass
 248
 249                 elif m.start("Class") >= 0:
 250                         # we found a class definition
 251                         thisindent = _indent(m.group("ClassIndent"))
 252                         # close all classes indented at least as much
 253                         while classstack and \
 254                               classstack[-1][1] >= thisindent:
 255                                 del classstack[-1]
 256                         lineno = lineno + \
 257                                  countnl(src, '\n', last_lineno_pos, start)
 258                         last_lineno_pos = start
 259                         class_name = m.group("ClassName")
 260                         inherit = m.group("ClassSupers")
 261                         if inherit:
 262                                 # the class inherits from other classes
 263                                 inherit = string.strip(inherit[1:-1])
 264                                 names = []
 265                                 for n in string.splitfields(inherit, ','):
 266                                         n = string.strip(n)
 267                                         if dict.has_key(n):
 268                                                 # we know this super class
 269                                                 n = dict[n]
 270                                         else:
 271                                                 c = string.splitfields(n, '.')
 272                                                 if len(c) > 1:
 273                                                         # super class
 274                                                         # is of the
 275                                                         # form module.class:
 276                                                         # look in
 277                                                         # module for class
 278                                                         m = c[-2]
 279                                                         c = c[-1]
 280                                                         if _modules.has_key(m):
 281                                                                 d = _modules[m]
 282                                                                 if d.has_key(c):
 283                                                                         n = d[c]
 284                                         names.append(n)
 285                                 inherit = names
 286                         # remember this class
 287                         cur_class = Class(module, class_name, inherit,
 288                                           file, lineno)
 289                         dict[class_name] = cur_class
 290                         classstack.append((cur_class, thisindent))
 291
 292                 elif m.start("Import") >= 0:
 293                         # import module
 294                         for n in string.split(m.group("ImportList"), ','):
 295                                 n = string.strip(n)
 296                                 try:
 297                                         # recursively read the imported module
 298                                         d = readmodule(n, path, inpackage)
 299                                 except:
 300                                         ##print 'module', n, 'not found'
 301                                         pass
 302
 303                 elif m.start("ImportFrom") >= 0:
 304                         # from module import stuff
 305                         mod = m.group("ImportFromPath")
 306                         names = string.split(m.group("ImportFromList"), ',')
 307                         try:
 308                                 # recursively read the imported module
 309                                 d = readmodule(mod, path, inpackage)
 310                         except:
 311                                 ##print 'module', mod, 'not found'
 312                                 continue
 313                         # add any classes that were defined in the
 314                         # imported module to our name space if they
 315                         # were mentioned in the list
 316                         for n in names:
 317                                 n = string.strip(n)
 318                                 if d.has_key(n):
 319                                         dict[n] = d[n]
 320                                 elif n == '*':
 321                                         # only add a name if not
 322                                         # already there (to mimic what
 323                                         # Python does internally)
 324                                         # also don't add names that
 325                                         # start with _
 326                                         for n in d.keys():
 327                                                 if n[0] != '_' and \
 328                                                    not dict.has_key(n):
 329                                                         dict[n] = d[n]
 330                 else:
 331                         assert 0, "regexp _getnext found something unexpected"
 332
 333         return dict
 334
 335 def _indent(ws, _expandtabs=string.expandtabs):
 336         return len(_expandtabs(ws, TABWIDTH))