Lib/pkgutil.py

   1 """Utilities to support packages."""
   2
   3 # NOTE: This module must remain compatible with Python 2.3, as it is shared
   4 # by setuptools for distribution with Python 2.3 and up.
   5
   6 import os
   7 import sys
   8 import imp
   9 import os.path
  10 from types import ModuleType
  11
  12 __all__ = [
  13     'get_importer', 'iter_importers', 'get_loader', 'find_loader',
  14     'walk_packages', 'iter_modules',
  15     'ImpImporter', 'ImpLoader', 'read_code', 'extend_path',
  16 ]
  17
  18 def read_code(stream):
  19     # This helper is needed in order for the PEP 302 emulation to
  20     # correctly handle compiled files
  21     import marshal
  22
  23     magic = stream.read(4)
  24     if magic != imp.get_magic():
  25         return None
  26
  27     stream.read(4) # Skip timestamp
  28     return marshal.load(stream)
  29
  30
  31 def simplegeneric(func):
  32     """Make a trivial single-dispatch generic function"""
  33     registry = {}
  34     def wrapper(*args, **kw):
  35         ob = args[0]
  36         try:
  37             cls = ob.__class__
  38         except AttributeError:
  39             cls = type(ob)
  40         try:
  41             mro = cls.__mro__
  42         except AttributeError:
  43             try:
  44                 class cls(cls, object):
  45                     pass
  46                 mro = cls.__mro__[1:]
  47             except TypeError:
  48                 mro = object,   # must be an ExtensionClass or some such  :(
  49         for t in mro:
  50             if t in registry:
  51                 return registry[t](*args, **kw)
  52         else:
  53             return func(*args, **kw)
  54     try:
  55         wrapper.__name__ = func.__name__
  56     except (TypeError, AttributeError):
  57         pass    # Python 2.3 doesn't allow functions to be renamed
  58
  59     def register(typ, func=None):
  60         if func is None:
  61             return lambda f: register(typ, f)
  62         registry[typ] = func
  63         return func
  64
  65     wrapper.__dict__ = func.__dict__
  66     wrapper.__doc__ = func.__doc__
  67     wrapper.register = register
  68     return wrapper
  69
  70
  71 def walk_packages(path=None, prefix='', onerror=None):
  72     """Yield submodule names+loaders recursively, for path or sys.path"""
  73
  74     def seen(p, m={}):
  75         if p in m:
  76             return True
  77         m[p] = True
  78
  79     for importer, name, ispkg in iter_modules(path, prefix):
  80         yield importer, name, ispkg
  81
  82         if ispkg:
  83             try:
  84                 __import__(name)
  85             except ImportError:
  86                 if onerror is not None:
  87                     onerror()
  88             else:
  89                 path = getattr(sys.modules[name], '__path__', None) or []
  90
  91                 # don't traverse path items we've seen before
  92                 path = [p for p in path if not seen(p)]
  93
  94                 for item in walk_packages(path, name+'.'):
  95                     yield item
  96
  97
  98 def iter_modules(path=None, prefix=''):
  99     """Yield submodule names+loaders for path or sys.path"""
 100     if path is None:
 101         importers = iter_importers()
 102     else:
 103         importers = map(get_importer, path)
 104
 105     yielded = {}
 106     for i in importers:
 107         for name, ispkg in iter_importer_modules(i, prefix):
 108             if name not in yielded:
 109                 yielded[name] = 1
 110                 yield i, name, ispkg
 111
 112
 113 #@simplegeneric
 114 def iter_importer_modules(importer, prefix=''):
 115     if not hasattr(importer, 'iter_modules'):
 116         return []
 117     return importer.iter_modules(prefix)
 118
 119 iter_importer_modules = simplegeneric(iter_importer_modules)
 120
 121
 122 class ImpImporter:
 123     """PEP 302 Importer that wraps Python's "classic" import algorithm
 124
 125     ImpImporter(dirname) produces a PEP 302 importer that searches that
 126     directory.  ImpImporter(None) produces a PEP 302 importer that searches
 127     the current sys.path, plus any modules that are frozen or built-in.
 128
 129     Note that ImpImporter does not currently support being used by placement
 130     on sys.meta_path.
 131     """
 132
 133     def __init__(self, path=None):
 134         self.path = path
 135
 136     def find_module(self, fullname, path=None):
 137         # Note: we ignore 'path' argument since it is only used via meta_path
 138         subname = fullname.split(".")[-1]
 139         if subname != fullname and self.path is None:
 140             return None
 141         if self.path is None:
 142             path = None
 143         else:
 144             path = [os.path.realpath(self.path)]
 145         try:
 146             file, filename, etc = imp.find_module(subname, path)
 147         except ImportError:
 148             return None
 149         return ImpLoader(fullname, file, filename, etc)
 150
 151     def iter_modules(self, prefix=''):
 152         if self.path is None or not os.path.isdir(self.path):
 153             return
 154
 155         yielded = {}
 156         import inspect
 157
 158         filenames = os.listdir(self.path)
 159         filenames.sort()  # handle packages before same-named modules
 160
 161         for fn in filenames:
 162             modname = inspect.getmodulename(fn)
 163             if modname=='__init__' or modname in yielded:
 164                 continue
 165
 166             path = os.path.join(self.path, fn)
 167             ispkg = False
 168
 169             if not modname and os.path.isdir(path) and '.' not in fn:
 170                 modname = fn
 171                 for fn in os.listdir(path):
 172                     subname = inspect.getmodulename(fn)
 173                     if subname=='__init__':
 174                         ispkg = True
 175                         break
 176                 else:
 177                     continue    # not a package
 178
 179             if modname and '.' not in modname:
 180                 yielded[modname] = 1
 181                 yield prefix + modname, ispkg
 182
 183
 184 class ImpLoader:
 185     """PEP 302 Loader that wraps Python's "classic" import algorithm
 186     """
 187     code = source = None
 188
 189     def __init__(self, fullname, file, filename, etc):
 190         self.file = file
 191         self.filename = filename
 192         self.fullname = fullname
 193         self.etc = etc
 194
 195     def load_module(self, fullname):
 196         self._reopen()
 197         try:
 198             mod = imp.load_module(fullname, self.file, self.filename, self.etc)
 199         finally:
 200             if self.file:
 201                 self.file.close()
 202         # Note: we don't set __loader__ because we want the module to look
 203         # normal; i.e. this is just a wrapper for standard import machinery
 204         return mod
 205
 206     def get_data(self, pathname):
 207         return open(pathname, "rb").read()
 208
 209     def _reopen(self):
 210         if self.file and self.file.closed:
 211             mod_type = self.etc[2]
 212             if mod_type==imp.PY_SOURCE:
 213                 self.file = open(self.filename, 'rU')
 214             elif mod_type in (imp.PY_COMPILED, imp.C_EXTENSION):
 215                 self.file = open(self.filename, 'rb')
 216
 217     def _fix_name(self, fullname):
 218         if fullname is None:
 219             fullname = self.fullname
 220         elif fullname != self.fullname:
 221             raise ImportError("Loader for module %s cannot handle "
 222                               "module %s" % (self.fullname, fullname))
 223         return fullname
 224
 225     def is_package(self, fullname):
 226         fullname = self._fix_name(fullname)
 227         return self.etc[2]==imp.PKG_DIRECTORY
 228
 229     def get_code(self, fullname=None):
 230         fullname = self._fix_name(fullname)
 231         if self.code is None:
 232             mod_type = self.etc[2]
 233             if mod_type==imp.PY_SOURCE:
 234                 source = self.get_source(fullname)
 235                 self.code = compile(source, self.filename, 'exec')
 236             elif mod_type==imp.PY_COMPILED:
 237                 self._reopen()
 238                 try:
 239                     self.code = read_code(self.file)
 240                 finally:
 241                     self.file.close()
 242             elif mod_type==imp.PKG_DIRECTORY:
 243                 self.code = self._get_delegate().get_code()
 244         return self.code
 245
 246     def get_source(self, fullname=None):
 247         fullname = self._fix_name(fullname)
 248         if self.source is None:
 249             mod_type = self.etc[2]
 250             if mod_type==imp.PY_SOURCE:
 251                 self._reopen()
 252                 try:
 253                     self.source = self.file.read()
 254                 finally:
 255                     self.file.close()
 256             elif mod_type==imp.PY_COMPILED:
 257                 if os.path.exists(self.filename[:-1]):
 258                     f = open(self.filename[:-1], 'rU')
 259                     self.source = f.read()
 260                     f.close()
 261             elif mod_type==imp.PKG_DIRECTORY:
 262                 self.source = self._get_delegate().get_source()
 263         return self.source
 264
 265
 266     def _get_delegate(self):
 267         return ImpImporter(self.filename).find_module('__init__')
 268
 269     def get_filename(self, fullname=None):
 270         fullname = self._fix_name(fullname)
 271         mod_type = self.etc[2]
 272         if self.etc[2]==imp.PKG_DIRECTORY:
 273             return self._get_delegate().get_filename()
 274         elif self.etc[2] in (imp.PY_SOURCE, imp.PY_COMPILED, imp.C_EXTENSION):
 275             return self.filename
 276         return None
 277
 278
 279 try:
 280     import zipimport
 281     from zipimport import zipimporter
 282
 283     def iter_zipimport_modules(importer, prefix=''):
 284         dirlist = zipimport._zip_directory_cache[importer.archive].keys()
 285         dirlist.sort()
 286         _prefix = importer.prefix
 287         plen = len(_prefix)
 288         yielded = {}
 289         import inspect
 290         for fn in dirlist:
 291             if not fn.startswith(_prefix):
 292                 continue
 293
 294             fn = fn[plen:].split(os.sep)
 295
 296             if len(fn)==2 and fn[1].startswith('__init__.py'):
 297                 if fn[0] not in yielded:
 298                     yielded[fn[0]] = 1
 299                     yield fn[0], True
 300
 301             if len(fn)!=1:
 302                 continue
 303
 304             modname = inspect.getmodulename(fn[0])
 305             if modname=='__init__':
 306                 continue
 307
 308             if modname and '.' not in modname and modname not in yielded:
 309                 yielded[modname] = 1
 310                 yield prefix + modname, False
 311
 312     iter_importer_modules.register(zipimporter, iter_zipimport_modules)
 313
 314 except ImportError:
 315     pass
 316
 317
 318 def get_importer(path_item):
 319     """Retrieve a PEP 302 importer for the given path item
 320
 321     The returned importer is cached in sys.path_importer_cache
 322     if it was newly created by a path hook.
 323
 324     If there is no importer, a wrapper around the basic import
 325     machinery is returned. This wrapper is never inserted into
 326     the importer cache (None is inserted instead).
 327
 328     The cache (or part of it) can be cleared manually if a
 329     rescan of sys.path_hooks is necessary.
 330     """
 331     try:
 332         importer = sys.path_importer_cache[path_item]
 333     except KeyError:
 334         for path_hook in sys.path_hooks:
 335             try:
 336                 importer = path_hook(path_item)
 337                 break
 338             except ImportError:
 339                 pass
 340         else:
 341             importer = None
 342         sys.path_importer_cache.setdefault(path_item, importer)
 343
 344     # The boolean values are used for caching valid and invalid
 345     # file paths for the built-in import machinery
 346     if importer in (None, True, False):
 347         try:
 348             importer = ImpImporter(path_item)
 349         except ImportError:
 350             importer = None
 351     return importer
 352
 353
 354 def iter_importers(fullname=""):
 355     """Yield PEP 302 importers for the given module name
 356
 357     If fullname contains a '.', the importers will be for the package
 358     containing fullname, otherwise they will be importers for sys.meta_path,
 359     sys.path, and Python's "classic" import machinery, in that order.  If
 360     the named module is in a package, that package is imported as a side
 361     effect of invoking this function.
 362
 363     Non PEP 302 mechanisms (e.g. the Windows registry) used by the
 364     standard import machinery to find files in alternative locations
 365     are partially supported, but are searched AFTER sys.path. Normally,
 366     these locations are searched BEFORE sys.path, preventing sys.path
 367     entries from shadowing them.
 368
 369     For this to cause a visible difference in behaviour, there must
 370     be a module or package name that is accessible via both sys.path
 371     and one of the non PEP 302 file system mechanisms. In this case,
 372     the emulation will find the former version, while the builtin
 373     import mechanism will find the latter.
 374
 375     Items of the following types can be affected by this discrepancy:
 376         imp.C_EXTENSION, imp.PY_SOURCE, imp.PY_COMPILED, imp.PKG_DIRECTORY
 377     """
 378     if fullname.startswith('.'):
 379         raise ImportError("Relative module names not supported")
 380     if '.' in fullname:
 381         # Get the containing package's __path__
 382         pkg = '.'.join(fullname.split('.')[:-1])
 383         if pkg not in sys.modules:
 384             __import__(pkg)
 385         path = getattr(sys.modules[pkg], '__path__', None) or []
 386     else:
 387         for importer in sys.meta_path:
 388             yield importer
 389         path = sys.path
 390     for item in path:
 391         yield get_importer(item)
 392     if '.' not in fullname:
 393         yield ImpImporter()
 394
 395 def get_loader(module_or_name):
 396     """Get a PEP 302 "loader" object for module_or_name
 397
 398     If the module or package is accessible via the normal import
 399     mechanism, a wrapper around the relevant part of that machinery
 400     is returned.  Returns None if the module cannot be found or imported.
 401     If the named module is not already imported, its containing package
 402     (if any) is imported, in order to establish the package __path__.
 403
 404     This function uses iter_importers(), and is thus subject to the same
 405     limitations regarding platform-specific special import locations such
 406     as the Windows registry.
 407     """
 408     if module_or_name in sys.modules:
 409         module_or_name = sys.modules[module_or_name]
 410     if isinstance(module_or_name, ModuleType):
 411         module = module_or_name
 412         loader = getattr(module, '__loader__', None)
 413         if loader is not None:
 414             return loader
 415         fullname = module.__name__
 416     else:
 417         fullname = module_or_name
 418     return find_loader(fullname)
 419
 420 def find_loader(fullname):
 421     """Find a PEP 302 "loader" object for fullname
 422
 423     If fullname contains dots, path must be the containing package's __path__.
 424     Returns None if the module cannot be found or imported. This function uses
 425     iter_importers(), and is thus subject to the same limitations regarding
 426     platform-specific special import locations such as the Windows registry.
 427     """
 428     for importer in iter_importers(fullname):
 429         loader = importer.find_module(fullname)
 430         if loader is not None:
 431             return loader
 432
 433     return None
 434
 435
 436 def extend_path(path, name):
 437     """Extend a package's path.
 438
 439     Intended use is to place the following code in a package's __init__.py:
 440
 441         from pkgutil import extend_path
 442         __path__ = extend_path(__path__, __name__)
 443
 444     This will add to the package's __path__ all subdirectories of
 445     directories on sys.path named after the package.  This is useful
 446     if one wants to distribute different parts of a single logical
 447     package as multiple directories.
 448
 449     It also looks for *.pkg files beginning where * matches the name
 450     argument.  This feature is similar to *.pth files (see site.py),
 451     except that it doesn't special-case lines starting with 'import'.
 452     A *.pkg file is trusted at face value: apart from checking for
 453     duplicates, all entries found in a *.pkg file are added to the
 454     path, regardless of whether they are exist the filesystem.  (This
 455     is a feature.)
 456
 457     If the input path is not a list (as is the case for frozen
 458     packages) it is returned unchanged.  The input path is not
 459     modified; an extended copy is returned.  Items are only appended
 460     to the copy at the end.
 461
 462     It is assumed that sys.path is a sequence.  Items of sys.path that
 463     are not (unicode or 8-bit) strings referring to existing
 464     directories are ignored.  Unicode items of sys.path that cause
 465     errors when used as filenames may cause this function to raise an
 466     exception (in line with os.path.isdir() behavior).
 467     """
 468
 469     if not isinstance(path, list):
 470         # This could happen e.g. when this is called from inside a
 471         # frozen package.  Return the path unchanged in that case.
 472         return path
 473
 474     pname = os.path.join(*name.split('.')) # Reconstitute as relative path
 475     # Just in case os.extsep != '.'
 476     sname = os.extsep.join(name.split('.'))
 477     sname_pkg = sname + os.extsep + "pkg"
 478     init_py = "__init__" + os.extsep + "py"
 479
 480     path = path[:] # Start with a copy of the existing path
 481
 482     for dir in sys.path:
 483         if not isinstance(dir, basestring) or not os.path.isdir(dir):
 484             continue
 485         subdir = os.path.join(dir, pname)
 486         # XXX This may still add duplicate entries to path on
 487         # case-insensitive filesystems
 488         initfile = os.path.join(subdir, init_py)
 489         if subdir not in path and os.path.isfile(initfile):
 490             path.append(subdir)
 491         # XXX Is this the right thing for subpackages like zope.app?
 492         # It looks for a file named "zope.app.pkg"
 493         pkgfile = os.path.join(dir, sname_pkg)
 494         if os.path.isfile(pkgfile):
 495             try:
 496                 f = open(pkgfile)
 497             except IOError, msg:
 498                 sys.stderr.write("Can't open %s: %s\n" %
 499                                  (pkgfile, msg))
 500             else:
 501                 for line in f:
 502                     line = line.rstrip('\n')
 503                     if not line or line.startswith('#'):
 504                         continue
 505                     path.append(line) # Don't check for existence!
 506                 f.close()
 507
 508     return path