Lib/ihooks.py

   1 """Import hook support.
   2
   3 Consistent use of this module will make it possible to change the
   4 different mechanisms involved in loading modules independently.
   5
   6 While the built-in module imp exports interfaces to the built-in
   7 module searching and loading algorithm, and it is possible to replace
   8 the built-in function __import__ in order to change the semantics of
   9 the import statement, until now it has been difficult to combine the
  10 effect of different __import__ hacks, like loading modules from URLs
  11 by rimport.py, or restricted execution by rexec.py.
  12
  13 This module defines three new concepts:
  14
  15 1) A "file system hooks" class provides an interface to a filesystem.
  16
  17 One hooks class is defined (Hooks), which uses the interface provided
  18 by standard modules os and os.path.  It should be used as the base
  19 class for other hooks classes.
  20
  21 2) A "module loader" class provides an interface to search for a
  22 module in a search path and to load it.  It defines a method which
  23 searches for a module in a single directory; by overriding this method
  24 one can redefine the details of the search.  If the directory is None,
  25 built-in and frozen modules are searched instead.
  26
  27 Two module loader class are defined, both implementing the search
  28 strategy used by the built-in __import__ function: ModuleLoader uses
  29 the imp module's find_module interface, while HookableModuleLoader
  30 uses a file system hooks class to interact with the file system.  Both
  31 use the imp module's load_* interfaces to actually load the module.
  32
  33 3) A "module importer" class provides an interface to import a
  34 module, as well as interfaces to reload and unload a module.  It also
  35 provides interfaces to install and uninstall itself instead of the
  36 default __import__ and reload (and unload) functions.
  37
  38 One module importer class is defined (ModuleImporter), which uses a
  39 module loader instance passed in (by default HookableModuleLoader is
  40 instantiated).
  41
  42 The classes defined here should be used as base classes for extended
  43 functionality along those lines.
  44
  45 If a module importer class supports dotted names, its import_module()
  46 must return a different value depending on whether it is called on
  47 behalf of a "from ... import ..." statement or not.  (This is caused
  48 by the way the __import__ hook is used by the Python interpreter.)  It
  49 would also do wise to install a different version of reload().
  50
  51 """
  52
  53
  54 import __builtin__
  55 import imp
  56 import os
  57 import sys
  58
  59 __all__ = ["BasicModuleLoader","Hooks","ModuleLoader","FancyModuleLoader",
  60            "BasicModuleImporter","ModuleImporter","install","uninstall"]
  61
  62 VERBOSE = 0
  63
  64
  65 from imp import C_EXTENSION, PY_SOURCE, PY_COMPILED
  66 from imp import C_BUILTIN, PY_FROZEN, PKG_DIRECTORY
  67 BUILTIN_MODULE = C_BUILTIN
  68 FROZEN_MODULE = PY_FROZEN
  69
  70
  71 class _Verbose:
  72
  73     def __init__(self, verbose = VERBOSE):
  74         self.verbose = verbose
  75
  76     def get_verbose(self):
  77         return self.verbose
  78
  79     def set_verbose(self, verbose):
  80         self.verbose = verbose
  81
  82     # XXX The following is an experimental interface
  83
  84     def note(self, *args):
  85         if self.verbose:
  86             self.message(*args)
  87
  88     def message(self, format, *args):
  89         if args:
  90             print format%args
  91         else:
  92             print format
  93
  94
  95 class BasicModuleLoader(_Verbose):
  96
  97     """Basic module loader.
  98
  99     This provides the same functionality as built-in import.  It
 100     doesn't deal with checking sys.modules -- all it provides is
 101     find_module() and a load_module(), as well as find_module_in_dir()
 102     which searches just one directory, and can be overridden by a
 103     derived class to change the module search algorithm when the basic
 104     dependency on sys.path is unchanged.
 105
 106     The interface is a little more convenient than imp's:
 107     find_module(name, [path]) returns None or 'stuff', and
 108     load_module(name, stuff) loads the module.
 109
 110     """
 111
 112     def find_module(self, name, path = None):
 113         if path is None:
 114             path = [None] + self.default_path()
 115         for dir in path:
 116             stuff = self.find_module_in_dir(name, dir)
 117             if stuff: return stuff
 118         return None
 119
 120     def default_path(self):
 121         return sys.path
 122
 123     def find_module_in_dir(self, name, dir):
 124         if dir is None:
 125             return self.find_builtin_module(name)
 126         else:
 127             try:
 128                 return imp.find_module(name, [dir])
 129             except ImportError:
 130                 return None
 131
 132     def find_builtin_module(self, name):
 133         # XXX frozen packages?
 134         if imp.is_builtin(name):
 135             return None, '', ('', '', BUILTIN_MODULE)
 136         if imp.is_frozen(name):
 137             return None, '', ('', '', FROZEN_MODULE)
 138         return None
 139
 140     def load_module(self, name, stuff):
 141         file, filename, info = stuff
 142         try:
 143             return imp.load_module(name, file, filename, info)
 144         finally:
 145             if file: file.close()
 146
 147
 148 class Hooks(_Verbose):
 149
 150     """Hooks into the filesystem and interpreter.
 151
 152     By deriving a subclass you can redefine your filesystem interface,
 153     e.g. to merge it with the URL space.
 154
 155     This base class behaves just like the native filesystem.
 156
 157     """
 158
 159     # imp interface
 160     def get_suffixes(self): return imp.get_suffixes()
 161     def new_module(self, name): return imp.new_module(name)
 162     def is_builtin(self, name): return imp.is_builtin(name)
 163     def init_builtin(self, name): return imp.init_builtin(name)
 164     def is_frozen(self, name): return imp.is_frozen(name)
 165     def init_frozen(self, name): return imp.init_frozen(name)
 166     def get_frozen_object(self, name): return imp.get_frozen_object(name)
 167     def load_source(self, name, filename, file=None):
 168         return imp.load_source(name, filename, file)
 169     def load_compiled(self, name, filename, file=None):
 170         return imp.load_compiled(name, filename, file)
 171     def load_dynamic(self, name, filename, file=None):
 172         return imp.load_dynamic(name, filename, file)
 173     def load_package(self, name, filename, file=None):
 174         return imp.load_module(name, file, filename, ("", "", PKG_DIRECTORY))
 175
 176     def add_module(self, name):
 177         d = self.modules_dict()
 178         if name in d: return d[name]
 179         d[name] = m = self.new_module(name)
 180         return m
 181
 182     # sys interface
 183     def modules_dict(self): return sys.modules
 184     def default_path(self): return sys.path
 185
 186     def path_split(self, x): return os.path.split(x)
 187     def path_join(self, x, y): return os.path.join(x, y)
 188     def path_isabs(self, x): return os.path.isabs(x)
 189     # etc.
 190
 191     def path_exists(self, x): return os.path.exists(x)
 192     def path_isdir(self, x): return os.path.isdir(x)
 193     def path_isfile(self, x): return os.path.isfile(x)
 194     def path_islink(self, x): return os.path.islink(x)
 195     # etc.
 196
 197     def openfile(self, *x): return open(*x)
 198     openfile_error = IOError
 199     def listdir(self, x): return os.listdir(x)
 200     listdir_error = os.error
 201     # etc.
 202
 203
 204 class ModuleLoader(BasicModuleLoader):
 205
 206     """Default module loader; uses file system hooks.
 207
 208     By defining suitable hooks, you might be able to load modules from
 209     other sources than the file system, e.g. from compressed or
 210     encrypted files, tar files or (if you're brave!) URLs.
 211
 212     """
 213
 214     def __init__(self, hooks = None, verbose = VERBOSE):
 215         BasicModuleLoader.__init__(self, verbose)
 216         self.hooks = hooks or Hooks(verbose)
 217
 218     def default_path(self):
 219         return self.hooks.default_path()
 220
 221     def modules_dict(self):
 222         return self.hooks.modules_dict()
 223
 224     def get_hooks(self):
 225         return self.hooks
 226
 227     def set_hooks(self, hooks):
 228         self.hooks = hooks
 229
 230     def find_builtin_module(self, name):
 231         # XXX frozen packages?
 232         if self.hooks.is_builtin(name):
 233             return None, '', ('', '', BUILTIN_MODULE)
 234         if self.hooks.is_frozen(name):
 235             return None, '', ('', '', FROZEN_MODULE)
 236         return None
 237
 238     def find_module_in_dir(self, name, dir, allow_packages=1):
 239         if dir is None:
 240             return self.find_builtin_module(name)
 241         if allow_packages:
 242             fullname = self.hooks.path_join(dir, name)
 243             if self.hooks.path_isdir(fullname):
 244                 stuff = self.find_module_in_dir("__init__", fullname, 0)
 245                 if stuff:
 246                     file = stuff[0]
 247                     if file: file.close()
 248                     return None, fullname, ('', '', PKG_DIRECTORY)
 249         for info in self.hooks.get_suffixes():
 250             suff, mode, type = info
 251             fullname = self.hooks.path_join(dir, name+suff)
 252             try:
 253                 fp = self.hooks.openfile(fullname, mode)
 254                 return fp, fullname, info
 255             except self.hooks.openfile_error:
 256                 pass
 257         return None
 258
 259     def load_module(self, name, stuff):
 260         file, filename, info = stuff
 261         (suff, mode, type) = info
 262         try:
 263             if type == BUILTIN_MODULE:
 264                 return self.hooks.init_builtin(name)
 265             if type == FROZEN_MODULE:
 266                 return self.hooks.init_frozen(name)
 267             if type == C_EXTENSION:
 268                 m = self.hooks.load_dynamic(name, filename, file)
 269             elif type == PY_SOURCE:
 270                 m = self.hooks.load_source(name, filename, file)
 271             elif type == PY_COMPILED:
 272                 m = self.hooks.load_compiled(name, filename, file)
 273             elif type == PKG_DIRECTORY:
 274                 m = self.hooks.load_package(name, filename, file)
 275             else:
 276                 raise ImportError, "Unrecognized module type (%s) for %s" % \
 277                       (`type`, name)
 278         finally:
 279             if file: file.close()
 280         m.__file__ = filename
 281         return m
 282
 283
 284 class FancyModuleLoader(ModuleLoader):
 285
 286     """Fancy module loader -- parses and execs the code itself."""
 287
 288     def load_module(self, name, stuff):
 289         file, filename, (suff, mode, type) = stuff
 290         realfilename = filename
 291         path = None
 292
 293         if type == PKG_DIRECTORY:
 294             initstuff = self.find_module_in_dir("__init__", filename, 0)
 295             if not initstuff:
 296                 raise ImportError, "No __init__ module in package %s" % name
 297             initfile, initfilename, initinfo = initstuff
 298             initsuff, initmode, inittype = initinfo
 299             if inittype not in (PY_COMPILED, PY_SOURCE):
 300                 if initfile: initfile.close()
 301                 raise ImportError, \
 302                     "Bad type (%s) for __init__ module in package %s" % (
 303                     `inittype`, name)
 304             path = [filename]
 305             file = initfile
 306             realfilename = initfilename
 307             type = inittype
 308
 309         if type == FROZEN_MODULE:
 310             code = self.hooks.get_frozen_object(name)
 311         elif type == PY_COMPILED:
 312             import marshal
 313             file.seek(8)
 314             code = marshal.load(file)
 315         elif type == PY_SOURCE:
 316             data = file.read()
 317             code = compile(data, realfilename, 'exec')
 318         else:
 319             return ModuleLoader.load_module(self, name, stuff)
 320
 321         m = self.hooks.add_module(name)
 322         if path:
 323             m.__path__ = path
 324         m.__file__ = filename
 325         exec code in m.__dict__
 326         return m
 327
 328
 329 class BasicModuleImporter(_Verbose):
 330
 331     """Basic module importer; uses module loader.
 332
 333     This provides basic import facilities but no package imports.
 334
 335     """
 336
 337     def __init__(self, loader = None, verbose = VERBOSE):
 338         _Verbose.__init__(self, verbose)
 339         self.loader = loader or ModuleLoader(None, verbose)
 340         self.modules = self.loader.modules_dict()
 341
 342     def get_loader(self):
 343         return self.loader
 344
 345     def set_loader(self, loader):
 346         self.loader = loader
 347
 348     def get_hooks(self):
 349         return self.loader.get_hooks()
 350
 351     def set_hooks(self, hooks):
 352         return self.loader.set_hooks(hooks)
 353
 354     def import_module(self, name, globals={}, locals={}, fromlist=[]):
 355         name = str(name)
 356         if name in self.modules:
 357             return self.modules[name] # Fast path
 358         stuff = self.loader.find_module(name)
 359         if not stuff:
 360             raise ImportError, "No module named %s" % name
 361         return self.loader.load_module(name, stuff)
 362
 363     def reload(self, module, path = None):
 364         name = str(module.__name__)
 365         stuff = self.loader.find_module(name, path)
 366         if not stuff:
 367             raise ImportError, "Module %s not found for reload" % name
 368         return self.loader.load_module(name, stuff)
 369
 370     def unload(self, module):
 371         del self.modules[str(module.__name__)]
 372         # XXX Should this try to clear the module's namespace?
 373
 374     def install(self):
 375         self.save_import_module = __builtin__.__import__
 376         self.save_reload = __builtin__.reload
 377         if not hasattr(__builtin__, 'unload'):
 378             __builtin__.unload = None
 379         self.save_unload = __builtin__.unload
 380         __builtin__.__import__ = self.import_module
 381         __builtin__.reload = self.reload
 382         __builtin__.unload = self.unload
 383
 384     def uninstall(self):
 385         __builtin__.__import__ = self.save_import_module
 386         __builtin__.reload = self.save_reload
 387         __builtin__.unload = self.save_unload
 388         if not __builtin__.unload:
 389             del __builtin__.unload
 390
 391
 392 class ModuleImporter(BasicModuleImporter):
 393
 394     """A module importer that supports packages."""
 395
 396     def import_module(self, name, globals=None, locals=None, fromlist=None):
 397         parent = self.determine_parent(globals)
 398         q, tail = self.find_head_package(parent, str(name))
 399         m = self.load_tail(q, tail)
 400         if not fromlist:
 401             return q
 402         if hasattr(m, "__path__"):
 403             self.ensure_fromlist(m, fromlist)
 404         return m
 405
 406     def determine_parent(self, globals):
 407         if not globals or not "__name__" in globals:
 408             return None
 409         pname = globals['__name__']
 410         if "__path__" in globals:
 411             parent = self.modules[pname]
 412             assert globals is parent.__dict__
 413             return parent
 414         if '.' in pname:
 415             i = pname.rfind('.')
 416             pname = pname[:i]
 417             parent = self.modules[pname]
 418             assert parent.__name__ == pname
 419             return parent
 420         return None
 421
 422     def find_head_package(self, parent, name):
 423         if '.' in name:
 424             i = name.find('.')
 425             head = name[:i]
 426             tail = name[i+1:]
 427         else:
 428             head = name
 429             tail = ""
 430         if parent:
 431             qname = "%s.%s" % (parent.__name__, head)
 432         else:
 433             qname = head
 434         q = self.import_it(head, qname, parent)
 435         if q: return q, tail
 436         if parent:
 437             qname = head
 438             parent = None
 439             q = self.import_it(head, qname, parent)
 440             if q: return q, tail
 441         raise ImportError, "No module named " + qname
 442
 443     def load_tail(self, q, tail):
 444         m = q
 445         while tail:
 446             i = tail.find('.')
 447             if i < 0: i = len(tail)
 448             head, tail = tail[:i], tail[i+1:]
 449             mname = "%s.%s" % (m.__name__, head)
 450             m = self.import_it(head, mname, m)
 451             if not m:
 452                 raise ImportError, "No module named " + mname
 453         return m
 454
 455     def ensure_fromlist(self, m, fromlist, recursive=0):
 456         for sub in fromlist:
 457             if sub == "*":
 458                 if not recursive:
 459                     try:
 460                         all = m.__all__
 461                     except AttributeError:
 462                         pass
 463                     else:
 464                         self.ensure_fromlist(m, all, 1)
 465                 continue
 466             if sub != "*" and not hasattr(m, sub):
 467                 subname = "%s.%s" % (m.__name__, sub)
 468                 submod = self.import_it(sub, subname, m)
 469                 if not submod:
 470                     raise ImportError, "No module named " + subname
 471
 472     def import_it(self, partname, fqname, parent, force_load=0):
 473         if not partname:
 474             raise ValueError, "Empty module name"
 475         if not force_load:
 476             try:
 477                 return self.modules[fqname]
 478             except KeyError:
 479                 pass
 480         try:
 481             path = parent and parent.__path__
 482         except AttributeError:
 483             return None
 484         partname = str(partname)
 485         stuff = self.loader.find_module(partname, path)
 486         if not stuff:
 487             return None
 488         fqname = str(fqname)
 489         m = self.loader.load_module(fqname, stuff)
 490         if parent:
 491             setattr(parent, partname, m)
 492         return m
 493
 494     def reload(self, module):
 495         name = str(module.__name__)
 496         if '.' not in name:
 497             return self.import_it(name, name, None, force_load=1)
 498         i = name.rfind('.')
 499         pname = name[:i]
 500         parent = self.modules[pname]
 501         return self.import_it(name[i+1:], name, parent, force_load=1)
 502
 503
 504 default_importer = None
 505 current_importer = None
 506
 507 def install(importer = None):
 508     global current_importer
 509     current_importer = importer or default_importer or ModuleImporter()
 510     current_importer.install()
 511
 512 def uninstall():
 513     global current_importer
 514     current_importer.uninstall()