Lib/ihooks.py

   1 """Import hook support.
   2
   3 Consistent use of this module will make it possible to change the
   4 different mechanisms involved in loading modules independently.
   5
   6 While the built-in module imp exports interfaces to the built-in
   7 module searching and loading algorithm, and it is possible to replace
   8 the built-in function __import__ in order to change the semantics of
   9 the import statement, until now it has been difficult to combine the
  10 effect of different __import__ hacks, like loading modules from URLs
  11 by rimport.py, or restricted execution by rexec.py.
  12
  13 This module defines three new concepts:
  14
  15 1) A "file system hooks" class provides an interface to a filesystem.
  16
  17 One hooks class is defined (Hooks), which uses the interface provided
  18 by standard modules os and os.path.  It should be used as the base
  19 class for other hooks classes.
  20
  21 2) A "module loader" class provides an interface to to search for a
  22 module in a search path and to load it.  It defines a method which
  23 searches for a module in a single directory; by overriding this method
  24 one can redefine the details of the search.  If the directory is None,
  25 built-in and frozen modules are searched instead.
  26
  27 Two module loader class are defined, both implementing the search
  28 strategy used by the built-in __import__ function: ModuleLoader uses
  29 the imp module's find_module interface, while HookableModuleLoader
  30 uses a file system hooks class to interact with the file system.  Both
  31 use the imp module's load_* interfaces to actually load the module.
  32
  33 3) A "module importer" class provides an interface to import a
  34 module, as well as interfaces to reload and unload a module.  It also
  35 provides interfaces to install and uninstall itself instead of the
  36 default __import__ and reload (and unload) functions.
  37
  38 One module importer class is defined (ModuleImporter), which uses a
  39 module loader instance passed in (by default HookableModuleLoader is
  40 instantiated).
  41
  42 The classes defined here should be used as base classes for extended
  43 functionality along those lines.
  44
  45 If a module importer class supports dotted names, its import_module()
  46 must return a different value depending on whether it is called on
  47 behalf of a "from ... import ..." statement or not.  (This is caused
  48 by the way the __import__ hook is used by the Python interpreter.)  It
  49 would also do wise to install a different version of reload().
  50
  51 """
  52
  53
  54 import __builtin__
  55 import imp
  56 import os
  57 import sys
  58
  59 __all__ = ["BasicModuleLoader","Hooks","ModuleLoader","FancyModuleLoader",
  60            "BasicModuleImporter","ModuleImporter","install","uninstall"]
  61
  62 VERBOSE = 0
  63
  64
  65 from imp import C_EXTENSION, PY_SOURCE, PY_COMPILED
  66 from imp import C_BUILTIN, PY_FROZEN, PKG_DIRECTORY
  67 BUILTIN_MODULE = C_BUILTIN
  68 FROZEN_MODULE = PY_FROZEN
  69
  70
  71 class _Verbose:
  72
  73     def __init__(self, verbose = VERBOSE):
  74         self.verbose = verbose
  75
  76     def get_verbose(self):
  77         return self.verbose
  78
  79     def set_verbose(self, verbose):
  80         self.verbose = verbose
  81
  82     # XXX The following is an experimental interface
  83
  84     def note(self, *args):
  85         if self.verbose:
  86             apply(self.message, args)
  87
  88     def message(self, format, *args):
  89         if args:
  90             print format%args
  91         else:
  92             print format
  93
  94
  95 class BasicModuleLoader(_Verbose):
  96
  97     """Basic module loader.
  98
  99     This provides the same functionality as built-in import.  It
 100     doesn't deal with checking sys.modules -- all it provides is
 101     find_module() and a load_module(), as well as find_module_in_dir()
 102     which searches just one directory, and can be overridden by a
 103     derived class to change the module search algorithm when the basic
 104     dependency on sys.path is unchanged.
 105
 106     The interface is a little more convenient than imp's:
 107     find_module(name, [path]) returns None or 'stuff', and
 108     load_module(name, stuff) loads the module.
 109
 110     """
 111
 112     def find_module(self, name, path = None):
 113         if path is None:
 114             path = [None] + self.default_path()
 115         for dir in path:
 116             stuff = self.find_module_in_dir(name, dir)
 117             if stuff: return stuff
 118         return None
 119
 120     def default_path(self):
 121         return sys.path
 122
 123     def find_module_in_dir(self, name, dir):
 124         if dir is None:
 125             return self.find_builtin_module(name)
 126         else:
 127             try:
 128                 return imp.find_module(name, [dir])
 129             except ImportError:
 130                 return None
 131
 132     def find_builtin_module(self, name):
 133         # XXX frozen packages?
 134         if imp.is_builtin(name):
 135             return None, '', ('', '', BUILTIN_MODULE)
 136         if imp.is_frozen(name):
 137             return None, '', ('', '', FROZEN_MODULE)
 138         return None
 139
 140     def load_module(self, name, stuff):
 141         file, filename, info = stuff
 142         try:
 143             return imp.load_module(name, file, filename, info)
 144         finally:
 145             if file: file.close()
 146
 147
 148 class Hooks(_Verbose):
 149
 150     """Hooks into the filesystem and interpreter.
 151
 152     By deriving a subclass you can redefine your filesystem interface,
 153     e.g. to merge it with the URL space.
 154
 155     This base class behaves just like the native filesystem.
 156
 157     """
 158
 159     # imp interface
 160     def get_suffixes(self): return imp.get_suffixes()
 161     def new_module(self, name): return imp.new_module(name)
 162     def is_builtin(self, name): return imp.is_builtin(name)
 163     def init_builtin(self, name): return imp.init_builtin(name)
 164     def is_frozen(self, name): return imp.is_frozen(name)
 165     def init_frozen(self, name): return imp.init_frozen(name)
 166     def get_frozen_object(self, name): return imp.get_frozen_object(name)
 167     def load_source(self, name, filename, file=None):
 168         return imp.load_source(name, filename, file)
 169     def load_compiled(self, name, filename, file=None):
 170         return imp.load_compiled(name, filename, file)
 171     def load_dynamic(self, name, filename, file=None):
 172         return imp.load_dynamic(name, filename, file)
 173     def load_package(self, name, filename, file=None):
 174         return imp.load_module(name, file, filename, ("", "", PKG_DIRECTORY))
 175
 176     def add_module(self, name):
 177         d = self.modules_dict()
 178         if d.has_key(name): return d[name]
 179         d[name] = m = self.new_module(name)
 180         return m
 181
 182     # sys interface
 183     def modules_dict(self): return sys.modules
 184     def default_path(self): return sys.path
 185
 186     def path_split(self, x): return os.path.split(x)
 187     def path_join(self, x, y): return os.path.join(x, y)
 188     def path_isabs(self, x): return os.path.isabs(x)
 189     # etc.
 190
 191     def path_exists(self, x): return os.path.exists(x)
 192     def path_isdir(self, x): return os.path.isdir(x)
 193     def path_isfile(self, x): return os.path.isfile(x)
 194     def path_islink(self, x): return os.path.islink(x)
 195     # etc.
 196
 197     def openfile(self, *x): return apply(open, x)
 198     openfile_error = IOError
 199     def listdir(self, x): return os.listdir(x)
 200     listdir_error = os.error
 201     # etc.
 202
 203
 204 class ModuleLoader(BasicModuleLoader):
 205
 206     """Default module loader; uses file system hooks.
 207
 208     By defining suitable hooks, you might be able to load modules from
 209     other sources than the file system, e.g. from compressed or
 210     encrypted files, tar files or (if you're brave!) URLs.
 211
 212     """
 213
 214     def __init__(self, hooks = None, verbose = VERBOSE):
 215         BasicModuleLoader.__init__(self, verbose)
 216         self.hooks = hooks or Hooks(verbose)
 217
 218     def default_path(self):
 219         return self.hooks.default_path()
 220
 221     def modules_dict(self):
 222         return self.hooks.modules_dict()
 223
 224     def get_hooks(self):
 225         return self.hooks
 226
 227     def set_hooks(self, hooks):
 228         self.hooks = hooks
 229
 230     def find_builtin_module(self, name):
 231         # XXX frozen packages?
 232         if self.hooks.is_builtin(name):
 233             return None, '', ('', '', BUILTIN_MODULE)
 234         if self.hooks.is_frozen(name):
 235             return None, '', ('', '', FROZEN_MODULE)
 236         return None
 237
 238     def find_module_in_dir(self, name, dir, allow_packages=1):
 239         if dir is None:
 240             return self.find_builtin_module(name)
 241         if allow_packages:
 242             fullname = self.hooks.path_join(dir, name)
 243             if self.hooks.path_isdir(fullname):
 244                 stuff = self.find_module_in_dir("__init__", fullname, 0)
 245                 if stuff:
 246                     file = stuff[0]
 247                     if file: file.close()
 248                     return None, fullname, ('', '', PKG_DIRECTORY)
 249         for info in self.hooks.get_suffixes():
 250             suff, mode, type = info
 251             fullname = self.hooks.path_join(dir, name+suff)
 252             try:
 253                 fp = self.hooks.openfile(fullname, mode)
 254                 return fp, fullname, info
 255             except self.hooks.openfile_error:
 256                 pass
 257         return None
 258
 259     def load_module(self, name, stuff):
 260         file, filename, info = stuff
 261         (suff, mode, type) = info
 262         try:
 263             if type == BUILTIN_MODULE:
 264                 return self.hooks.init_builtin(name)
 265             if type == FROZEN_MODULE:
 266                 return self.hooks.init_frozen(name)
 267             if type == C_EXTENSION:
 268                 m = self.hooks.load_dynamic(name, filename, file)
 269             elif type == PY_SOURCE:
 270                 m = self.hooks.load_source(name, filename, file)
 271             elif type == PY_COMPILED:
 272                 m = self.hooks.load_compiled(name, filename, file)
 273             elif type == PKG_DIRECTORY:
 274                 m = self.hooks.load_package(name, filename, file)
 275             else:
 276                 raise ImportError, "Unrecognized module type (%s) for %s" % \
 277                       (`type`, name)
 278         finally:
 279             if file: file.close()
 280         m.__file__ = filename
 281         return m
 282
 283
 284 class FancyModuleLoader(ModuleLoader):
 285
 286     """Fancy module loader -- parses and execs the code itself."""
 287
 288     def load_module(self, name, stuff):
 289         file, filename, (suff, mode, type) = stuff
 290         realfilename = filename
 291         path = None
 292
 293         if type == PKG_DIRECTORY:
 294             initstuff = self.find_module_in_dir("__init__", filename, 0)
 295             if not initstuff:
 296                 raise ImportError, "No __init__ module in package %s" % name
 297             initfile, initfilename, initinfo = initstuff
 298             initsuff, initmode, inittype = initinfo
 299             if inittype not in (PY_COMPILED, PY_SOURCE):
 300                 if initfile: initfile.close()
 301                 raise ImportError, \
 302                     "Bad type (%s) for __init__ module in package %s" % (
 303                     `inittype`, name)
 304             path = [filename]
 305             file = initfile
 306             realfilename = initfilename
 307             type = inittype
 308
 309         if type == FROZEN_MODULE:
 310             code = self.hooks.get_frozen_object(name)
 311         elif type == PY_COMPILED:
 312             import marshal
 313             file.seek(8)
 314             code = marshal.load(file)
 315         elif type == PY_SOURCE:
 316             data = file.read()
 317             code = compile(data, realfilename, 'exec')
 318         else:
 319             return ModuleLoader.load_module(self, name, stuff)
 320
 321         m = self.hooks.add_module(name)
 322         if path:
 323             m.__path__ = path
 324         m.__file__ = filename
 325         exec code in m.__dict__
 326         return m
 327
 328
 329 class BasicModuleImporter(_Verbose):
 330
 331     """Basic module importer; uses module loader.
 332
 333     This provides basic import facilities but no package imports.
 334
 335     """
 336
 337     def __init__(self, loader = None, verbose = VERBOSE):
 338         _Verbose.__init__(self, verbose)
 339         self.loader = loader or ModuleLoader(None, verbose)
 340         self.modules = self.loader.modules_dict()
 341
 342     def get_loader(self):
 343         return self.loader
 344
 345     def set_loader(self, loader):
 346         self.loader = loader
 347
 348     def get_hooks(self):
 349         return self.loader.get_hooks()
 350
 351     def set_hooks(self, hooks):
 352         return self.loader.set_hooks(hooks)
 353
 354     def import_module(self, name, globals={}, locals={}, fromlist=[]):
 355         if self.modules.has_key(name):
 356             return self.modules[name] # Fast path
 357         stuff = self.loader.find_module(name)
 358         if not stuff:
 359             raise ImportError, "No module named %s" % name
 360         return self.loader.load_module(name, stuff)
 361
 362     def reload(self, module, path = None):
 363         name = module.__name__
 364         stuff = self.loader.find_module(name, path)
 365         if not stuff:
 366             raise ImportError, "Module %s not found for reload" % name
 367         return self.loader.load_module(name, stuff)
 368
 369     def unload(self, module):
 370         del self.modules[module.__name__]
 371         # XXX Should this try to clear the module's namespace?
 372
 373     def install(self):
 374         self.save_import_module = __builtin__.__import__
 375         self.save_reload = __builtin__.reload
 376         if not hasattr(__builtin__, 'unload'):
 377             __builtin__.unload = None
 378         self.save_unload = __builtin__.unload
 379         __builtin__.__import__ = self.import_module
 380         __builtin__.reload = self.reload
 381         __builtin__.unload = self.unload
 382
 383     def uninstall(self):
 384         __builtin__.__import__ = self.save_import_module
 385         __builtin__.reload = self.save_reload
 386         __builtin__.unload = self.save_unload
 387         if not __builtin__.unload:
 388             del __builtin__.unload
 389
 390
 391 class ModuleImporter(BasicModuleImporter):
 392
 393     """A module importer that supports packages."""
 394
 395     def import_module(self, name, globals=None, locals=None, fromlist=None):
 396         parent = self.determine_parent(globals)
 397         q, tail = self.find_head_package(parent, name)
 398         m = self.load_tail(q, tail)
 399         if not fromlist:
 400             return q
 401         if hasattr(m, "__path__"):
 402             self.ensure_fromlist(m, fromlist)
 403         return m
 404
 405     def determine_parent(self, globals):
 406         if not globals or not globals.has_key("__name__"):
 407             return None
 408         pname = globals['__name__']
 409         if globals.has_key("__path__"):
 410             parent = self.modules[pname]
 411             assert globals is parent.__dict__
 412             return parent
 413         if '.' in pname:
 414             i = pname.rfind('.')
 415             pname = pname[:i]
 416             parent = self.modules[pname]
 417             assert parent.__name__ == pname
 418             return parent
 419         return None
 420
 421     def find_head_package(self, parent, name):
 422         if '.' in name:
 423             i = name.find('.')
 424             head = name[:i]
 425             tail = name[i+1:]
 426         else:
 427             head = name
 428             tail = ""
 429         if parent:
 430             qname = "%s.%s" % (parent.__name__, head)
 431         else:
 432             qname = head
 433         q = self.import_it(head, qname, parent)
 434         if q: return q, tail
 435         if parent:
 436             qname = head
 437             parent = None
 438             q = self.import_it(head, qname, parent)
 439             if q: return q, tail
 440         raise ImportError, "No module named " + qname
 441
 442     def load_tail(self, q, tail):
 443         m = q
 444         while tail:
 445             i = tail.find('.')
 446             if i < 0: i = len(tail)
 447             head, tail = tail[:i], tail[i+1:]
 448             mname = "%s.%s" % (m.__name__, head)
 449             m = self.import_it(head, mname, m)
 450             if not m:
 451                 raise ImportError, "No module named " + mname
 452         return m
 453
 454     def ensure_fromlist(self, m, fromlist, recursive=0):
 455         for sub in fromlist:
 456             if sub == "*":
 457                 if not recursive:
 458                     try:
 459                         all = m.__all__
 460                     except AttributeError:
 461                         pass
 462                     else:
 463                         self.ensure_fromlist(m, all, 1)
 464                 continue
 465             if sub != "*" and not hasattr(m, sub):
 466                 subname = "%s.%s" % (m.__name__, sub)
 467                 submod = self.import_it(sub, subname, m)
 468                 if not submod:
 469                     raise ImportError, "No module named " + subname
 470
 471     def import_it(self, partname, fqname, parent, force_load=0):
 472         if not partname:
 473             raise ValueError, "Empty module name"
 474         if not force_load:
 475             try:
 476                 return self.modules[fqname]
 477             except KeyError:
 478                 pass
 479         try:
 480             path = parent and parent.__path__
 481         except AttributeError:
 482             return None
 483         stuff = self.loader.find_module(partname, path)
 484         if not stuff:
 485             return None
 486         m = self.loader.load_module(fqname, stuff)
 487         if parent:
 488             setattr(parent, partname, m)
 489         return m
 490
 491     def reload(self, module):
 492         name = module.__name__
 493         if '.' not in name:
 494             return self.import_it(name, name, None, force_load=1)
 495         i = name.rfind('.')
 496         pname = name[:i]
 497         parent = self.modules[pname]
 498         return self.import_it(name[i+1:], name, parent, force_load=1)
 499
 500
 501 default_importer = None
 502 current_importer = None
 503
 504 def install(importer = None):
 505     global current_importer
 506     current_importer = importer or default_importer or ModuleImporter()
 507     current_importer.install()
 508
 509 def uninstall():
 510     global current_importer
 511     current_importer.uninstall()