Lib/ihooks.py

   1 """Import hook support.
   2
   3 Consistent use of this module will make it possible to change the
   4 different mechanisms involved in loading modules independently.
   5
   6 While the built-in module imp exports interfaces to the built-in
   7 module searching and loading algorithm, and it is possible to replace
   8 the built-in function __import__ in order to change the semantics of
   9 the import statement, until now it has been difficult to combine the
  10 effect of different __import__ hacks, like loading modules from URLs
  11 by rimport.py, or restricted execution by rexec.py.
  12
  13 This module defines three new concepts:
  14
  15 1) A "file system hooks" class provides an interface to a filesystem.
  16
  17 One hooks class is defined (Hooks), which uses the interface provided
  18 by standard modules os and os.path.  It should be used as the base
  19 class for other hooks classes.
  20
  21 2) A "module loader" class provides an interface to to search for a
  22 module in a search path and to load it.  It defines a method which
  23 searches for a module in a single directory; by overriding this method
  24 one can redefine the details of the search.  If the directory is None,
  25 built-in and frozen modules are searched instead.
  26
  27 Two module loader class are defined, both implementing the search
  28 strategy used by the built-in __import__ function: ModuleLoader uses
  29 the imp module's find_module interface, while HookableModuleLoader
  30 uses a file system hooks class to interact with the file system.  Both
  31 use the imp module's load_* interfaces to actually load the module.
  32
  33 3) A "module importer" class provides an interface to import a
  34 module, as well as interfaces to reload and unload a module.  It also
  35 provides interfaces to install and uninstall itself instead of the
  36 default __import__ and reload (and unload) functions.
  37
  38 One module importer class is defined (ModuleImporter), which uses a
  39 module loader instance passed in (by default HookableModuleLoader is
  40 instantiated).
  41
  42 The classes defined here should be used as base classes for extended
  43 functionality along those lines.
  44
  45 If a module mporter class supports dotted names, its import_module()
  46 must return a different value depending on whether it is called on
  47 behalf of a "from ... import ..." statement or not.  (This is caused
  48 by the way the __import__ hook is used by the Python interpreter.)  It
  49 would also do wise to install a different version of reload().
  50
  51 """
  52
  53
  54 import __builtin__
  55 import imp
  56 import os
  57 import sys
  58 import string
  59
  60
  61 VERBOSE = 0
  62
  63
  64 from imp import C_EXTENSION, PY_SOURCE, PY_COMPILED
  65 from imp import C_BUILTIN, PY_FROZEN, PKG_DIRECTORY
  66 BUILTIN_MODULE = C_BUILTIN
  67 FROZEN_MODULE = PY_FROZEN
  68
  69
  70 class _Verbose:
  71
  72     def __init__(self, verbose = VERBOSE):
  73         self.verbose = verbose
  74
  75     def get_verbose(self):
  76         return self.verbose
  77
  78     def set_verbose(self, verbose):
  79         self.verbose = verbose
  80
  81     # XXX The following is an experimental interface
  82
  83     def note(self, *args):
  84         if self.verbose:
  85             apply(self.message, args)
  86
  87     def message(self, format, *args):
  88         if args:
  89             print format%args
  90         else:
  91             print format
  92
  93
  94 class BasicModuleLoader(_Verbose):
  95
  96     """Basic module loader.
  97
  98     This provides the same functionality as built-in import.  It
  99     doesn't deal with checking sys.modules -- all it provides is
 100     find_module() and a load_module(), as well as find_module_in_dir()
 101     which searches just one directory, and can be overridden by a
 102     derived class to change the module search algorithm when the basic
 103     dependency on sys.path is unchanged.
 104
 105     The interface is a little more convenient than imp's:
 106     find_module(name, [path]) returns None or 'stuff', and
 107     load_module(name, stuff) loads the module.
 108
 109     """
 110
 111     def find_module(self, name, path = None):
 112         if path is None:
 113             path = [None] + self.default_path()
 114         for dir in path:
 115             stuff = self.find_module_in_dir(name, dir)
 116             if stuff: return stuff
 117         return None
 118
 119     def default_path(self):
 120         return sys.path
 121
 122     def find_module_in_dir(self, name, dir):
 123         if dir is None:
 124             return self.find_builtin_module(name)
 125         else:
 126             try:
 127                 return imp.find_module(name, [dir])
 128             except ImportError:
 129                 return None
 130
 131     def find_builtin_module(self, name):
 132         # XXX frozen packages?
 133         if imp.is_builtin(name):
 134             return None, '', ('', '', BUILTIN_MODULE)
 135         if imp.is_frozen(name):
 136             return None, '', ('', '', FROZEN_MODULE)
 137         return None
 138
 139     def load_module(self, name, stuff):
 140         file, filename, info = stuff
 141         try:
 142             return imp.load_module(name, file, filename, info)
 143         finally:
 144             if file: file.close()
 145
 146
 147 class Hooks(_Verbose):
 148
 149     """Hooks into the filesystem and interpreter.
 150
 151     By deriving a subclass you can redefine your filesystem interface,
 152     e.g. to merge it with the URL space.
 153
 154     This base class behaves just like the native filesystem.
 155
 156     """
 157
 158     # imp interface
 159     def get_suffixes(self): return imp.get_suffixes()
 160     def new_module(self, name): return imp.new_module(name)
 161     def is_builtin(self, name): return imp.is_builtin(name)
 162     def init_builtin(self, name): return imp.init_builtin(name)
 163     def is_frozen(self, name): return imp.is_frozen(name)
 164     def init_frozen(self, name): return imp.init_frozen(name)
 165     def get_frozen_object(self, name): return imp.get_frozen_object(name)
 166     def load_source(self, name, filename, file=None):
 167         return imp.load_source(name, filename, file)
 168     def load_compiled(self, name, filename, file=None):
 169         return imp.load_compiled(name, filename, file)
 170     def load_dynamic(self, name, filename, file=None):
 171         return imp.load_dynamic(name, filename, file)
 172     def load_package(self, name, filename, file=None):
 173         return imp.load_module(name, file, filename, ("", "", PKG_DIRECTORY))
 174
 175     def add_module(self, name):
 176         d = self.modules_dict()
 177         if d.has_key(name): return d[name]
 178         d[name] = m = self.new_module(name)
 179         return m
 180
 181     # sys interface
 182     def modules_dict(self): return sys.modules
 183     def default_path(self): return sys.path
 184
 185     def path_split(self, x): return os.path.split(x)
 186     def path_join(self, x, y): return os.path.join(x, y)
 187     def path_isabs(self, x): return os.path.isabs(x)
 188     # etc.
 189
 190     def path_exists(self, x): return os.path.exists(x)
 191     def path_isdir(self, x): return os.path.isdir(x)
 192     def path_isfile(self, x): return os.path.isfile(x)
 193     def path_islink(self, x): return os.path.islink(x)
 194     # etc.
 195
 196     def openfile(self, *x): return apply(open, x)
 197     openfile_error = IOError
 198     def listdir(self, x): return os.listdir(x)
 199     listdir_error = os.error
 200     # etc.
 201
 202
 203 class ModuleLoader(BasicModuleLoader):
 204
 205     """Default module loader; uses file system hooks.
 206
 207     By defining suitable hooks, you might be able to load modules from
 208     other sources than the file system, e.g. from compressed or
 209     encrypted files, tar files or (if you're brave!) URLs.
 210
 211     """
 212
 213     def __init__(self, hooks = None, verbose = VERBOSE):
 214         BasicModuleLoader.__init__(self, verbose)
 215         self.hooks = hooks or Hooks(verbose)
 216
 217     def default_path(self):
 218         return self.hooks.default_path()
 219
 220     def modules_dict(self):
 221         return self.hooks.modules_dict()
 222
 223     def get_hooks(self):
 224         return self.hooks
 225
 226     def set_hooks(self, hooks):
 227         self.hooks = hooks
 228
 229     def find_builtin_module(self, name):
 230         # XXX frozen packages?
 231         if self.hooks.is_builtin(name):
 232             return None, '', ('', '', BUILTIN_MODULE)
 233         if self.hooks.is_frozen(name):
 234             return None, '', ('', '', FROZEN_MODULE)
 235         return None
 236
 237     def find_module_in_dir(self, name, dir, allow_packages=1):
 238         if dir is None:
 239             return self.find_builtin_module(name)
 240         if allow_packages:
 241             fullname = self.hooks.path_join(dir, name)
 242             if self.hooks.path_isdir(fullname):
 243                 stuff = self.find_module_in_dir("__init__", fullname, 0)
 244                 if stuff:
 245                     file = stuff[0]
 246                     if file: file.close()
 247                     return None, fullname, ('', '', PKG_DIRECTORY)
 248         for info in self.hooks.get_suffixes():
 249             suff, mode, type = info
 250             fullname = self.hooks.path_join(dir, name+suff)
 251             try:
 252                 fp = self.hooks.openfile(fullname, mode)
 253                 return fp, fullname, info
 254             except self.hooks.openfile_error:
 255                 pass
 256         return None
 257
 258     def load_module(self, name, stuff):
 259         file, filename, info = stuff
 260         (suff, mode, type) = info
 261         try:
 262             if type == BUILTIN_MODULE:
 263                 return self.hooks.init_builtin(name)
 264             if type == FROZEN_MODULE:
 265                 return self.hooks.init_frozen(name)
 266             if type == C_EXTENSION:
 267                 m = self.hooks.load_dynamic(name, filename, file)
 268             elif type == PY_SOURCE:
 269                 m = self.hooks.load_source(name, filename, file)
 270             elif type == PY_COMPILED:
 271                 m = self.hooks.load_compiled(name, filename, file)
 272             elif type == PKG_DIRECTORY:
 273                 m = self.hooks.load_package(name, filename, file)
 274             else:
 275                 raise ImportError, "Unrecognized module type (%s) for %s" % \
 276                       (`type`, name)
 277         finally:
 278             if file: file.close()
 279         m.__file__ = filename
 280         return m
 281
 282
 283 class FancyModuleLoader(ModuleLoader):
 284
 285     """Fancy module loader -- parses and execs the code itself."""
 286
 287     def load_module(self, name, stuff):
 288         file, filename, (suff, mode, type) = stuff
 289         realfilename = filename
 290         path = None
 291
 292         if type == PKG_DIRECTORY:
 293             initstuff = self.find_module_in_dir("__init__", filename, 0)
 294             if not initstuff:
 295                 raise ImportError, "No __init__ module in package %s" % name
 296             initfile, initfilename, initinfo = initstuff
 297             initsuff, initmode, inittype = initinfo
 298             if inittype not in (PY_COMPILED, PY_SOURCE):
 299                 if initfile: initfile.close()
 300                 raise ImportError, \
 301                     "Bad type (%s) for __init__ module in package %s" % (
 302                     `inittype`, name)
 303             path = [filename]
 304             file = initfile
 305             realfilename = initfilename
 306             type = inittype
 307
 308         if type == FROZEN_MODULE:
 309             code = self.hooks.get_frozen_object(name)
 310         elif type == PY_COMPILED:
 311             import marshal
 312             file.seek(8)
 313             code = marshal.load(file)
 314         elif type == PY_SOURCE:
 315             data = file.read()
 316             code = compile(data, realfilename, 'exec')
 317         else:
 318             return ModuleLoader.load_module(self, name, stuff)
 319
 320         m = self.hooks.add_module(name)
 321         if path:
 322             m.__path__ = path
 323         m.__file__ = filename
 324         exec code in m.__dict__
 325         return m
 326
 327
 328 class BasicModuleImporter(_Verbose):
 329
 330     """Basic module importer; uses module loader.
 331
 332     This provides basic import facilities but no package imports.
 333
 334     """
 335
 336     def __init__(self, loader = None, verbose = VERBOSE):
 337         _Verbose.__init__(self, verbose)
 338         self.loader = loader or ModuleLoader(None, verbose)
 339         self.modules = self.loader.modules_dict()
 340
 341     def get_loader(self):
 342         return self.loader
 343
 344     def set_loader(self, loader):
 345         self.loader = loader
 346
 347     def get_hooks(self):
 348         return self.loader.get_hooks()
 349
 350     def set_hooks(self, hooks):
 351         return self.loader.set_hooks(hooks)
 352
 353     def import_module(self, name, globals={}, locals={}, fromlist=[]):
 354         if self.modules.has_key(name):
 355             return self.modules[name] # Fast path
 356         stuff = self.loader.find_module(name)
 357         if not stuff:
 358             raise ImportError, "No module named %s" % name
 359         return self.loader.load_module(name, stuff)
 360
 361     def reload(self, module, path = None):
 362         name = module.__name__
 363         stuff = self.loader.find_module(name, path)
 364         if not stuff:
 365             raise ImportError, "Module %s not found for reload" % name
 366         return self.loader.load_module(name, stuff)
 367
 368     def unload(self, module):
 369         del self.modules[module.__name__]
 370         # XXX Should this try to clear the module's namespace?
 371
 372     def install(self):
 373         self.save_import_module = __builtin__.__import__
 374         self.save_reload = __builtin__.reload
 375         if not hasattr(__builtin__, 'unload'):
 376             __builtin__.unload = None
 377         self.save_unload = __builtin__.unload
 378         __builtin__.__import__ = self.import_module
 379         __builtin__.reload = self.reload
 380         __builtin__.unload = self.unload
 381
 382     def uninstall(self):
 383         __builtin__.__import__ = self.save_import_module
 384         __builtin__.reload = self.save_reload
 385         __builtin__.unload = self.save_unload
 386         if not __builtin__.unload:
 387             del __builtin__.unload
 388
 389
 390 class ModuleImporter(BasicModuleImporter):
 391
 392     """A module importer that supports packages."""
 393
 394     def import_module(self, name, globals=None, locals=None, fromlist=None):
 395         parent = self.determine_parent(globals)
 396         q, tail = self.find_head_package(parent, name)
 397         m = self.load_tail(q, tail)
 398         if not fromlist:
 399             return q
 400         if hasattr(m, "__path__"):
 401             self.ensure_fromlist(m, fromlist)
 402         return m
 403
 404     def determine_parent(self, globals):
 405         if not globals or not globals.has_key("__name__"):
 406             return None
 407         pname = globals['__name__']
 408         if globals.has_key("__path__"):
 409             parent = self.modules[pname]
 410             assert globals is parent.__dict__
 411             return parent
 412         if '.' in pname:
 413             i = string.rfind(pname, '.')
 414             pname = pname[:i]
 415             parent = self.modules[pname]
 416             assert parent.__name__ == pname
 417             return parent
 418         return None
 419
 420     def find_head_package(self, parent, name):
 421         if '.' in name:
 422             i = string.find(name, '.')
 423             head = name[:i]
 424             tail = name[i+1:]
 425         else:
 426             head = name
 427             tail = ""
 428         if parent:
 429             qname = "%s.%s" % (parent.__name__, head)
 430         else:
 431             qname = head
 432         q = self.import_it(head, qname, parent)
 433         if q: return q, tail
 434         if parent:
 435             qname = head
 436             parent = None
 437             q = self.import_it(head, qname, parent)
 438             if q: return q, tail
 439         raise ImportError, "No module named " + qname
 440
 441     def load_tail(self, q, tail):
 442         m = q
 443         while tail:
 444             i = string.find(tail, '.')
 445             if i < 0: i = len(tail)
 446             head, tail = tail[:i], tail[i+1:]
 447             mname = "%s.%s" % (m.__name__, head)
 448             m = self.import_it(head, mname, m)
 449             if not m:
 450                 raise ImportError, "No module named " + mname
 451         return m
 452
 453     def ensure_fromlist(self, m, fromlist, recursive=0):
 454         for sub in fromlist:
 455             if sub == "*":
 456                 if not recursive:
 457                     try:
 458                         all = m.__all__
 459                     except AttributeError:
 460                         pass
 461                     else:
 462                         self.ensure_fromlist(m, all, 1)
 463                 continue
 464             if sub != "*" and not hasattr(m, sub):
 465                 subname = "%s.%s" % (m.__name__, sub)
 466                 submod = self.import_it(sub, subname, m)
 467                 if not submod:
 468                     raise ImportError, "No module named " + subname
 469
 470     def import_it(self, partname, fqname, parent):
 471         if not partname:
 472             raise ValueError, "Empty module name"
 473         try:
 474             return self.modules[fqname]
 475         except KeyError:
 476             pass
 477         try:
 478             path = parent and parent.__path__
 479         except AttributeError:
 480             return None
 481         stuff = self.loader.find_module(partname, path)
 482         if not stuff:
 483             return None
 484         m = self.loader.load_module(fqname, stuff)
 485         if parent:
 486             setattr(parent, partname, m)
 487         return m
 488
 489     def reload(self, module):
 490         name = module.__name__
 491         if '.' not in name:
 492             return self.import_it(name, name, None)
 493         i = string.rfind(name, '.')
 494         pname = name[:i]
 495         parent = self.modules[pname]
 496         return self.import_it(name[i+1:], name, parent)
 497
 498
 499 default_importer = None
 500 current_importer = None
 501
 502 def install(importer = None):
 503     global current_importer
 504     current_importer = importer or default_importer or ModuleImporter()
 505     current_importer.install()
 506
 507 def uninstall():
 508     global current_importer
 509     current_importer.uninstall()