Lib/imputil.py

   1 #
   2 # imputil.py: import utilities
   3 #
   4
   5 ### docco needed here and in Docs/ ...
   6
   7 # note: avoid importing non-builtin modules
   8 import imp                      ### not available in JPython?
   9 import sys
  10 import strop
  11 import __builtin__
  12
  13 # for the DirectoryImporter
  14 import struct
  15 import marshal
  16
  17 __all__ = ["ImportManager","Importer","BuiltinImporter"]
  18
  19 _StringType = type('')
  20 _ModuleType = type(sys)         ### doesn't work in JPython...
  21
  22 class ImportManager:
  23     "Manage the import process."
  24
  25     def install(self, namespace=vars(__builtin__)):
  26         "Install this ImportManager into the specified namespace."
  27
  28         if isinstance(namespace, _ModuleType):
  29             namespace = vars(namespace)
  30
  31         ### Note that we have no notion of "uninstall" or "chaining"
  32
  33         namespace['__import__'] = self._import_hook
  34         ### fix this
  35         #namespace['reload'] = self._reload_hook
  36
  37     def add_suffix(self, suffix, importFunc):
  38         assert callable(importFunc)
  39         self.fs_imp.add_suffix(suffix, importFunc)
  40
  41     ######################################################################
  42     #
  43     # PRIVATE METHODS
  44     #
  45
  46     clsFilesystemImporter = None
  47
  48     def __init__(self, fs_imp=None):
  49         # we're definitely going to be importing something in the future,
  50         # so let's just load the OS-related facilities.
  51         if not _os_stat:
  52             _os_bootstrap()
  53
  54         # This is the Importer that we use for grabbing stuff from the
  55         # filesystem. It defines one more method (import_from_dir) for our use.
  56         if not fs_imp:
  57             cls = self.clsFilesystemImporter or _FilesystemImporter
  58             fs_imp = cls()
  59         self.fs_imp = fs_imp
  60
  61         # Initialize the set of suffixes that we recognize and import.
  62         # The default will import dynamic-load modules first, followed by
  63         # .py files (or a .py file's cached bytecode)
  64         for desc in imp.get_suffixes():
  65             if desc[2] == imp.C_EXTENSION:
  66                 self.add_suffix(desc[0],
  67                                 DynLoadSuffixImporter(desc).import_file)
  68         self.add_suffix('.py', py_suffix_importer)
  69
  70     def _import_hook(self, fqname, globals=None, locals=None, fromlist=None):
  71         """Python calls this hook to locate and import a module."""
  72
  73         parts = strop.split(fqname, '.')
  74
  75         # determine the context of this import
  76         parent = self._determine_import_context(globals)
  77
  78         # if there is a parent, then its importer should manage this import
  79         if parent:
  80             module = parent.__importer__._do_import(parent, parts, fromlist)
  81             if module:
  82                 return module
  83
  84         # has the top module already been imported?
  85         try:
  86             top_module = sys.modules[parts[0]]
  87         except KeyError:
  88
  89             # look for the topmost module
  90             top_module = self._import_top_module(parts[0])
  91             if not top_module:
  92                 # the topmost module wasn't found at all.
  93                 raise ImportError, 'No module named ' + fqname
  94
  95         # fast-path simple imports
  96         if len(parts) == 1:
  97             if not fromlist:
  98                 return top_module
  99
 100             if not top_module.__dict__.get('__ispkg__'):
 101                 # __ispkg__ isn't defined (the module was not imported by us),
 102                 # or it is zero.
 103                 #
 104                 # In the former case, there is no way that we could import
 105                 # sub-modules that occur in the fromlist (but we can't raise an
 106                 # error because it may just be names) because we don't know how
 107                 # to deal with packages that were imported by other systems.
 108                 #
 109                 # In the latter case (__ispkg__ == 0), there can't be any sub-
 110                 # modules present, so we can just return.
 111                 #
 112                 # In both cases, since len(parts) == 1, the top_module is also
 113                 # the "bottom" which is the defined return when a fromlist
 114                 # exists.
 115                 return top_module
 116
 117         importer = top_module.__dict__.get('__importer__')
 118         if importer:
 119             return importer._finish_import(top_module, parts[1:], fromlist)
 120
 121         # If the importer does not exist, then we have to bail. A missing
 122         # importer means that something else imported the module, and we have
 123         # no knowledge of how to get sub-modules out of the thing.
 124         raise ImportError, 'No module named ' + fqname
 125
 126     def _determine_import_context(self, globals):
 127         """Returns the context in which a module should be imported.
 128
 129         The context could be a loaded (package) module and the imported module
 130         will be looked for within that package. The context could also be None,
 131         meaning there is no context -- the module should be looked for as a
 132         "top-level" module.
 133         """
 134
 135         if not globals or not globals.get('__importer__'):
 136             # globals does not refer to one of our modules or packages. That
 137             # implies there is no relative import context (as far as we are
 138             # concerned), and it should just pick it off the standard path.
 139             return None
 140
 141         # The globals refer to a module or package of ours. It will define
 142         # the context of the new import. Get the module/package fqname.
 143         parent_fqname = globals['__name__']
 144
 145         # if a package is performing the import, then return itself (imports
 146         # refer to pkg contents)
 147         if globals['__ispkg__']:
 148             parent = sys.modules[parent_fqname]
 149             assert globals is parent.__dict__
 150             return parent
 151
 152         i = strop.rfind(parent_fqname, '.')
 153
 154         # a module outside of a package has no particular import context
 155         if i == -1:
 156             return None
 157
 158         # if a module in a package is performing the import, then return the
 159         # package (imports refer to siblings)
 160         parent_fqname = parent_fqname[:i]
 161         parent = sys.modules[parent_fqname]
 162         assert parent.__name__ == parent_fqname
 163         return parent
 164
 165     def _import_top_module(self, name):
 166         # scan sys.path looking for a location in the filesystem that contains
 167         # the module, or an Importer object that can import the module.
 168         for item in sys.path:
 169             if isinstance(item, _StringType):
 170                 module = self.fs_imp.import_from_dir(item, name)
 171             else:
 172                 module = item.import_top(name)
 173             if module:
 174                 return module
 175         return None
 176
 177     def _reload_hook(self, module):
 178         "Python calls this hook to reload a module."
 179
 180         # reloading of a module may or may not be possible (depending on the
 181         # importer), but at least we can validate that it's ours to reload
 182         importer = module.__dict__.get('__importer__')
 183         if not importer:
 184             ### oops. now what...
 185             pass
 186
 187         # okay. it is using the imputil system, and we must delegate it, but
 188         # we don't know what to do (yet)
 189         ### we should blast the module dict and do another get_code(). need to
 190         ### flesh this out and add proper docco...
 191         raise SystemError, "reload not yet implemented"
 192
 193
 194 class Importer:
 195     "Base class for replacing standard import functions."
 196
 197     def import_top(self, name):
 198         "Import a top-level module."
 199         return self._import_one(None, name, name)
 200
 201     ######################################################################
 202     #
 203     # PRIVATE METHODS
 204     #
 205     def _finish_import(self, top, parts, fromlist):
 206         # if "a.b.c" was provided, then load the ".b.c" portion down from
 207         # below the top-level module.
 208         bottom = self._load_tail(top, parts)
 209
 210         # if the form is "import a.b.c", then return "a"
 211         if not fromlist:
 212             # no fromlist: return the top of the import tree
 213             return top
 214
 215         # the top module was imported by self.
 216         #
 217         # this means that the bottom module was also imported by self (just
 218         # now, or in the past and we fetched it from sys.modules).
 219         #
 220         # since we imported/handled the bottom module, this means that we can
 221         # also handle its fromlist (and reliably use __ispkg__).
 222
 223         # if the bottom node is a package, then (potentially) import some
 224         # modules.
 225         #
 226         # note: if it is not a package, then "fromlist" refers to names in
 227         #       the bottom module rather than modules.
 228         # note: for a mix of names and modules in the fromlist, we will
 229         #       import all modules and insert those into the namespace of
 230         #       the package module. Python will pick up all fromlist names
 231         #       from the bottom (package) module; some will be modules that
 232         #       we imported and stored in the namespace, others are expected
 233         #       to be present already.
 234         if bottom.__ispkg__:
 235             self._import_fromlist(bottom, fromlist)
 236
 237         # if the form is "from a.b import c, d" then return "b"
 238         return bottom
 239
 240     def _import_one(self, parent, modname, fqname):
 241         "Import a single module."
 242
 243         # has the module already been imported?
 244         try:
 245             return sys.modules[fqname]
 246         except KeyError:
 247             pass
 248
 249         # load the module's code, or fetch the module itself
 250         result = self.get_code(parent, modname, fqname)
 251         if result is None:
 252             return None
 253
 254         module = self._process_result(result, fqname)
 255
 256         # insert the module into its parent
 257         if parent:
 258             setattr(parent, modname, module)
 259         return module
 260
 261     def _process_result(self, (ispkg, code, values), fqname):
 262         # did get_code() return an actual module? (rather than a code object)
 263         is_module = isinstance(code, _ModuleType)
 264
 265         # use the returned module, or create a new one to exec code into
 266         if is_module:
 267             module = code
 268         else:
 269             module = imp.new_module(fqname)
 270
 271         ### record packages a bit differently??
 272         module.__importer__ = self
 273         module.__ispkg__ = ispkg
 274
 275         # insert additional values into the module (before executing the code)
 276         module.__dict__.update(values)
 277
 278         # the module is almost ready... make it visible
 279         sys.modules[fqname] = module
 280
 281         # execute the code within the module's namespace
 282         if not is_module:
 283             exec code in module.__dict__
 284
 285         # fetch from sys.modules instead of returning module directly.
 286         return sys.modules[fqname]
 287
 288     def _load_tail(self, m, parts):
 289         """Import the rest of the modules, down from the top-level module.
 290
 291         Returns the last module in the dotted list of modules.
 292         """
 293         for part in parts:
 294             fqname = "%s.%s" % (m.__name__, part)
 295             m = self._import_one(m, part, fqname)
 296             if not m:
 297                 raise ImportError, "No module named " + fqname
 298         return m
 299
 300     def _import_fromlist(self, package, fromlist):
 301         'Import any sub-modules in the "from" list.'
 302
 303         # if '*' is present in the fromlist, then look for the '__all__'
 304         # variable to find additional items (modules) to import.
 305         if '*' in fromlist:
 306             fromlist = list(fromlist) + \
 307                        list(package.__dict__.get('__all__', []))
 308
 309         for sub in fromlist:
 310             # if the name is already present, then don't try to import it (it
 311             # might not be a module!).
 312             if sub != '*' and not hasattr(package, sub):
 313                 subname = "%s.%s" % (package.__name__, sub)
 314                 submod = self._import_one(package, sub, subname)
 315                 if not submod:
 316                     raise ImportError, "cannot import name " + subname
 317
 318     def _do_import(self, parent, parts, fromlist):
 319         """Attempt to import the module relative to parent.
 320
 321         This method is used when the import context specifies that <self>
 322         imported the parent module.
 323         """
 324         top_name = parts[0]
 325         top_fqname = parent.__name__ + '.' + top_name
 326         top_module = self._import_one(parent, top_name, top_fqname)
 327         if not top_module:
 328             # this importer and parent could not find the module (relatively)
 329             return None
 330
 331         return self._finish_import(top_module, parts[1:], fromlist)
 332
 333     ######################################################################
 334     #
 335     # METHODS TO OVERRIDE
 336     #
 337     def get_code(self, parent, modname, fqname):
 338         """Find and retrieve the code for the given module.
 339
 340         parent specifies a parent module to define a context for importing. It
 341         may be None, indicating no particular context for the search.
 342
 343         modname specifies a single module (not dotted) within the parent.
 344
 345         fqname specifies the fully-qualified module name. This is a
 346         (potentially) dotted name from the "root" of the module namespace
 347         down to the modname.
 348         If there is no parent, then modname==fqname.
 349
 350         This method should return None, or a 3-tuple.
 351
 352         * If the module was not found, then None should be returned.
 353
 354         * The first item of the 2- or 3-tuple should be the integer 0 or 1,
 355             specifying whether the module that was found is a package or not.
 356
 357         * The second item is the code object for the module (it will be
 358             executed within the new module's namespace). This item can also
 359             be a fully-loaded module object (e.g. loaded from a shared lib).
 360
 361         * The third item is a dictionary of name/value pairs that will be
 362             inserted into new module before the code object is executed. This
 363             is provided in case the module's code expects certain values (such
 364             as where the module was found). When the second item is a module
 365             object, then these names/values will be inserted *after* the module
 366             has been loaded/initialized.
 367         """
 368         raise RuntimeError, "get_code not implemented"
 369
 370
 371 ######################################################################
 372 #
 373 # Some handy stuff for the Importers
 374 #
 375
 376 # byte-compiled file suffix character
 377 _suffix_char = __debug__ and 'c' or 'o'
 378
 379 # byte-compiled file suffix
 380 _suffix = '.py' + _suffix_char
 381
 382 def _compile(pathname, timestamp):
 383     """Compile (and cache) a Python source file.
 384
 385     The file specified by <pathname> is compiled to a code object and
 386     returned.
 387
 388     Presuming the appropriate privileges exist, the bytecodes will be
 389     saved back to the filesystem for future imports. The source file's
 390     modification timestamp must be provided as a Long value.
 391     """
 392     codestring = open(pathname, 'r').read()
 393     if codestring and codestring[-1] != '\n':
 394         codestring = codestring + '\n'
 395     code = __builtin__.compile(codestring, pathname, 'exec')
 396
 397     # try to cache the compiled code
 398     try:
 399         f = open(pathname + _suffix_char, 'wb')
 400     except IOError:
 401         pass
 402     else:
 403         f.write('\0\0\0\0')
 404         f.write(struct.pack('<I', timestamp))
 405         marshal.dump(code, f)
 406         f.flush()
 407         f.seek(0, 0)
 408         f.write(imp.get_magic())
 409         f.close()
 410
 411     return code
 412
 413 _os_stat = _os_path_join = None
 414 def _os_bootstrap():
 415     "Set up 'os' module replacement functions for use during import bootstrap."
 416
 417     names = sys.builtin_module_names
 418
 419     join = None
 420     if 'posix' in names:
 421         sep = '/'
 422         from posix import stat
 423     elif 'nt' in names:
 424         sep = '\\'
 425         from nt import stat
 426     elif 'dos' in names:
 427         sep = '\\'
 428         from dos import stat
 429     elif 'os2' in names:
 430         sep = '\\'
 431         from os2 import stat
 432     elif 'mac' in names:
 433         from mac import stat
 434         def join(a, b):
 435             if a == '':
 436                 return b
 437             path = s
 438             if ':' not in a:
 439                 a = ':' + a
 440             if a[-1:] != ':':
 441                 a = a + ':'
 442             return a + b
 443     else:
 444         raise ImportError, 'no os specific module found'
 445
 446     if join is None:
 447         def join(a, b, sep=sep):
 448             if a == '':
 449                 return b
 450             lastchar = a[-1:]
 451             if lastchar == '/' or lastchar == sep:
 452                 return a + b
 453             return a + sep + b
 454
 455     global _os_stat
 456     _os_stat = stat
 457
 458     global _os_path_join
 459     _os_path_join = join
 460
 461 def _os_path_isdir(pathname):
 462     "Local replacement for os.path.isdir()."
 463     try:
 464         s = _os_stat(pathname)
 465     except OSError:
 466         return None
 467     return (s[0] & 0170000) == 0040000
 468
 469 def _timestamp(pathname):
 470     "Return the file modification time as a Long."
 471     try:
 472         s = _os_stat(pathname)
 473     except OSError:
 474         return None
 475     return long(s[8])
 476
 477
 478 ######################################################################
 479 #
 480 # Emulate the import mechanism for builtin and frozen modules
 481 #
 482 class BuiltinImporter(Importer):
 483     def get_code(self, parent, modname, fqname):
 484         if parent:
 485             # these modules definitely do not occur within a package context
 486             return None
 487
 488         # look for the module
 489         if imp.is_builtin(modname):
 490             type = imp.C_BUILTIN
 491         elif imp.is_frozen(modname):
 492             type = imp.PY_FROZEN
 493         else:
 494             # not found
 495             return None
 496
 497         # got it. now load and return it.
 498         module = imp.load_module(modname, None, modname, ('', '', type))
 499         return 0, module, { }
 500
 501
 502 ######################################################################
 503 #
 504 # Internal importer used for importing from the filesystem
 505 #
 506 class _FilesystemImporter(Importer):
 507     def __init__(self):
 508         self.suffixes = [ ]
 509
 510     def add_suffix(self, suffix, importFunc):
 511         assert callable(importFunc)
 512         self.suffixes.append((suffix, importFunc))
 513
 514     def import_from_dir(self, dir, fqname):
 515         result = self._import_pathname(_os_path_join(dir, fqname), fqname)
 516         if result:
 517             return self._process_result(result, fqname)
 518         return None
 519
 520     def get_code(self, parent, modname, fqname):
 521         # This importer is never used with an empty parent. Its existence is
 522         # private to the ImportManager. The ImportManager uses the
 523         # import_from_dir() method to import top-level modules/packages.
 524         # This method is only used when we look for a module within a package.
 525         assert parent
 526
 527         return self._import_pathname(_os_path_join(parent.__pkgdir__, modname),
 528                                      fqname)
 529
 530     def _import_pathname(self, pathname, fqname):
 531         if _os_path_isdir(pathname):
 532             result = self._import_pathname(_os_path_join(pathname, '__init__'),
 533                                            fqname)
 534             if result:
 535                 values = result[2]
 536                 values['__pkgdir__'] = pathname
 537                 values['__path__'] = [ pathname ]
 538                 return 1, result[1], values
 539             return None
 540
 541         for suffix, importFunc in self.suffixes:
 542             filename = pathname + suffix
 543             try:
 544                 finfo = _os_stat(filename)
 545             except OSError:
 546                 pass
 547             else:
 548                 return importFunc(filename, finfo, fqname)
 549         return None
 550
 551 ######################################################################
 552 #
 553 # SUFFIX-BASED IMPORTERS
 554 #
 555
 556 def py_suffix_importer(filename, finfo, fqname):
 557     file = filename[:-3] + _suffix
 558     t_py = long(finfo[8])
 559     t_pyc = _timestamp(file)
 560
 561     code = None
 562     if t_pyc is not None and t_pyc >= t_py:
 563         f = open(file, 'rb')
 564         if f.read(4) == imp.get_magic():
 565             t = struct.unpack('<I', f.read(4))[0]
 566             if t == t_py:
 567                 code = marshal.load(f)
 568         f.close()
 569     if code is None:
 570         file = filename
 571         code = _compile(file, t_py)
 572
 573     return 0, code, { '__file__' : file }
 574
 575 class DynLoadSuffixImporter:
 576     def __init__(self, desc):
 577         self.desc = desc
 578
 579     def import_file(self, filename, finfo, fqname):
 580         fp = open(filename, self.desc[1])
 581         module = imp.load_module(fqname, fp, filename, self.desc)
 582         module.__file__ = filename
 583         return 0, module, { }
 584
 585
 586 ######################################################################
 587
 588 def _print_importers():
 589     items = sys.modules.items()
 590     items.sort()
 591     for name, module in items:
 592         if module:
 593             print name, module.__dict__.get('__importer__', '-- no importer')
 594         else:
 595             print name, '-- non-existent module'
 596
 597 def _test_revamp():
 598     ImportManager().install()
 599     sys.path.insert(0, BuiltinImporter())
 600
 601 ######################################################################
 602
 603 #
 604 # TODO
 605 #
 606 # from Finn Bock:
 607 #   remove use of "strop" -- not available in JPython
 608 #   type(sys) is not a module in JPython. what to use instead?
 609 #   imp.C_EXTENSION is not in JPython. same for get_suffixes and new_module
 610 #
 611 #   given foo.py of:
 612 #      import sys
 613 #      sys.modules['foo'] = sys
 614 #
 615 #   ---- standard import mechanism
 616 #   >>> import foo
 617 #   >>> foo
 618 #   <module 'sys' (built-in)>
 619 #
 620 #   ---- revamped import mechanism
 621 #   >>> import imputil
 622 #   >>> imputil._test_revamp()
 623 #   >>> import foo
 624 #   >>> foo
 625 #   <module 'foo' from 'foo.py'>
 626 #
 627 #
 628 # from MAL:
 629 #   should BuiltinImporter exist in sys.path or hard-wired in ImportManager?
 630 #   need __path__ processing
 631 #   performance
 632 #   move chaining to a subclass [gjs: it's been nuked]
 633 #   avoid strop
 634 #   deinstall should be possible
 635 #   query mechanism needed: is a specific Importer installed?
 636 #   py/pyc/pyo piping hooks to filter/process these files
 637 #   wish list:
 638 #     distutils importer hooked to list of standard Internet repositories
 639 #     module->file location mapper to speed FS-based imports
 640 #     relative imports
 641 #     keep chaining so that it can play nice with other import hooks
 642 #
 643 # from Gordon:
 644 #   push MAL's mapper into sys.path[0] as a cache (hard-coded for apps)
 645 #
 646 # from Guido:
 647 #   need to change sys.* references for rexec environs
 648 #   need hook for MAL's walk-me-up import strategy, or Tim's absolute strategy
 649 #   watch out for sys.modules[...] is None
 650 #   flag to force absolute imports? (speeds _determine_import_context and
 651 #       checking for a relative module)
 652 #   insert names of archives into sys.path  (see quote below)
 653 #   note: reload does NOT blast module dict
 654 #   shift import mechanisms and policies around; provide for hooks, overrides
 655 #       (see quote below)
 656 #   add get_source stuff
 657 #   get_topcode and get_subcode
 658 #   CRLF handling in _compile
 659 #   race condition in _compile
 660 #   refactoring of os.py to deal with _os_bootstrap problem
 661 #   any special handling to do for importing a module with a SyntaxError?
 662 #       (e.g. clean up the traceback)
 663 #   implement "domain" for path-type functionality using pkg namespace
 664 #       (rather than FS-names like __path__)
 665 #   don't use the word "private"... maybe "internal"
 666 #
 667 #
 668 # Guido's comments on sys.path caching:
 669 #
 670 # We could cache this in a dictionary: the ImportManager can have a
 671 # cache dict mapping pathnames to importer objects, and a separate
 672 # method for coming up with an importer given a pathname that's not yet
 673 # in the cache.  The method should do a stat and/or look at the
 674 # extension to decide which importer class to use; you can register new
 675 # importer classes by registering a suffix or a Boolean function, plus a
 676 # class.  If you register a new importer class, the cache is zapped.
 677 # The cache is independent from sys.path (but maintained per
 678 # ImportManager instance) so that rearrangements of sys.path do the
 679 # right thing.  If a path is dropped from sys.path the corresponding
 680 # cache entry is simply no longer used.
 681 #
 682 # My/Guido's comments on factoring ImportManager and Importer:
 683 #
 684 # > However, we still have a tension occurring here:
 685 # >
 686 # > 1) implementing policy in ImportManager assists in single-point policy
 687 # >    changes for app/rexec situations
 688 # > 2) implementing policy in Importer assists in package-private policy
 689 # >    changes for normal, operating conditions
 690 # >
 691 # > I'll see if I can sort out a way to do this. Maybe the Importer class will
 692 # > implement the methods (which can be overridden to change policy) by
 693 # > delegating to ImportManager.
 694 #
 695 # Maybe also think about what kind of policies an Importer would be
 696 # likely to want to change.  I have a feeling that a lot of the code
 697 # there is actually not so much policy but a *necessity* to get things
 698 # working given the calling conventions for the __import__ hook: whether
 699 # to return the head or tail of a dotted name, or when to do the "finish
 700 # fromlist" stuff.
 701 #