third_party/cython/src/pyximport/pyximport.py

   1 """
   2 Import hooks; when installed with the install() function, these hooks
   3 allow importing .pyx files as if they were Python modules.
   4
   5 If you want the hook installed every time you run Python
   6 you can add it to your Python version by adding these lines to
   7 sitecustomize.py (which you can create from scratch in site-packages
   8 if it doesn't exist there or somewhere else on your python path)::
   9
  10     import pyximport
  11     pyximport.install()
  12
  13 For instance on the Mac with a non-system Python 2.3, you could create
  14 sitecustomize.py with only those two lines at
  15 /usr/local/lib/python2.3/site-packages/sitecustomize.py .
  16
  17 A custom distutils.core.Extension instance and setup() args
  18 (Distribution) for for the build can be defined by a <modulename>.pyxbld
  19 file like:
  20
  21 # examplemod.pyxbld
  22 def make_ext(modname, pyxfilename):
  23     from distutils.extension import Extension
  24     return Extension(name = modname,
  25                      sources=[pyxfilename, 'hello.c'],
  26                      include_dirs=['/myinclude'] )
  27 def make_setup_args():
  28     return dict(script_args=["--compiler=mingw32"])
  29
  30 Extra dependencies can be defined by a <modulename>.pyxdep .
  31 See README.
  32
  33 Since Cython 0.11, the :mod:`pyximport` module also has experimental
  34 compilation support for normal Python modules.  This allows you to
  35 automatically run Cython on every .pyx and .py module that Python
  36 imports, including parts of the standard library and installed
  37 packages.  Cython will still fail to compile a lot of Python modules,
  38 in which case the import mechanism will fall back to loading the
  39 Python source modules instead.  The .py import mechanism is installed
  40 like this::
  41
  42     pyximport.install(pyimport = True)
  43
  44 Running this module as a top-level script will run a test and then print
  45 the documentation.
  46
  47 This code is based on the Py2.3+ import protocol as described in PEP 302.
  48 """
  49
  50 import sys
  51 import os
  52 import glob
  53 import imp
  54
  55 mod_name = "pyximport"
  56
  57 assert sys.hexversion >= 0x2030000, "need Python 2.3 or later"
  58
  59 PYX_EXT = ".pyx"
  60 PYXDEP_EXT = ".pyxdep"
  61 PYXBLD_EXT = ".pyxbld"
  62
  63 DEBUG_IMPORT = False
  64
  65 def _print(message, args):
  66     if args:
  67         message = message % args
  68     print(message)
  69
  70 def _debug(message, *args):
  71     if DEBUG_IMPORT:
  72         _print(message, args)
  73
  74 def _info(message, *args):
  75     _print(message, args)
  76
  77 # Performance problem: for every PYX file that is imported, we will
  78 # invoke the whole distutils infrastructure even if the module is
  79 # already built. It might be more efficient to only do it when the
  80 # mod time of the .pyx is newer than the mod time of the .so but
  81 # the question is how to get distutils to tell me the name of the .so
  82 # before it builds it. Maybe it is easy...but maybe the peformance
  83 # issue isn't real.
  84 def _load_pyrex(name, filename):
  85     "Load a pyrex file given a name and filename."
  86
  87 def get_distutils_extension(modname, pyxfilename, language_level=None):
  88 #    try:
  89 #        import hashlib
  90 #    except ImportError:
  91 #        import md5 as hashlib
  92 #    extra = "_" + hashlib.md5(open(pyxfilename).read()).hexdigest()
  93 #    modname = modname + extra
  94     extension_mod,setup_args = handle_special_build(modname, pyxfilename)
  95     if not extension_mod:
  96         if not isinstance(pyxfilename, str):
  97             # distutils is stupid in Py2 and requires exactly 'str'
  98             # => encode accidentally coerced unicode strings back to str
  99             pyxfilename = pyxfilename.encode(sys.getfilesystemencoding())
 100         from distutils.extension import Extension
 101         extension_mod = Extension(name = modname, sources=[pyxfilename])
 102         if language_level is not None:
 103             extension_mod.cython_directives = {'language_level': language_level}
 104     return extension_mod,setup_args
 105
 106 def handle_special_build(modname, pyxfilename):
 107     special_build = os.path.splitext(pyxfilename)[0] + PYXBLD_EXT
 108     ext = None
 109     setup_args={}
 110     if os.path.exists(special_build):
 111         # globls = {}
 112         # locs = {}
 113         # execfile(special_build, globls, locs)
 114         # ext = locs["make_ext"](modname, pyxfilename)
 115         mod = imp.load_source("XXXX", special_build, open(special_build))
 116         make_ext = getattr(mod,'make_ext',None)
 117         if make_ext:
 118             ext = make_ext(modname, pyxfilename)
 119             assert ext and ext.sources, ("make_ext in %s did not return Extension"
 120                                          % special_build)
 121         make_setup_args = getattr(mod,'make_setup_args',None)
 122         if make_setup_args:
 123             setup_args = make_setup_args()
 124             assert isinstance(setup_args,dict), ("make_setup_args in %s did not return a dict"
 125                                          % special_build)
 126         assert set or setup_args, ("neither make_ext nor make_setup_args %s"
 127                                          % special_build)
 128         ext.sources = [os.path.join(os.path.dirname(special_build), source)
 129                        for source in ext.sources]
 130     return ext, setup_args
 131
 132 def handle_dependencies(pyxfilename):
 133     testing = '_test_files' in globals()
 134     dependfile = os.path.splitext(pyxfilename)[0] + PYXDEP_EXT
 135
 136     # by default let distutils decide whether to rebuild on its own
 137     # (it has a better idea of what the output file will be)
 138
 139     # but we know more about dependencies so force a rebuild if
 140     # some of the dependencies are newer than the pyxfile.
 141     if os.path.exists(dependfile):
 142         depends = open(dependfile).readlines()
 143         depends = [depend.strip() for depend in depends]
 144
 145         # gather dependencies in the "files" variable
 146         # the dependency file is itself a dependency
 147         files = [dependfile]
 148         for depend in depends:
 149             fullpath = os.path.join(os.path.dirname(dependfile),
 150                                     depend)
 151             files.extend(glob.glob(fullpath))
 152
 153         # only for unit testing to see we did the right thing
 154         if testing:
 155             _test_files[:] = []  #$pycheck_no
 156
 157         # if any file that the pyxfile depends upon is newer than
 158         # the pyx file, 'touch' the pyx file so that distutils will
 159         # be tricked into rebuilding it.
 160         for file in files:
 161             from distutils.dep_util import newer
 162             if newer(file, pyxfilename):
 163                 _debug("Rebuilding %s because of %s", pyxfilename, file)
 164                 filetime = os.path.getmtime(file)
 165                 os.utime(pyxfilename, (filetime, filetime))
 166                 if testing:
 167                     _test_files.append(file)
 168
 169 def build_module(name, pyxfilename, pyxbuild_dir=None, inplace=False, language_level=None):
 170     assert os.path.exists(pyxfilename), (
 171         "Path does not exist: %s" % pyxfilename)
 172     handle_dependencies(pyxfilename)
 173
 174     extension_mod,setup_args = get_distutils_extension(name, pyxfilename, language_level)
 175     build_in_temp=pyxargs.build_in_temp
 176     sargs=pyxargs.setup_args.copy()
 177     sargs.update(setup_args)
 178     build_in_temp=sargs.pop('build_in_temp',build_in_temp)
 179
 180     import pyxbuild
 181     so_path = pyxbuild.pyx_to_dll(pyxfilename, extension_mod,
 182                                   build_in_temp=build_in_temp,
 183                                   pyxbuild_dir=pyxbuild_dir,
 184                                   setup_args=sargs,
 185                                   inplace=inplace,
 186                                   reload_support=pyxargs.reload_support)
 187     assert os.path.exists(so_path), "Cannot find: %s" % so_path
 188
 189     junkpath = os.path.join(os.path.dirname(so_path), name+"_*") #very dangerous with --inplace ? yes, indeed, trying to eat my files ;)
 190     junkstuff = glob.glob(junkpath)
 191     for path in junkstuff:
 192         if path!=so_path:
 193             try:
 194                 os.remove(path)
 195             except IOError:
 196                 _info("Couldn't remove %s", path)
 197
 198     return so_path
 199
 200 def load_module(name, pyxfilename, pyxbuild_dir=None, is_package=False,
 201                 build_inplace=False, language_level=None, so_path=None):
 202     try:
 203         if so_path is None:
 204             if is_package:
 205                 module_name = name + '.__init__'
 206             else:
 207                 module_name = name
 208             so_path = build_module(module_name, pyxfilename, pyxbuild_dir,
 209                                    inplace=build_inplace, language_level=language_level)
 210         mod = imp.load_dynamic(name, so_path)
 211         if is_package and not hasattr(mod, '__path__'):
 212             mod.__path__ = [os.path.dirname(so_path)]
 213         assert mod.__file__ == so_path, (mod.__file__, so_path)
 214     except Exception:
 215         if pyxargs.load_py_module_on_import_failure and pyxfilename.endswith('.py'):
 216             # try to fall back to normal import
 217             mod = imp.load_source(name, pyxfilename)
 218             assert mod.__file__ in (pyxfilename, pyxfilename+'c', pyxfilename+'o'), (mod.__file__, pyxfilename)
 219         else:
 220             import traceback
 221             raise ImportError("Building module %s failed: %s" %
 222                               (name,
 223                                traceback.format_exception_only(*sys.exc_info()[:2]))), None, sys.exc_info()[2]
 224     return mod
 225
 226
 227 # import hooks
 228
 229 class PyxImporter(object):
 230     """A meta-path importer for .pyx files.
 231     """
 232     def __init__(self, extension=PYX_EXT, pyxbuild_dir=None, inplace=False,
 233                  language_level=None):
 234         self.extension = extension
 235         self.pyxbuild_dir = pyxbuild_dir
 236         self.inplace = inplace
 237         self.language_level = language_level
 238
 239     def find_module(self, fullname, package_path=None):
 240         if fullname in sys.modules  and  not pyxargs.reload_support:
 241             return None  # only here when reload()
 242         try:
 243             fp, pathname, (ext,mode,ty) = imp.find_module(fullname,package_path)
 244             if fp: fp.close()  # Python should offer a Default-Loader to avoid this double find/open!
 245             if pathname and ty == imp.PKG_DIRECTORY:
 246                 pkg_file = os.path.join(pathname, '__init__'+self.extension)
 247                 if os.path.isfile(pkg_file):
 248                     return PyxLoader(fullname, pathname,
 249                         init_path=pkg_file,
 250                         pyxbuild_dir=self.pyxbuild_dir,
 251                         inplace=self.inplace,
 252                         language_level=self.language_level)
 253             if pathname and pathname.endswith(self.extension):
 254                 return PyxLoader(fullname, pathname,
 255                                  pyxbuild_dir=self.pyxbuild_dir,
 256                                  inplace=self.inplace,
 257                                  language_level=self.language_level)
 258             if ty != imp.C_EXTENSION: # only when an extension, check if we have a .pyx next!
 259                 return None
 260
 261             # find .pyx fast, when .so/.pyd exist --inplace
 262             pyxpath = os.path.splitext(pathname)[0]+self.extension
 263             if os.path.isfile(pyxpath):
 264                 return PyxLoader(fullname, pyxpath,
 265                                  pyxbuild_dir=self.pyxbuild_dir,
 266                                  inplace=self.inplace,
 267                                  language_level=self.language_level)
 268
 269             # .so/.pyd's on PATH should not be remote from .pyx's
 270             # think no need to implement PyxArgs.importer_search_remote here?
 271
 272         except ImportError:
 273             pass
 274
 275         # searching sys.path ...
 276
 277         #if DEBUG_IMPORT:  print "SEARCHING", fullname, package_path
 278         if '.' in fullname: # only when package_path anyway?
 279             mod_parts = fullname.split('.')
 280             module_name = mod_parts[-1]
 281         else:
 282             module_name = fullname
 283         pyx_module_name = module_name + self.extension
 284         # this may work, but it returns the file content, not its path
 285         #import pkgutil
 286         #pyx_source = pkgutil.get_data(package, pyx_module_name)
 287
 288         if package_path:
 289             paths = package_path
 290         else:
 291             paths = sys.path
 292         join_path = os.path.join
 293         is_file = os.path.isfile
 294         is_abs = os.path.isabs
 295         abspath = os.path.abspath
 296         #is_dir = os.path.isdir
 297         sep = os.path.sep
 298         for path in paths:
 299             if not path:
 300                 path = os.getcwd()
 301             elif not is_abs(path):
 302                 path = abspath(path)
 303             if is_file(path+sep+pyx_module_name):
 304                 return PyxLoader(fullname, join_path(path, pyx_module_name),
 305                                  pyxbuild_dir=self.pyxbuild_dir,
 306                                  inplace=self.inplace,
 307                                  language_level=self.language_level)
 308
 309         # not found, normal package, not a .pyx file, none of our business
 310         _debug("%s not found" % fullname)
 311         return None
 312
 313 class PyImporter(PyxImporter):
 314     """A meta-path importer for normal .py files.
 315     """
 316     def __init__(self, pyxbuild_dir=None, inplace=False, language_level=None):
 317         if language_level is None:
 318             language_level = sys.version_info[0]
 319         self.super = super(PyImporter, self)
 320         self.super.__init__(extension='.py', pyxbuild_dir=pyxbuild_dir, inplace=inplace,
 321                             language_level=language_level)
 322         self.uncompilable_modules = {}
 323         self.blocked_modules = ['Cython', 'pyxbuild', 'pyximport.pyxbuild',
 324                                 'distutils.extension', 'distutils.sysconfig']
 325
 326     def find_module(self, fullname, package_path=None):
 327         if fullname in sys.modules:
 328             return None
 329         if fullname.startswith('Cython.'):
 330             return None
 331         if fullname in self.blocked_modules:
 332             # prevent infinite recursion
 333             return None
 334         if _lib_loader.knows(fullname):
 335             return _lib_loader
 336         _debug("trying import of module '%s'", fullname)
 337         if fullname in self.uncompilable_modules:
 338             path, last_modified = self.uncompilable_modules[fullname]
 339             try:
 340                 new_last_modified = os.stat(path).st_mtime
 341                 if new_last_modified > last_modified:
 342                     # import would fail again
 343                     return None
 344             except OSError:
 345                 # module is no longer where we found it, retry the import
 346                 pass
 347
 348         self.blocked_modules.append(fullname)
 349         try:
 350             importer = self.super.find_module(fullname, package_path)
 351             if importer is not None:
 352                 if importer.init_path:
 353                     path = importer.init_path
 354                     real_name = fullname + '.__init__'
 355                 else:
 356                     path = importer.path
 357                     real_name = fullname
 358                 _debug("importer found path %s for module %s", path, real_name)
 359                 try:
 360                     so_path = build_module(
 361                         real_name, path,
 362                         pyxbuild_dir=self.pyxbuild_dir,
 363                         language_level=self.language_level,
 364                         inplace=self.inplace)
 365                     _lib_loader.add_lib(fullname, path, so_path,
 366                                         is_package=bool(importer.init_path))
 367                     return _lib_loader
 368                 except Exception:
 369                     if DEBUG_IMPORT:
 370                         import traceback
 371                         traceback.print_exc()
 372                     # build failed, not a compilable Python module
 373                     try:
 374                         last_modified = os.stat(path).st_mtime
 375                     except OSError:
 376                         last_modified = 0
 377                     self.uncompilable_modules[fullname] = (path, last_modified)
 378                     importer = None
 379         finally:
 380             self.blocked_modules.pop()
 381         return importer
 382
 383 class LibLoader(object):
 384     def __init__(self):
 385         self._libs = {}
 386
 387     def load_module(self, fullname):
 388         try:
 389             source_path, so_path, is_package = self._libs[fullname]
 390         except KeyError:
 391             raise ValueError("invalid module %s" % fullname)
 392         _debug("Loading shared library module '%s' from %s", fullname, so_path)
 393         return load_module(fullname, source_path, so_path=so_path, is_package=is_package)
 394
 395     def add_lib(self, fullname, path, so_path, is_package):
 396         self._libs[fullname] = (path, so_path, is_package)
 397
 398     def knows(self, fullname):
 399         return fullname in self._libs
 400
 401 _lib_loader = LibLoader()
 402
 403 class PyxLoader(object):
 404     def __init__(self, fullname, path, init_path=None, pyxbuild_dir=None,
 405                  inplace=False, language_level=None):
 406         _debug("PyxLoader created for loading %s from %s (init path: %s)",
 407                fullname, path, init_path)
 408         self.fullname = fullname
 409         self.path, self.init_path = path, init_path
 410         self.pyxbuild_dir = pyxbuild_dir
 411         self.inplace = inplace
 412         self.language_level = language_level
 413
 414     def load_module(self, fullname):
 415         assert self.fullname == fullname, (
 416             "invalid module, expected %s, got %s" % (
 417             self.fullname, fullname))
 418         if self.init_path:
 419             # package
 420             #print "PACKAGE", fullname
 421             module = load_module(fullname, self.init_path,
 422                                  self.pyxbuild_dir, is_package=True,
 423                                  build_inplace=self.inplace,
 424                                  language_level=self.language_level)
 425             module.__path__ = [self.path]
 426         else:
 427             #print "MODULE", fullname
 428             module = load_module(fullname, self.path,
 429                                  self.pyxbuild_dir,
 430                                  build_inplace=self.inplace,
 431                                  language_level=self.language_level)
 432         return module
 433
 434
 435 #install args
 436 class PyxArgs(object):
 437     build_dir=True
 438     build_in_temp=True
 439     setup_args={}   #None
 440
 441 ##pyxargs=None
 442
 443 def _have_importers():
 444     has_py_importer = False
 445     has_pyx_importer = False
 446     for importer in sys.meta_path:
 447         if isinstance(importer, PyxImporter):
 448             if isinstance(importer, PyImporter):
 449                 has_py_importer = True
 450             else:
 451                 has_pyx_importer = True
 452
 453     return has_py_importer, has_pyx_importer
 454
 455 def install(pyximport=True, pyimport=False, build_dir=None, build_in_temp=True,
 456             setup_args={}, reload_support=False,
 457             load_py_module_on_import_failure=False, inplace=False,
 458             language_level=None):
 459     """Main entry point. Call this to install the .pyx import hook in
 460     your meta-path for a single Python process.  If you want it to be
 461     installed whenever you use Python, add it to your sitecustomize
 462     (as described above).
 463
 464     You can pass ``pyimport=True`` to also install the .py import hook
 465     in your meta-path.  Note, however, that it is highly experimental,
 466     will not work for most .py files, and will therefore only slow
 467     down your imports.  Use at your own risk.
 468
 469     By default, compiled modules will end up in a ``.pyxbld``
 470     directory in the user's home directory.  Passing a different path
 471     as ``build_dir`` will override this.
 472
 473     ``build_in_temp=False`` will produce the C files locally. Working
 474     with complex dependencies and debugging becomes more easy. This
 475     can principally interfere with existing files of the same name.
 476     build_in_temp can be overriden by <modulename>.pyxbld/make_setup_args()
 477     by a dict item of 'build_in_temp'
 478
 479     ``setup_args``: dict of arguments for Distribution - see
 480     distutils.core.setup() . They are extended/overriden by those of
 481     <modulename>.pyxbld/make_setup_args()
 482
 483     ``reload_support``:  Enables support for dynamic
 484     reload(<pyxmodulename>), e.g. after a change in the Cython code.
 485     Additional files <so_path>.reloadNN may arise on that account, when
 486     the previously loaded module file cannot be overwritten.
 487
 488     ``load_py_module_on_import_failure``: If the compilation of a .py
 489     file succeeds, but the subsequent import fails for some reason,
 490     retry the import with the normal .py module instead of the
 491     compiled module.  Note that this may lead to unpredictable results
 492     for modules that change the system state during their import, as
 493     the second import will rerun these modifications in whatever state
 494     the system was left after the import of the compiled module
 495     failed.
 496
 497     ``inplace``: Install the compiled module next to the source file.
 498
 499     ``language_level``: The source language level to use: 2 or 3.
 500     The default is to use the language level of the current Python
 501     runtime for .py files and Py2 for .pyx files.
 502     """
 503     if not build_dir:
 504         build_dir = os.path.join(os.path.expanduser('~'), '.pyxbld')
 505
 506     global pyxargs
 507     pyxargs = PyxArgs()  #$pycheck_no
 508     pyxargs.build_dir = build_dir
 509     pyxargs.build_in_temp = build_in_temp
 510     pyxargs.setup_args = (setup_args or {}).copy()
 511     pyxargs.reload_support = reload_support
 512     pyxargs.load_py_module_on_import_failure = load_py_module_on_import_failure
 513
 514     has_py_importer, has_pyx_importer = _have_importers()
 515     py_importer, pyx_importer = None, None
 516
 517     if pyimport and not has_py_importer:
 518         py_importer = PyImporter(pyxbuild_dir=build_dir, inplace=inplace,
 519                                  language_level=language_level)
 520         # make sure we import Cython before we install the import hook
 521         import Cython.Compiler.Main, Cython.Compiler.Pipeline, Cython.Compiler.Optimize
 522         sys.meta_path.insert(0, py_importer)
 523
 524     if pyximport and not has_pyx_importer:
 525         pyx_importer = PyxImporter(pyxbuild_dir=build_dir, inplace=inplace,
 526                                    language_level=language_level)
 527         sys.meta_path.append(pyx_importer)
 528
 529     return py_importer, pyx_importer
 530
 531 def uninstall(py_importer, pyx_importer):
 532     """
 533     Uninstall an import hook.
 534     """
 535     try:
 536         sys.meta_path.remove(py_importer)
 537     except ValueError:
 538         pass
 539
 540     try:
 541         sys.meta_path.remove(pyx_importer)
 542     except ValueError:
 543         pass
 544
 545 # MAIN
 546
 547 def show_docs():
 548     import __main__
 549     __main__.__name__ = mod_name
 550     for name in dir(__main__):
 551         item = getattr(__main__, name)
 552         try:
 553             setattr(item, "__module__", mod_name)
 554         except (AttributeError, TypeError):
 555             pass
 556     help(__main__)
 557
 558 if __name__ == '__main__':
 559     show_docs()