Lib/distutils/command/build_ext.py

   1 """distutils.command.build_ext
   2
   3 Implements the Distutils 'build_ext' command, for building extension
   4 modules (currently limited to C extensions, should accommodate C++
   5 extensions ASAP)."""
   6
   7 # created 1999/08/09, Greg Ward
   8
   9 __revision__ = "$Id$"
  10
  11 import sys, os, string, re
  12 from types import *
  13 from distutils.core import Command
  14 from distutils.errors import *
  15 from distutils.sysconfig import customize_compiler
  16 from distutils.dep_util import newer_group
  17 from distutils.extension import Extension
  18
  19 # An extension name is just a dot-separated list of Python NAMEs (ie.
  20 # the same as a fully-qualified module name).
  21 extension_name_re = re.compile \
  22     (r'^[a-zA-Z_][a-zA-Z_0-9]*(\.[a-zA-Z_][a-zA-Z_0-9]*)*$')
  23
  24
  25 def show_compilers ():
  26     from distutils.ccompiler import show_compilers
  27     show_compilers()
  28
  29
  30 class build_ext (Command):
  31
  32     description = "build C/C++ extensions (compile/link to build directory)"
  33
  34     # XXX thoughts on how to deal with complex command-line options like
  35     # these, i.e. how to make it so fancy_getopt can suck them off the
  36     # command line and make it look like setup.py defined the appropriate
  37     # lists of tuples of what-have-you.
  38     #   - each command needs a callback to process its command-line options
  39     #   - Command.__init__() needs access to its share of the whole
  40     #     command line (must ultimately come from
  41     #     Distribution.parse_command_line())
  42     #   - it then calls the current command class' option-parsing
  43     #     callback to deal with weird options like -D, which have to
  44     #     parse the option text and churn out some custom data
  45     #     structure
  46     #   - that data structure (in this case, a list of 2-tuples)
  47     #     will then be present in the command object by the time
  48     #     we get to finalize_options() (i.e. the constructor
  49     #     takes care of both command-line and client options
  50     #     in between initialize_options() and finalize_options())
  51
  52     sep_by = " (separated by '%s')" % os.pathsep
  53     user_options = [
  54         ('build-lib=', 'b',
  55          "directory for compiled extension modules"),
  56         ('build-temp=', 't',
  57          "directory for temporary files (build by-products)"),
  58         ('inplace', 'i',
  59          "ignore build-lib and put compiled extensions into the source " +
  60          "directory alongside your pure Python modules"),
  61         ('include-dirs=', 'I',
  62          "list of directories to search for header files" + sep_by),
  63         ('define=', 'D',
  64          "C preprocessor macros to define"),
  65         ('undef=', 'U',
  66          "C preprocessor macros to undefine"),
  67         ('libraries=', 'l',
  68          "external C libraries to link with"),
  69         ('library-dirs=', 'L',
  70          "directories to search for external C libraries" + sep_by),
  71         ('rpath=', 'R',
  72          "directories to search for shared C libraries at runtime"),
  73         ('link-objects=', 'O',
  74          "extra explicit link objects to include in the link"),
  75         ('debug', 'g',
  76          "compile/link with debugging information"),
  77         ('force', 'f',
  78          "forcibly build everything (ignore file timestamps)"),
  79         ('compiler=', 'c',
  80          "specify the compiler type"),
  81         ('swig-cpp', None,
  82          "make SWIG create C++ files (default is C)"),
  83         ]
  84
  85     boolean_options = ['inplace', 'debug', 'force', 'swig-cpp']
  86
  87     help_options = [
  88         ('help-compiler', None,
  89          "list available compilers", show_compilers),
  90         ]
  91
  92     def initialize_options (self):
  93         self.extensions = None
  94         self.build_lib = None
  95         self.build_temp = None
  96         self.inplace = 0
  97         self.package = None
  98
  99         self.include_dirs = None
 100         self.define = None
 101         self.undef = None
 102         self.libraries = None
 103         self.library_dirs = None
 104         self.rpath = None
 105         self.link_objects = None
 106         self.debug = None
 107         self.force = None
 108         self.compiler = None
 109         self.swig_cpp = None
 110
 111
 112     def finalize_options (self):
 113         from distutils import sysconfig
 114
 115         self.set_undefined_options('build',
 116                                    ('build_lib', 'build_lib'),
 117                                    ('build_temp', 'build_temp'),
 118                                    ('compiler', 'compiler'),
 119                                    ('debug', 'debug'),
 120                                    ('force', 'force'))
 121
 122         if self.package is None:
 123             self.package = self.distribution.ext_package
 124
 125         self.extensions = self.distribution.ext_modules
 126
 127
 128         # Make sure Python's include directories (for Python.h, config.h,
 129         # etc.) are in the include search path.
 130         py_include = sysconfig.get_python_inc()
 131         plat_py_include = sysconfig.get_python_inc(plat_specific=1)
 132         if self.include_dirs is None:
 133             self.include_dirs = self.distribution.include_dirs or []
 134         if type(self.include_dirs) is StringType:
 135             self.include_dirs = string.split(self.include_dirs, os.pathsep)
 136
 137         # Put the Python "system" include dir at the end, so that
 138         # any local include dirs take precedence.
 139         self.include_dirs.append(py_include)
 140         if plat_py_include != py_include:
 141             self.include_dirs.append(plat_py_include)
 142
 143         if type(self.libraries) is StringType:
 144             self.libraries = [self.libraries]
 145
 146         # Life is easier if we're not forever checking for None, so
 147         # simplify these options to empty lists if unset
 148         if self.libraries is None:
 149             self.libraries = []
 150         if self.library_dirs is None:
 151             self.library_dirs = []
 152         elif type(self.library_dirs) is StringType:
 153             self.library_dirs = string.split(self.library_dirs, os.pathsep)
 154
 155         if self.rpath is None:
 156             self.rpath = []
 157         elif type(self.rpath) is StringType:
 158             self.rpath = string.split(self.rpath, os.pathsep)
 159
 160         # for extensions under windows use different directories
 161         # for Release and Debug builds.
 162         # also Python's library directory must be appended to library_dirs
 163         if os.name == 'nt':
 164             self.library_dirs.append(os.path.join(sys.exec_prefix, 'libs'))
 165             if self.debug:
 166                 self.build_temp = os.path.join(self.build_temp, "Debug")
 167             else:
 168                 self.build_temp = os.path.join(self.build_temp, "Release")
 169
 170         # for extensions under Cygwin Python's library directory must be
 171         # appended to library_dirs
 172         if sys.platform[:6] == 'cygwin':
 173             if string.find(sys.executable, sys.exec_prefix) != -1:
 174                 # building third party extensions
 175                 self.library_dirs.append(os.path.join(sys.prefix, "lib", "python" + sys.version[:3], "config"))
 176             else:
 177                 # building python standard extensions
 178                 self.library_dirs.append('.')
 179
 180         # The argument parsing will result in self.define being a string, but
 181         # it has to be a list of 2-tuples.  All the preprocessor symbols
 182         # specified by the 'define' option will be set to '1'.  Multiple
 183         # symbols can be separated with commas.
 184
 185         if self.define:
 186             defines = string.split(self.define, ',')
 187             self.define = map(lambda symbol: (symbol, '1'), defines)
 188
 189         # The option for macros to undefine is also a string from the
 190         # option parsing, but has to be a list.  Multiple symbols can also
 191         # be separated with commas here.
 192         if self.undef:
 193             self.undef = string.split(self.undef, ',')
 194
 195     # finalize_options ()
 196
 197
 198     def run (self):
 199
 200         from distutils.ccompiler import new_compiler
 201
 202         # 'self.extensions', as supplied by setup.py, is a list of
 203         # Extension instances.  See the documentation for Extension (in
 204         # distutils.extension) for details.
 205         #
 206         # For backwards compatibility with Distutils 0.8.2 and earlier, we
 207         # also allow the 'extensions' list to be a list of tuples:
 208         #    (ext_name, build_info)
 209         # where build_info is a dictionary containing everything that
 210         # Extension instances do except the name, with a few things being
 211         # differently named.  We convert these 2-tuples to Extension
 212         # instances as needed.
 213
 214         if not self.extensions:
 215             return
 216
 217         # If we were asked to build any C/C++ libraries, make sure that the
 218         # directory where we put them is in the library search path for
 219         # linking extensions.
 220         if self.distribution.has_c_libraries():
 221             build_clib = self.get_finalized_command('build_clib')
 222             self.libraries.extend(build_clib.get_library_names() or [])
 223             self.library_dirs.append(build_clib.build_clib)
 224
 225         # Setup the CCompiler object that we'll use to do all the
 226         # compiling and linking
 227         self.compiler = new_compiler(compiler=self.compiler,
 228                                      verbose=self.verbose,
 229                                      dry_run=self.dry_run,
 230                                      force=self.force)
 231         customize_compiler(self.compiler)
 232
 233         # And make sure that any compile/link-related options (which might
 234         # come from the command-line or from the setup script) are set in
 235         # that CCompiler object -- that way, they automatically apply to
 236         # all compiling and linking done here.
 237         if self.include_dirs is not None:
 238             self.compiler.set_include_dirs(self.include_dirs)
 239         if self.define is not None:
 240             # 'define' option is a list of (name,value) tuples
 241             for (name,value) in self.define:
 242                 self.compiler.define_macro(name, value)
 243         if self.undef is not None:
 244             for macro in self.undef:
 245                 self.compiler.undefine_macro(macro)
 246         if self.libraries is not None:
 247             self.compiler.set_libraries(self.libraries)
 248         if self.library_dirs is not None:
 249             self.compiler.set_library_dirs(self.library_dirs)
 250         if self.rpath is not None:
 251             self.compiler.set_runtime_library_dirs(self.rpath)
 252         if self.link_objects is not None:
 253             self.compiler.set_link_objects(self.link_objects)
 254
 255         # Now actually compile and link everything.
 256         self.build_extensions()
 257
 258     # run ()
 259
 260
 261     def check_extensions_list (self, extensions):
 262         """Ensure that the list of extensions (presumably provided as a
 263         command option 'extensions') is valid, i.e. it is a list of
 264         Extension objects.  We also support the old-style list of 2-tuples,
 265         where the tuples are (ext_name, build_info), which are converted to
 266         Extension instances here.
 267
 268         Raise DistutilsSetupError if the structure is invalid anywhere;
 269         just returns otherwise.
 270         """
 271         if type(extensions) is not ListType:
 272             raise DistutilsSetupError, \
 273                   "'ext_modules' option must be a list of Extension instances"
 274
 275         for i in range(len(extensions)):
 276             ext = extensions[i]
 277             if isinstance(ext, Extension):
 278                 continue                # OK! (assume type-checking done
 279                                         # by Extension constructor)
 280
 281             (ext_name, build_info) = ext
 282             self.warn(("old-style (ext_name, build_info) tuple found in "
 283                        "ext_modules for extension '%s'"
 284                        "-- please convert to Extension instance" % ext_name))
 285             if type(ext) is not TupleType and len(ext) != 2:
 286                 raise DistutilsSetupError, \
 287                       ("each element of 'ext_modules' option must be an "
 288                        "Extension instance or 2-tuple")
 289
 290             if not (type(ext_name) is StringType and
 291                     extension_name_re.match(ext_name)):
 292                 raise DistutilsSetupError, \
 293                       ("first element of each tuple in 'ext_modules' "
 294                        "must be the extension name (a string)")
 295
 296             if type(build_info) is not DictionaryType:
 297                 raise DistutilsSetupError, \
 298                       ("second element of each tuple in 'ext_modules' "
 299                        "must be a dictionary (build info)")
 300
 301             # OK, the (ext_name, build_info) dict is type-safe: convert it
 302             # to an Extension instance.
 303             ext = Extension(ext_name, build_info['sources'])
 304
 305             # Easy stuff: one-to-one mapping from dict elements to
 306             # instance attributes.
 307             for key in ('include_dirs',
 308                         'library_dirs',
 309                         'libraries',
 310                         'extra_objects',
 311                         'extra_compile_args',
 312                         'extra_link_args'):
 313                 val = build_info.get(key)
 314                 if val is not None:
 315                     setattr(ext, key, val)
 316
 317             # Medium-easy stuff: same syntax/semantics, different names.
 318             ext.runtime_library_dirs = build_info.get('rpath')
 319             if build_info.has_key('def_file'):
 320                 self.warn("'def_file' element of build info dict "
 321                           "no longer supported")
 322
 323             # Non-trivial stuff: 'macros' split into 'define_macros'
 324             # and 'undef_macros'.
 325             macros = build_info.get('macros')
 326             if macros:
 327                 ext.define_macros = []
 328                 ext.undef_macros = []
 329                 for macro in macros:
 330                     if not (type(macro) is TupleType and
 331                             1 <= len(macro) <= 2):
 332                         raise DistutilsSetupError, \
 333                               ("'macros' element of build info dict "
 334                                "must be 1- or 2-tuple")
 335                     if len(macro) == 1:
 336                         ext.undef_macros.append(macro[0])
 337                     elif len(macro) == 2:
 338                         ext.define_macros.append(macro)
 339
 340             extensions[i] = ext
 341
 342         # for extensions
 343
 344     # check_extensions_list ()
 345
 346
 347     def get_source_files (self):
 348         self.check_extensions_list(self.extensions)
 349         filenames = []
 350
 351         # Wouldn't it be neat if we knew the names of header files too...
 352         for ext in self.extensions:
 353             filenames.extend(ext.sources)
 354
 355         return filenames
 356
 357
 358     def get_outputs (self):
 359
 360         # Sanity check the 'extensions' list -- can't assume this is being
 361         # done in the same run as a 'build_extensions()' call (in fact, we
 362         # can probably assume that it *isn't*!).
 363         self.check_extensions_list(self.extensions)
 364
 365         # And build the list of output (built) filenames.  Note that this
 366         # ignores the 'inplace' flag, and assumes everything goes in the
 367         # "build" tree.
 368         outputs = []
 369         for ext in self.extensions:
 370             fullname = self.get_ext_fullname(ext.name)
 371             outputs.append(os.path.join(self.build_lib,
 372                                         self.get_ext_filename(fullname)))
 373         return outputs
 374
 375     # get_outputs ()
 376
 377     def build_extensions(self):
 378
 379         # First, sanity-check the 'extensions' list
 380         self.check_extensions_list(self.extensions)
 381
 382         for ext in self.extensions:
 383             self.build_extension(ext)
 384
 385     def build_extension(self, ext):
 386
 387         sources = ext.sources
 388         if sources is None or type(sources) not in (ListType, TupleType):
 389             raise DistutilsSetupError, \
 390                   ("in 'ext_modules' option (extension '%s'), " +
 391                    "'sources' must be present and must be " +
 392                    "a list of source filenames") % ext.name
 393         sources = list(sources)
 394
 395         fullname = self.get_ext_fullname(ext.name)
 396         if self.inplace:
 397             # ignore build-lib -- put the compiled extension into
 398             # the source tree along with pure Python modules
 399
 400             modpath = string.split(fullname, '.')
 401             package = string.join(modpath[0:-1], '.')
 402             base = modpath[-1]
 403
 404             build_py = self.get_finalized_command('build_py')
 405             package_dir = build_py.get_package_dir(package)
 406             ext_filename = os.path.join(package_dir,
 407                                         self.get_ext_filename(base))
 408         else:
 409             ext_filename = os.path.join(self.build_lib,
 410                                         self.get_ext_filename(fullname))
 411
 412         if not (self.force or newer_group(sources, ext_filename, 'newer')):
 413             self.announce("skipping '%s' extension (up-to-date)" %
 414                           ext.name)
 415             return
 416         else:
 417             self.announce("building '%s' extension" % ext.name)
 418
 419         # First, scan the sources for SWIG definition files (.i), run
 420         # SWIG on 'em to create .c files, and modify the sources list
 421         # accordingly.
 422         sources = self.swig_sources(sources)
 423
 424         # Next, compile the source code to object files.
 425
 426         # XXX not honouring 'define_macros' or 'undef_macros' -- the
 427         # CCompiler API needs to change to accommodate this, and I
 428         # want to do one thing at a time!
 429
 430         # Two possible sources for extra compiler arguments:
 431         #   - 'extra_compile_args' in Extension object
 432         #   - CFLAGS environment variable (not particularly
 433         #     elegant, but people seem to expect it and I
 434         #     guess it's useful)
 435         # The environment variable should take precedence, and
 436         # any sensible compiler will give precedence to later
 437         # command line args.  Hence we combine them in order:
 438         extra_args = ext.extra_compile_args or []
 439
 440         macros = ext.define_macros[:]
 441         for undef in ext.undef_macros:
 442             macros.append((undef,))
 443
 444         # XXX and if we support CFLAGS, why not CC (compiler
 445         # executable), CPPFLAGS (pre-processor options), and LDFLAGS
 446         # (linker options) too?
 447         # XXX should we use shlex to properly parse CFLAGS?
 448
 449         if os.environ.has_key('CFLAGS'):
 450             extra_args.extend(string.split(os.environ['CFLAGS']))
 451
 452         objects = self.compiler.compile(sources,
 453                                         output_dir=self.build_temp,
 454                                         macros=macros,
 455                                         include_dirs=ext.include_dirs,
 456                                         debug=self.debug,
 457                                         extra_postargs=extra_args)
 458
 459         # Now link the object files together into a "shared object" --
 460         # of course, first we have to figure out all the other things
 461         # that go into the mix.
 462         if ext.extra_objects:
 463             objects.extend(ext.extra_objects)
 464         extra_args = ext.extra_link_args or []
 465
 466
 467         self.compiler.link_shared_object(
 468             objects, ext_filename,
 469             libraries=self.get_libraries(ext),
 470             library_dirs=ext.library_dirs,
 471             runtime_library_dirs=ext.runtime_library_dirs,
 472             extra_postargs=extra_args,
 473             export_symbols=self.get_export_symbols(ext),
 474             debug=self.debug,
 475             build_temp=self.build_temp)
 476
 477
 478     def swig_sources (self, sources):
 479
 480         """Walk the list of source files in 'sources', looking for SWIG
 481         interface (.i) files.  Run SWIG on all that are found, and
 482         return a modified 'sources' list with SWIG source files replaced
 483         by the generated C (or C++) files.
 484         """
 485
 486         new_sources = []
 487         swig_sources = []
 488         swig_targets = {}
 489
 490         # XXX this drops generated C/C++ files into the source tree, which
 491         # is fine for developers who want to distribute the generated
 492         # source -- but there should be an option to put SWIG output in
 493         # the temp dir.
 494
 495         if self.swig_cpp:
 496             target_ext = '.cpp'
 497         else:
 498             target_ext = '.c'
 499
 500         for source in sources:
 501             (base, ext) = os.path.splitext(source)
 502             if ext == ".i":             # SWIG interface file
 503                 new_sources.append(base + target_ext)
 504                 swig_sources.append(source)
 505                 swig_targets[source] = new_sources[-1]
 506             else:
 507                 new_sources.append(source)
 508
 509         if not swig_sources:
 510             return new_sources
 511
 512         swig = self.find_swig()
 513         swig_cmd = [swig, "-python", "-dnone", "-ISWIG"]
 514         if self.swig_cpp:
 515             swig_cmd.append("-c++")
 516
 517         for source in swig_sources:
 518             target = swig_targets[source]
 519             self.announce("swigging %s to %s" % (source, target))
 520             self.spawn(swig_cmd + ["-o", target, source])
 521
 522         return new_sources
 523
 524     # swig_sources ()
 525
 526     def find_swig (self):
 527         """Return the name of the SWIG executable.  On Unix, this is
 528         just "swig" -- it should be in the PATH.  Tries a bit harder on
 529         Windows.
 530         """
 531
 532         if os.name == "posix":
 533             return "swig"
 534         elif os.name == "nt":
 535
 536             # Look for SWIG in its standard installation directory on
 537             # Windows (or so I presume!).  If we find it there, great;
 538             # if not, act like Unix and assume it's in the PATH.
 539             for vers in ("1.3", "1.2", "1.1"):
 540                 fn = os.path.join("c:\\swig%s" % vers, "swig.exe")
 541                 if os.path.isfile(fn):
 542                     return fn
 543             else:
 544                 return "swig.exe"
 545
 546         else:
 547             raise DistutilsPlatformError, \
 548                   ("I don't know how to find (much less run) SWIG "
 549                    "on platform '%s'") % os.name
 550
 551     # find_swig ()
 552
 553     # -- Name generators -----------------------------------------------
 554     # (extension names, filenames, whatever)
 555
 556     def get_ext_fullname (self, ext_name):
 557         if self.package is None:
 558             return ext_name
 559         else:
 560             return self.package + '.' + ext_name
 561
 562     def get_ext_filename (self, ext_name):
 563         r"""Convert the name of an extension (eg. "foo.bar") into the name
 564         of the file from which it will be loaded (eg. "foo/bar.so", or
 565         "foo\bar.pyd").
 566         """
 567
 568         from distutils.sysconfig import get_config_var
 569         ext_path = string.split(ext_name, '.')
 570         # extensions in debug_mode are named 'module_d.pyd' under windows
 571         so_ext = get_config_var('SO')
 572         if os.name == 'nt' and self.debug:
 573             return apply(os.path.join, ext_path) + '_d' + so_ext
 574         return apply(os.path.join, ext_path) + so_ext
 575
 576     def get_export_symbols (self, ext):
 577         """Return the list of symbols that a shared extension has to
 578         export.  This either uses 'ext.export_symbols' or, if it's not
 579         provided, "init" + module_name.  Only relevant on Windows, where
 580         the .pyd file (DLL) must export the module "init" function.
 581         """
 582
 583         initfunc_name = "init" + string.split(ext.name,'.')[-1]
 584         if initfunc_name not in ext.export_symbols:
 585             ext.export_symbols.append(initfunc_name)
 586         return ext.export_symbols
 587
 588     def get_libraries (self, ext):
 589         """Return the list of libraries to link against when building a
 590         shared extension.  On most platforms, this is just 'ext.libraries';
 591         on Windows, we add the Python library (eg. python20.dll).
 592         """
 593         # The python library is always needed on Windows.  For MSVC, this
 594         # is redundant, since the library is mentioned in a pragma in
 595         # config.h that MSVC groks.  The other Windows compilers all seem
 596         # to need it mentioned explicitly, though, so that's what we do.
 597         # Append '_d' to the python import library on debug builds.
 598         from distutils.msvccompiler import MSVCCompiler
 599         if sys.platform == "win32" and \
 600            not isinstance(self.compiler, MSVCCompiler):
 601             template = "python%d%d"
 602             if self.debug:
 603                 template = template + '_d'
 604             pythonlib = (template %
 605                    (sys.hexversion >> 24, (sys.hexversion >> 16) & 0xff))
 606             # don't extend ext.libraries, it may be shared with other
 607             # extensions, it is a reference to the original list
 608             return ext.libraries + [pythonlib]
 609         elif sys.platform[:6] == "cygwin":
 610             template = "python%d.%d"
 611             pythonlib = (template %
 612                    (sys.hexversion >> 24, (sys.hexversion >> 16) & 0xff))
 613             # don't extend ext.libraries, it may be shared with other
 614             # extensions, it is a reference to the original list
 615             return ext.libraries + [pythonlib]
 616         else:
 617             return ext.libraries
 618
 619 # class build_ext